Merge branch '3.0' into cpwu/3.0
This commit is contained in:
cpwu
commit
d97ded9488
|
|
@ -46,7 +46,7 @@ ENDIF ()
|
|||
|
||||
IF (TD_WINDOWS)
|
||||
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
|
||||
SET(COMMON_FLAGS "/w /D_WIN32")
|
||||
SET(COMMON_FLAGS "/w /D_WIN32 /Zi")
|
||||
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
|
||||
# IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
|
||||
# SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
|
||||
|
|
@ -71,8 +71,8 @@ ELSE ()
|
|||
ENDIF ()
|
||||
|
||||
IF (${SANITIZER} MATCHES "true")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3")
|
||||
MESSAGE(STATUS "Will compile with Address Sanitizer!")
|
||||
ELSE ()
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
|
||||
|
|
|
|||
|
|
@ -146,5 +146,6 @@ option(
|
|||
option(
|
||||
BUILD_WITH_INVERTEDINDEX
|
||||
"If use invertedIndex"
|
||||
ON
|
||||
OFF
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT})
|
|||
|
||||
# addr2line
|
||||
if(${BUILD_ADDR2LINE})
|
||||
cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
if(NOT ${TD_WINDOWS})
|
||||
cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
endif(NOT ${TD_WINDOWS})
|
||||
endif(${BUILD_ADDR2LINE})
|
||||
|
||||
# download dependencies
|
||||
|
|
@ -335,45 +337,47 @@ endif(${BUILD_WITH_SQLITE})
|
|||
|
||||
# addr2line
|
||||
if(${BUILD_ADDR2LINE})
|
||||
check_include_file( "sys/types.h" HAVE_SYS_TYPES_H)
|
||||
check_include_file( "sys/stat.h" HAVE_SYS_STAT_H )
|
||||
check_include_file( "inttypes.h" HAVE_INTTYPES_H )
|
||||
check_include_file( "stddef.h" HAVE_STDDEF_H )
|
||||
check_include_file( "stdlib.h" HAVE_STDLIB_H )
|
||||
check_include_file( "string.h" HAVE_STRING_H )
|
||||
check_include_file( "memory.h" HAVE_MEMORY_H )
|
||||
check_include_file( "strings.h" HAVE_STRINGS_H )
|
||||
check_include_file( "stdint.h" HAVE_STDINT_H )
|
||||
check_include_file( "unistd.h" HAVE_UNISTD_H )
|
||||
check_include_file( "sgidefs.h" HAVE_SGIDEFS_H )
|
||||
check_include_file( "stdafx.h" HAVE_STDAFX_H )
|
||||
check_include_file( "elf.h" HAVE_ELF_H )
|
||||
check_include_file( "libelf.h" HAVE_LIBELF_H )
|
||||
check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H)
|
||||
check_include_file( "alloca.h" HAVE_ALLOCA_H )
|
||||
check_include_file( "elfaccess.h" HAVE_ELFACCESS_H)
|
||||
check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H )
|
||||
check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H)
|
||||
check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
|
||||
check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H )
|
||||
set(VERSION 0.3.1)
|
||||
set(PACKAGE_VERSION "\"${VERSION}\"")
|
||||
configure_file(libdwarf/cmake/config.h.cmake config.h)
|
||||
file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
|
||||
add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
|
||||
set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf")
|
||||
if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H)
|
||||
target_link_libraries(libdwarf PUBLIC libelf)
|
||||
endif()
|
||||
target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR})
|
||||
file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT)
|
||||
string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}")
|
||||
add_library(addr2line STATIC "addr2line/addr2line.c")
|
||||
target_link_libraries(addr2line PUBLIC libdwarf dl z)
|
||||
target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
|
||||
if(NOT ${TD_WINDOWS})
|
||||
check_include_file( "sys/types.h" HAVE_SYS_TYPES_H)
|
||||
check_include_file( "sys/stat.h" HAVE_SYS_STAT_H )
|
||||
check_include_file( "inttypes.h" HAVE_INTTYPES_H )
|
||||
check_include_file( "stddef.h" HAVE_STDDEF_H )
|
||||
check_include_file( "stdlib.h" HAVE_STDLIB_H )
|
||||
check_include_file( "string.h" HAVE_STRING_H )
|
||||
check_include_file( "memory.h" HAVE_MEMORY_H )
|
||||
check_include_file( "strings.h" HAVE_STRINGS_H )
|
||||
check_include_file( "stdint.h" HAVE_STDINT_H )
|
||||
check_include_file( "unistd.h" HAVE_UNISTD_H )
|
||||
check_include_file( "sgidefs.h" HAVE_SGIDEFS_H )
|
||||
check_include_file( "stdafx.h" HAVE_STDAFX_H )
|
||||
check_include_file( "elf.h" HAVE_ELF_H )
|
||||
check_include_file( "libelf.h" HAVE_LIBELF_H )
|
||||
check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H)
|
||||
check_include_file( "alloca.h" HAVE_ALLOCA_H )
|
||||
check_include_file( "elfaccess.h" HAVE_ELFACCESS_H)
|
||||
check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H )
|
||||
check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H)
|
||||
check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
|
||||
check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H )
|
||||
set(VERSION 0.3.1)
|
||||
set(PACKAGE_VERSION "\"${VERSION}\"")
|
||||
configure_file(libdwarf/cmake/config.h.cmake config.h)
|
||||
file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
|
||||
add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
|
||||
set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf")
|
||||
if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H)
|
||||
target_link_libraries(libdwarf PUBLIC libelf)
|
||||
endif()
|
||||
target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR})
|
||||
file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT)
|
||||
string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
|
||||
file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}")
|
||||
add_library(addr2line STATIC "addr2line/addr2line.c")
|
||||
target_link_libraries(addr2line PUBLIC libdwarf dl z)
|
||||
target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
|
||||
endif(NOT ${TD_WINDOWS})
|
||||
endif(${BUILD_ADDR2LINE})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,6 @@ TDengine的主要功能如下：
|
|||
- [用 InfluxDB 开源的性能测试工具对比 InfluxDB 和 TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html)
|
||||
- [TDengine 与 OpenTSDB 对比测试](https://www.taosdata.com/blog/2019/08/21/621.html)
|
||||
- [TDengine 与 Cassandra 对比测试](https://www.taosdata.com/blog/2019/08/14/573.html)
|
||||
- [TDengine 与 InfluxDB 对比测试](https://www.taosdata.com/blog/2019/07/19/419.html)
|
||||
- [TDengine VS InfluxDB ，写入性能大 PK ！](https://www.taosdata.com/2021/11/05/3248.html)
|
||||
- [TDengine 和 InfluxDB 查询性能对比测试报告](https://www.taosdata.com/2022/02/22/5969.html)
|
||||
- [TDengine 与 InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse 等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf)
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速
|
|||
关键不同点在于：
|
||||
|
||||
1. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但性能要下降 30%左右。
|
||||
2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](reference/connector/cpp#数据订阅接口)等等。
|
||||
2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](/reference/connector/cpp#订阅和消费-api)等等。
|
||||
|
||||
## 安装客户端驱动 taosc
|
||||
|
||||
|
|
|
|||
|
|
@ -20,21 +20,21 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
|
|||
3. 数据库名最大长度为 33；
|
||||
4. 一条 SQL 语句的最大长度为 65480 个字符；
|
||||
5. 创建数据库时可用的参数有：
|
||||
- cache: [Description](/reference/config/#cache)
|
||||
- blocks: [Description](/reference/config/#blocks)
|
||||
- days: [Description](/reference/config/#days)
|
||||
- keep: [Description](/reference/config/#keep)
|
||||
- minRows: [Description](/reference/config/#minrows)
|
||||
- maxRows: [Description](/reference/config/#maxrows)
|
||||
- wal: [Description](/reference/config/#wallevel)
|
||||
- fsync: [Description](/reference/config/#fsync)
|
||||
- update: [Description](/reference/config/#update)
|
||||
- cacheLast: [Description](/reference/config/#cachelast)
|
||||
- replica: [Description](/reference/config/#replica)
|
||||
- quorum: [Description](/reference/config/#quorum)
|
||||
- maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb)
|
||||
- comp: [Description](/reference/config/#comp)
|
||||
- precision: [Description](/reference/config/#precision)
|
||||
- cache: [详细说明](/reference/config/#cache)
|
||||
- blocks: [详细说明](/reference/config/#blocks)
|
||||
- days: [详细说明](/reference/config/#days)
|
||||
- keep: [详细说明](/reference/config/#keep)
|
||||
- minRows: [详细说明](/reference/config/#minrows)
|
||||
- maxRows: [详细说明](/reference/config/#maxrows)
|
||||
- wal: [详细说明](/reference/config/#wallevel)
|
||||
- fsync: [详细说明](/reference/config/#fsync)
|
||||
- update: [详细说明](/reference/config/#update)
|
||||
- cacheLast: [详细说明](/reference/config/#cachelast)
|
||||
- replica: [详细说明](/reference/config/#replica)
|
||||
- quorum: [详细说明](/reference/config/#quorum)
|
||||
- maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb)
|
||||
- comp: [详细说明](/reference/config/#comp)
|
||||
- precision: [详细说明](/reference/config/#precision)
|
||||
6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。
|
||||
|
||||
:::
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
title: 性能优化
|
||||
---
|
||||
|
||||
因数据行 [update](/train-faq/faq/#update)、表删除、数据过期等原因，TDengine 的磁盘存储文件有可能出现数据碎片，影响查询操作的性能表现。从 2.1.3.0 版本开始，新增 SQL 指令 COMPACT 来启动碎片重整过程：
|
||||
|
||||
```sql
|
||||
COMPACT VNODES IN (vg_id1, vg_id2, ...)
|
||||
```
|
||||
|
||||
COMPACT 命令对指定的一个或多个 VGroup 启动碎片重整，系统会通过任务队列尽快安排重整操作的具体执行。COMPACT 指令所需的 VGroup id，可以通过 `SHOW VGROUPS;` 指令的输出结果获取；而且在 `SHOW VGROUPS;` 中会有一个 compacting 列，值为 2 时表示对应的 VGroup 处于排队等待进行重整的状态，值为 1 时表示正在进行碎片重整，为 0 时则表示并没有处于重整状态（未要求进行重整或已经完成重整）。
|
||||
|
||||
需要注意的是，碎片重整操作会大幅消耗磁盘 I/O。因此在重整进行期间，有可能会影响节点的写入和查询性能，甚至在极端情况下导致短时间的阻写。
|
||||
|
||||
## 存储参数优化
|
||||
|
||||
不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine 提供如下存储相关的系统配置参数（既可以作为 create database 指令的参数，也可以写在 taos.cfg 配置文件中用来设定创建新数据库时所采用的默认值）：
|
||||
|
||||
| # | 配置参数名称 | 单位 | 含义 | **取值范围** | **缺省值** |
|
||||
| --- | ------------ | ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ---------- |
|
||||
| 1 | days | 天 | 一个数据文件存储数据的时间跨度 | 1-3650 | 10 |
|
||||
| 2 | keep | 天 | （可通过 alter database 修改）数据库中数据保留的天数。 | 1-36500 | 3650 |
|
||||
| 3 | cache | MB | 内存块的大小 | 1-128 | 16 |
|
||||
| 4 | blocks | | （可通过 alter database 修改）每个 VNODE（TSDB）中有多少个 cache 大小的内存块。因此一个 VNODE 使用的内存大小粗略为（cache \* blocks）。 | 3-10000 | 6 |
|
||||
| 5 | quorum | | （可通过 alter database 修改）多副本环境下指令执行的确认数要求 | 1-2 | 1 |
|
||||
| 6 | minRows | | 文件块中记录的最小条数 | 10-1000 | 100 |
|
||||
| 7 | maxRows | | 文件块中记录的最大条数 | 200-10000 | 4096 |
|
||||
| 8 | comp | | （可通过 alter database 修改）文件压缩标志位 | 0：关闭，1:一阶段压缩，2:两阶段压缩 | 2 |
|
||||
| 9 | walLevel | | （作为 database 的参数时名为 wal；在 taos.cfg 中作为参数时需要写作 walLevel）WAL 级别 | 1：写 WAL，但不执行 fsync；2：写 WAL, 而且执行 fsync | 1 |
|
||||
| 10 | fsync | 毫秒 | 当 wal 设置为 2 时，执行 fsync 的周期。设置为 0，表示每次写入，立即执行 fsync。 | | 3000 |
|
||||
| 11 | replica | | （可通过 alter database 修改）副本个数 | 1-3 | 1 |
|
||||
| 12 | precision | | 时间戳精度标识（2.1.2.0 版本之前、2.0.20.7 版本之前在 taos.cfg 文件中不支持此参数。）（从 2.1.5.0 版本开始，新增对纳秒时间精度的支持） | ms 表示毫秒，us 表示微秒，ns 表示纳秒 | ms |
|
||||
| 13 | update | | 是否允许数据更新（从 2.1.7.0 版本开始此参数支持 0 ～ 2 的取值范围，在此之前取值只能是 [0, 1]；而 2.0.8.0 之前的版本在 SQL 指令中不支持此参数。） | 0：不允许；1：允许更新整行；2：允许部分列更新。 | 0 |
|
||||
| 14 | cacheLast | | （可通过 alter database 修改）是否在内存中缓存子表的最近数据（从 2.1.2.0 版本开始此参数支持 0 ～ 3 的取值范围，在此之前取值只能是 [0, 1]；而 2.0.11.0 之前的版本在 SQL 指令中不支持此参数。）（2.1.2.0 版本之前、2.0.20.7 版本之前在 taos.cfg 文件中不支持此参数。） | 0：关闭；1：缓存子表最近一行数据；2：缓存子表每一列的最近的非 NULL 值；3：同时打开缓存最近行和列功能 | 0 |
|
||||
|
||||
对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine 允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述 SQL：
|
||||
|
||||
```sql
|
||||
CREATE DATABASE demo DAYS 10 CACHE 32 BLOCKS 8 REPLICA 3 UPDATE 1;
|
||||
```
|
||||
|
||||
该 SQL 创建了一个库 demo, 每个数据文件存储 10 天数据，内存块为 32 兆字节，每个 VNODE 占用 8 个内存块，副本数为 3，允许更新，而其他参数与系统配置完全一致。
|
||||
|
||||
一个数据库创建成功后，仅部分参数可以修改并实时生效，其余参数不能修改：
|
||||
|
||||
| **参数名** | **能否修改** | **范围** | **修改语法示例** |
|
||||
| ----------- | ------------ | ---------------------------------------------------------- | -------------------------------------- |
|
||||
| name | | | |
|
||||
| create time | | | |
|
||||
| ntables | | | |
|
||||
| vgroups | | | |
|
||||
| replica | **YES** | 在线 dnode 数目为:<br/>1：1-1；<br/>2：1-2；<br/>\>=3：1-3 | ALTER DATABASE <dbname\> REPLICA _n_ |
|
||||
| quorum | **YES** | 1-2 | ALTER DATABASE <dbname\> QUORUM _n_ |
|
||||
| days | | | |
|
||||
| keep | **YES** | days-365000 | ALTER DATABASE <dbname\> KEEP _n_ |
|
||||
| cache | | | |
|
||||
| blocks | **YES** | 3-1000 | ALTER DATABASE <dbname\> BLOCKS _n_ |
|
||||
| minrows | | | |
|
||||
| maxrows | | | |
|
||||
| wal | | | |
|
||||
| fsync | | | |
|
||||
| comp | **YES** | 0-2 | ALTER DATABASE <dbname\> COMP _n_ |
|
||||
| precision | | | |
|
||||
| status | | | |
|
||||
| update | | | |
|
||||
| cachelast | **YES** | 0 \| 1 \| 2 \| 3 | ALTER DATABASE <dbname\> CACHELAST _n_ |
|
||||
|
||||
**说明：**在 2.1.3.0 版本之前，通过 ALTER DATABASE 语句修改这些参数后，需要重启服务器才能生效。
|
||||
|
||||
TDengine 集群中加入一个新的 dnode 时，涉及集群相关的一些参数必须与已有集群的配置相同，否则不能成功加入到集群中。会进行校验的参数如下：
|
||||
|
||||
- numOfMnodes：系统中管理节点个数。默认值：3。（2.0 版本从 2.0.20.11 开始、2.1 及以上版本从 2.1.6.0 开始，numOfMnodes 默认值改为 1。）
|
||||
- mnodeEqualVnodeNum: 一个 mnode 等同于 vnode 消耗的个数。默认值：4。
|
||||
- offlineThreshold: dnode 离线阈值，超过该时间将导致该 dnode 从集群中删除。单位为秒，默认值：86400\*10（即 10 天）。
|
||||
- statusInterval: dnode 向 mnode 报告状态时长。单位为秒，默认值：1。
|
||||
- maxTablesPerVnode: 每个 vnode 中能够创建的最大表个数。默认值：1000000。
|
||||
- maxVgroupsPerDb: 每个数据库中能够使用的最大 vgroup 个数。
|
||||
- arbitrator: 系统中裁决器的 endpoint，缺省为空。
|
||||
- timezone、locale、charset 的配置见客户端配置。（2.0.20.0 及以上的版本里，集群中加入新节点已不要求 locale 和 charset 参数取值一致）
|
||||
- balance：是否启用负载均衡。0：否，1：是。默认值：1。
|
||||
- flowctrl：是否启用非阻塞流控。0：否，1：是。默认值：1。
|
||||
- slaveQuery：是否启用 slave vnode 参与查询。0：否，1：是。默认值：1。
|
||||
- adjustMaster：是否启用 vnode master 负载均衡。0：否，1：是。默认值：1。
|
||||
|
||||
为方便调试，可通过 SQL 语句临时调整每个 dnode 的日志配置，系统重启后会失效：
|
||||
|
||||
```sql
|
||||
ALTER DNODE <dnode_id> <config>
|
||||
```
|
||||
|
||||
- dnode_id: 可以通过 SQL 语句"SHOW DNODES"命令获取
|
||||
- config: 要调整的日志参数，在如下列表中取值
|
||||
> resetlog 截断旧日志文件，创建一个新日志文件
|
||||
> debugFlag < 131 | 135 | 143 > 设置 debugFlag 为 131、135 或者 143
|
||||
|
||||
例如：
|
||||
|
||||
```
|
||||
alter dnode 1 debugFlag 135;
|
||||
```
|
||||
|
|
@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
|
|||
|
||||
`TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。
|
||||
|
||||
`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) 文档自行编写。
|
||||
`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [REST API](/reference/rest-api/) 文档自行编写。
|
||||
|
||||
本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`，并通过 `TDengine.Connector` 连接 TDengine 集群，进行数据写入、查询等基本操作。
|
||||
|
||||
|
|
|
|||
|
|
@ -1,256 +0,0 @@
|
|||
---
|
||||
sidebar_label: 数据复制模块设计
|
||||
title: 数据复制模块设计
|
||||
---
|
||||
|
||||
## 数据复制概述
|
||||
|
||||
数据复制(Replication)是指同一份数据在多个物理地点保存。它的目的是防止数据丢失，提高系统的高可用性(High Availability)，而且通过应用访问多个副本，提升数据查询性能。
|
||||
|
||||
在高可靠的大数据系统里，数据复制是必不可少的一大功能。数据复制又分为实时复制与非实时复制。实时复制是指任何数据的更新(包括数据的增加、删除、修改）操作，会被实时的复制到所有副本，这样任何一台机器宕机或网络出现故障，整个系统还能提供最新的数据，保证系统的正常工作。而非实时复制，是指传统的数据备份操作，按照固定的时间周期，将一份数据全量或增量复制到其他地方。如果主节点宕机，副本是很大可能没有最新数据，因此在有些场景是无法满足要求的。
|
||||
|
||||
TDengine面向的是物联网场景，需要支持数据的实时复制，来最大程度保证系统的可靠性。实时复制有两种方式，一种是异步复制，一种是同步复制。异步复制(Asynchronous Replication)是指数据由Master转发给Slave后，Master并不需要等待Slave回复确认，这种方式效率高，但有极小的概率会丢失数据。同步复制是指Master将数据转发给Slave后，需要等待Slave的回复确认，才会通知应用写入成功，这种方式效率偏低，但能保证数据绝不丢失。
|
||||
|
||||
数据复制是与数据存储（写入、读取）密切相关的，但两者又是相对独立，可以完全脱耦的。在TDengine系统中，有两种不同类型的数据，一种是时序数据，由TSDB模块负责；一种是元数据(Meta Data), 由MNODE负责。这两种性质不同的数据都需要同步功能。数据复制模块通过不同的实例启动配置参数，为这两种类型数据都提供同步功能。
|
||||
|
||||
在阅读本文之前，请先阅读《[TDengine 2.0 整体架构](/tdinternal/arch/)》，了解TDengine的集群设计和基本概念
|
||||
|
||||
特别注明：本文中提到数据更新操作包括数据的增加、删除与修改。
|
||||
|
||||
## 基本概念和定义
|
||||
|
||||
TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储元数据。但从同步数据复制的模块来看，两者没有本质的区别，因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgroup也指mnode group, 除非特别注明。
|
||||
|
||||
**版本(version)**：
|
||||
|
||||
一个虚拟节点组里多个虚拟节点互为备份，来保证数据的有效与可靠，是依靠虚拟节点组的数据版本号来维持的。TDengine2.0设计里，对于版本的定义如下：客户端发起增加、删除、修改的流程，无论是一条记录还是多条，只要是在一个请求里，这个数据更新请求被TDengine的一个虚拟节点收到后，经过合法性检查后，可以被写入系统时，就会被分配一个版本号。这个版本号在一个虚拟节点里从1开始，是单调连续递增的。无论这条记录是采集的时序数据还是meta data, 一样处理。当Master转发一个写入请求到slave时，必须带上版本号。一个虚拟节点将一数据更新请求写入WAL时，需要带上版本号。
|
||||
|
||||
不同虚拟节点组的数据版本号是完全独立的，互不相干的。版本号本质上是数据更新记录的transaction ID，但用来标识数据集的版本。
|
||||
|
||||
**角色(role)：**
|
||||
|
||||
一个虚拟节点可以是master, slave, unsynced或offline状态。
|
||||
|
||||
- master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master.
|
||||
- slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。
|
||||
- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务。
|
||||
- offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。
|
||||
|
||||
**Quorum:**
|
||||
|
||||
指数据写入成功所需要的确认数。对于异步复制，quorum设为1，具有master角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于2。原则上，Quorum >=1 并且 Quorum <= replication(副本数）。这个参数在启动一个同步模块实例时需要提供。
|
||||
|
||||
**WAL：**
|
||||
|
||||
TDengine的WAL(Write Ahead Log)与cassandra的commit log, mySQL的bin log, Postgres的WAL没本质区别。没有写入数据库文件，还保存在内存的数据都会先存在WAL。当数据已经成功写入数据库数据文件，相应的WAL会被删除。但需要特别指明的是，在TDengine系统里，有几点：
|
||||
|
||||
- 每个虚拟节点有自己独立的wal
|
||||
- WAL里包含而且仅仅包含来自客户端的数据更新操作，每个更新操作都会被打上一个版本号
|
||||
|
||||
**复制实例：**
|
||||
|
||||
复制模块只是一可执行的代码，复制实例是指正在运行的复制模块的一个实例，一个节点里，可以存在多个实例。原则上，一个节点有多少虚拟节点，就可以启动多少实例。对于副本数为1的场景，应用可以决定是否需要启动同步实例。应用启动一个同步模块的实例时，需要提供的就是虚拟节点组的配置信息，包括：
|
||||
|
||||
- 虚拟节点个数，即replication number
|
||||
- 各虚拟节点所在节点的信息，包括node的end point
|
||||
- quorum, 需要的数据写入成功的确认数
|
||||
- 虚拟节点的初始版本号
|
||||
|
||||
## 数据复制模块的基本工作原理
|
||||
|
||||
TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性算法比较一致。总结下来，有几点：
|
||||
|
||||
1. 一个vgroup里有一到多个虚拟节点，每个虚拟节点都有自己的角色
|
||||
2. 客户端只能向角色是master的虚拟节点发起数据更新操作，因为master具有最新版本的数据，如果向非Master发起数据更新操作，会直接收到错误
|
||||
3. 客户端可以向master, 也可以向角色是Slave的虚拟节点发起查询操作，但不能对unsynced的虚拟节点发起任何操作
|
||||
4. 如果master不存在，这个vgroup是不能对外提供数据更新和查询服务的
|
||||
5. master收到客户端的数据更新操作时，会将其转发给slave节点
|
||||
6. 一个虚拟节点的版本号比master低的时候，会发起数据恢复流程，成功后，才会成为slave
|
||||
|
||||
数据实时复制有三个主要流程：选主、数据转发、数据恢复。后续做详细讨论。
|
||||
|
||||
## 虚拟节点之间的网络连接
|
||||
|
||||
虚拟节点之间通过TCP进行连接，节点之间的状态交换、数据包的转发都是通过这个TCP连接(peerFd)进行。为避免竞争，两个虚拟节点之间的TCP连接，总是由IP地址(UINT32)小的节点作为TCP客户端发起。一旦TCP连接被中断，虚拟节点能通过TCP socket自动检测到，将对方标为offline。如果监测到任何错误（比如数据恢复流程），虚拟节点将主动重置该连接。
|
||||
|
||||
一旦作为客户端的节点连接不成或中断，它将周期性的每隔一秒钟去试图去连接一次。因为TCP本身有心跳机制，虚拟节点之间不再另行提供心跳。
|
||||
|
||||
如果一个unsynced节点要发起数据恢复流程，它与Master将建立起专有的TCP连接(syncFd)。数据恢复完成后，该连接会被关闭。而且为限制资源的使用，系统只容许一定数量(配置参数tsMaxSyncNum)的数据恢复的socket存在。如果超过这个数字，系统会将新的数据恢复请求延后处理。
|
||||
|
||||
任意一个节点，无论有多少虚拟节点，都会启动而且只会启动一个TCP server, 来接受来自其他虚拟节点的上述两类TCP的连接请求。当TCP socket建立起来，客户端侧发送的消息体里会带有vgId（全局唯一的vgroup ID), TCP 服务器侧会检查该vgId是否已经在该节点启动运行。如果已经启动运行，就接受其请求。如果不存在，就直接将连接请求关闭。在TDengine代码里，mnode group的vgId设置为1。
|
||||
|
||||
## 选主流程
|
||||
|
||||
当同一组的两个虚拟节点之间(vnode A, vnode B)建立连接后，他们互换status消息。status消息里包含本地存储的同一虚拟节点组内所有虚拟节点的role和version。
|
||||
|
||||
如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的连接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的连接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下：
|
||||
|
||||
1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced.
|
||||
2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程。
|
||||
3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。
|
||||
4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。
|
||||
5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version)。
|
||||
|
||||
具体的流程图如下：
|
||||
|
||||
|
||||
|
||||
选择Master的具体规则如下：
|
||||
|
||||
1. 如果只有一个副本，该副本永远就是master
|
||||
2. 所有副本都在线时，版本最高的被选为master
|
||||
3. 在线的虚拟节点数过半，而且有虚拟节点是slave的话，该虚拟节点自动成为master
|
||||
4. 对于2和3，如果多个虚拟节点满足成为master的要求，那么虚拟节点组的节点列表里，最前面的选为master
|
||||
|
||||
按照上面的规则，如果所有虚拟节点都是unsynced(比如全部重启），只有所有虚拟节点上线，才能选出master，该虚拟节点组才能开始对外提供服务。当一个虚拟节点的role发生改变时，sync模块回通过回调函数notifyRole通知应用。
|
||||
|
||||
## 数据转发流程
|
||||
|
||||
如果vnode A是master, vnode B是slave, vnode A能接受客户端的写请求，而vnode B不能。当vnode A收到写的请求后，遵循下面的流程：
|
||||
|
||||
|
||||
|
||||
1. 应用对写请求做基本的合法性检查，通过，则给该请求包打上一个版本号(version, 单调递增）
|
||||
2. 应用将打上版本号的写请求封装一个WAL Head, 写入WAL(Write Ahead Log)
|
||||
3. 应用调用API syncForwardToPeer，如果vnode B是slave状态，sync模块将包含WAL Head的数据包通过Forward消息发送给vnode B，否则就不转发。
|
||||
4. vnode B收到Forward消息后，调用回调函数writeToCache, 交给应用处理
|
||||
5. vnode B应用在写入成功后，都需要调用syncConfirmForward通知sync模块已经写入成功。
|
||||
6. 如果quorum大于1，vnode B需要等待应用的回复确认，收到确认后，vnode B发送Forward Response消息给node A。
|
||||
7. 如果quorum大于1，vnode A需要等待vnode B或其他副本对Forward消息的确认。
|
||||
8. 如果quorum大于1，vnode A收到quorum-1条确认消息后，调用回调函数confirmForward，通知应用写入成功。
|
||||
9. 如果quorum为1，上述6，7，8步不会发生。
|
||||
10. 如果要等待slave的确认，master会启动2秒的定时器（可配置），如果超时，则认为失败。
|
||||
|
||||
对于回复确认，sync模块提供的是异步回调函数，因此APP在调用syncForwardToPeer之后，无需等待，可以处理下一个操作。在Master与Slave的TCP连接管道里，可能有多个Forward消息，这些消息是严格按照应用提供的顺序排好的。对于Forward Response也是一样，TCP管道里存在多个，但都是排序好的。这个顺序，SYNC模块并没有做特别的事情，是由APP单线程顺序写来保证的(TDengine里每个vnode的写数据，都是单线程）。
|
||||
|
||||
## 数据恢复流程
|
||||
|
||||
如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。
|
||||
|
||||
1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mode里的文件，数量是固定的，对应于acct, user, db, table等文件。
|
||||
2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。
|
||||
3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。
|
||||
4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则
|
||||
* index 为0，表示获取最老的文件，同时修改index返回给sync模块。如果index不为0，表示获取指定位置的文件。
|
||||
* 如果name为空，表示sync想获取位于index位置的文件信息，包括magic, size。Master节点会这么调用
|
||||
* 如果name不为空，表示sync想获取指定文件名和index的信息，slave节点会这么调用
|
||||
* 如果某个index的文件不存在，magic返回0，表示文件已经是最后一个。因此整个系统里，文件的index必须是连续的一段整数。
|
||||
5. 当sync模块调用回调函数getWalInfo获得wal信息时，有如下规则
|
||||
* index为0，表示获得最老的WAL文件, 返回时，index更新为具体的数字
|
||||
* 如果返回0，表示这是最新的一个WAL文件，如果返回值是1，表示后面还有更新的WAL文件
|
||||
* 返回的文件名为空，那表示没有WAL文件
|
||||
6. 无论是getFileInfo, 还是getWalInfo, 只要获取出错（不是文件不存在），返回-1即可，系统会报错，停止同步
|
||||
|
||||
整个数据恢复流程分为两大步骤，第一步，先恢复archived data(file), 然后恢复wal。具体流程如下：
|
||||
|
||||
|
||||
|
||||
1. 通过已经建立的TCP连接，发送sync req给master节点
|
||||
2. master收到sync req后，以client的身份，向vnode B主动建立一新的专用于同步的TCP连接（syncFd)
|
||||
3. 新的TCP连接建立成功后，master将开始retrieve流程，对应的，vnode B将同步启动restore流程
|
||||
4. Retrieve/Restore流程里，先处理所有archived data (vnode里的data, head, last文件），后处理WAL data。
|
||||
5. 对于archived data，master将通过回调函数getFileInfo获取数据文件的基本信息，包括文件名、magic以及文件大小。
|
||||
6. master 将获得的文件名、magic以及文件大小发给vnode B
|
||||
7. vnode B将回调函数getFile获得magic和文件大小，如果两者一致，就认为无需同步，如果两者不一致 ，就认为需要同步。vnode B将结果通过消息FileAck发回master
|
||||
8. 如果文件需要同步，master就调用sendfile把整个文件发往vnode B
|
||||
9. 如果文件不需要同步，master(vnode A)就重复5，6，7，8，直到所有文件被处理完
|
||||
|
||||
对于WAL同步，流程如下：
|
||||
|
||||
1. master节点调用回调函数getWalInfo，获取WAL的文件名。
|
||||
2. 如果getWalInfo返回值大于0，表示该文件还不是最后一个WAL，因此master调用sendfile一下把该文件发送给vnode B
|
||||
3. 如果getWalInfo返回时为0，表示该文件是最后一个WAL，因为文件可能还处于写的状态中，sync模块要根据WAL Head的定义逐条读出记录，然后发往vnode B。
|
||||
4. vnode A读取TCP连接传来的数据，按照WAL Head，逐条读取，如果版本号比现有的大，调用回调函数writeToCache，交给应用处理。如果小，直接扔掉。
|
||||
5. 上述流程循环，直到所有WAL文件都被处理完。处理完后，master就会将新来的数据包通过Forward消息转发给slave。
|
||||
|
||||
从同步文件启动起，sync模块会通过inotify监控所有处理过的file以及wal。一旦发现被处理过的文件有更新变化，同步流程将中止，会重新启动。因为有可能落盘操作正在进行（比如历史数据导入，内存数据落盘），把已经处理过的文件进行了修改，需要重新同步才行。
|
||||
|
||||
对于最后一个WAL (LastWal)的处理逻辑有点复杂，因为这个文件往往是打开写的状态，有很多场景需要考虑，比如：
|
||||
|
||||
- LastWal文件size在增长，需要重新读；
|
||||
- LastWal文件虽然已经打开写，但内容为空；
|
||||
- LastWal文件已经被关闭，应用生成了新的Last WAL文件;
|
||||
- LastWal文件没有被关闭，但数据落盘的原因，没有读到完整的一条记录;
|
||||
- LastWal文件没有被关闭，但数据落盘的原因，还有部分记录暂时读取不到；
|
||||
|
||||
sync模块通过inotify监控LastWal文件的更新和关闭操作。而且在确认已经尽可能读完LastWal的数据后，会将对方同步状态设置为SYNC_CACHE。该状态下，master节点会将新的记录转发给vnode B，而此时vnode B并没有完成同步，需要把这些转发包先存在recv buffer里，等WAL处理完后，vnode A再把recv buffer里的数据包通过回调writeToCache交给应用处理。
|
||||
|
||||
等vnode B把这些buffered forwards处理完，同步流程才算结束，vnode B正式变为slave。
|
||||
|
||||
## Master分布均匀性问题
|
||||
|
||||
因为Master负责写、转发，消耗的资源会更多，因此Master在整个集群里分布均匀比较理想。
|
||||
|
||||
但在TDengine的设计里，如果多个虚拟节点都符合master条件，TDengine选在列表中最前面的做Master, 这样是否导致在集群里，Master数量的分布不均匀问题呢？这取决于应用的设计。
|
||||
|
||||
给一个具体例子，系统里仅仅有三个节点，IP地址分别为IP1, IP2, IP3. 在各个节点上，TDengine创建了多个虚拟节点组，每个虚拟节点组都有三个副本。如果三个副本的顺序在所有虚拟节点组里都是IP1, IP2, IP3, 那毫无疑问，master将集中在IP1这个节点，这是我们不想看到的。
|
||||
|
||||
但是，如果在创建虚拟节点组时，增加随机性，这个问题就不存在了。比如在vgroup 1, 顺序是IP1, IP2, IP3, 在vgroup 2里，顺序是IP2, IP3, IP1, 在vgroup 3里，顺序是IP3, IP1, IP2。最后master的分布会是均匀的。
|
||||
|
||||
因此在创建一个虚拟节点组时，应用需要保证节点的顺序是round robin或完全随机。
|
||||
|
||||
## 少数虚拟节点写入成功的问题
|
||||
|
||||
在某种情况下，写入成功的确认数大于0，但小于配置的Quorum, 虽然有虚拟节点数据更新成功，master仍然会认为数据更新失败，并通知客户端写入失败。
|
||||
|
||||
这个时候，系统存在数据不一致的问题，因为有的虚拟节点已经写入成功，而有的写入失败。一个处理方式是，Master重置(reset)与其他虚拟节点的连接，该虚拟节点组将自动进入选举流程。按照规则，已经成功写入数据的虚拟节点将成为新的master，组内的其他虚拟节点将从master那里恢复数据。
|
||||
|
||||
因为写入失败，客户端会重新写入数据。但对于TDengine而言，是OK的。因为时序数据都是有时间戳的，时间戳相同的数据更新操作，第一次会执行，但第二次会自动扔掉。对于Meta Data(增加、删除库、表等等）的操作，也是OK的。一张表、库已经被创建或删除，再创建或删除，不会被执行的。
|
||||
|
||||
在TDengine的设计里，虚拟节点与虚拟节点之间，是一个TCP连接，是一个pipeline，数据块一个接一个按顺序在这个pipeline里等待处理。一旦某个数据块的处理失败，这个连接会被重置，后续的数据块的处理都会失败。因此不会存在Pipeline里一个数据块更新失败，但下一个数据块成功的可能。
|
||||
|
||||
## Split Brain的问题
|
||||
|
||||
选举流程中，有个强制要求，那就是一定有超过半数的虚拟节点在线。但是如果replication正好是偶数，这个时候，完全可能存在splt brain问题。
|
||||
|
||||
为解决这个问题，TDengine提供Arbitrator的解决方法。Arbitrator是一个节点，它的任务就是接受任何虚拟节点的连接请求，并保持它。
|
||||
|
||||
在启动复制模块实例时，在配置参数中，应用可以提供Arbitrator的IP地址。如果是奇数个副本，复制模块不会与这个arbitrator去建立连接，但如果是偶数个副本，就会主动去建立连接。
|
||||
|
||||
Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系统时，会在bin目录生成。命令行参数“-？”查看可以配置的参数，比如绑定的IP地址，监听的端口号。
|
||||
|
||||
## 与RAFT相比的异同
|
||||
|
||||
数据一致性协议流行的有两种，Paxos与Raft. 本设计的实现与Raft有很多类同之处，下面做一些比较
|
||||
|
||||
相同之处：
|
||||
|
||||
- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程。
|
||||
- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的。
|
||||
- 数据转发流程完全一样，Master(leader)需要等待回复确认。
|
||||
- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步。
|
||||
|
||||
不同之处：
|
||||
|
||||
- 选举流程不一样：Raft里任何一个节点是candidate时，主动向其他节点发出vote request，如果超过半数回答Yes，这个candidate就成为Leader，开始一个新的term。而TDengine的实现里，节点上线、离线或角色改变都会触发状态消息在节点组内传播，等节点组里状态稳定一致之后才触发选举流程，因为状态稳定一致，基于同样的状态信息，每个节点做出的决定会是一致的，一旦某个节点符合成为master的条件，无需其他节点认可，它会自动将自己设为master。TDengine里，任何一个节点检测到其他节点或自己的角色发生改变，就会向节点组内其他节点进行广播。Raft里不存在这样的机制，因此需要投票来解决。
|
||||
- 对WAL的一条记录，Raft用term + index来做唯一标识。但TDengine只用version（类似index)，在TDengine实现里，仅仅用version是完全可行的, 因为TDengine的选举机制，没有term的概念。
|
||||
|
||||
如果整个虚拟节点组全部宕机，重启，但不是所有虚拟节点都上线，这个时候TDengine是不会选出master的，因为未上线的节点有可能有最高version的数据。而RAFT协议，只要超过半数上线，就会选出Leader。
|
||||
|
||||
## Meta Data的数据复制
|
||||
|
||||
TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析。
|
||||
|
||||
TDengine里Meta Data包括以下：
|
||||
|
||||
- account 信息
|
||||
- 一个account下面，可以有多个user, 多个DB
|
||||
- 一个DB下面有多个vgroup
|
||||
- 一个DB下面有多个stable
|
||||
- 一个vgroup下面有多个table
|
||||
- 整个系统有多个mnode, dnode
|
||||
- 一个dnode可以有多个vnode
|
||||
|
||||
上述的account, user, DB, vgroup, table, stable, mnode, dnode都有自己的属性，这些属性是TDengine自己定义的，不会开放给用户进行修改。这些Meta Data的查询都比较简单，都可以采用key-value模型进行存储。这些Meta Data还具有几个特点：
|
||||
|
||||
1. 上述的Meta Data之间有一定的层级关系，比如必须先创建DB，才能创建table, stable。只有先创建dnode，才可能创建vnode, 才可能创建vgroup。因此他们创建的顺序是绝对不能错的。
|
||||
2. 在客户端应用的数据更新操作得到TDengine服务器侧确认后，所执行的数据更新操作绝对不能丢失。否则会造成客户端应用与服务器的数据不一致。
|
||||
3. 上述的Meta Data是容许重复操作的。比如插入新记录后，再插入一次，删除一次后，再删除一次，更新一次后，再更新一次，不会对系统产生任何影响，不会改变系统任何状态。
|
||||
|
||||
对于特点1，本设计里，数据的写入是单线程的，按照到达的先后顺序，给每个数据更新操作打上版本号，版本号大的记录一定是晚于版本号小的写入系统，数据写入顺序是100%保证的，绝对不会让版本号大的记录先写入。复制过程中，数据块的转发也是严格按照顺序进行的，因此TDengine的数据复制设计是能保证Meta Data的创建顺序的。
|
||||
|
||||
对于特点2，只要Quorum数设置等于replica，那么一定能保证回复确认过的数据更新操作不会在服务器侧丢失。即使某节点永不起来，只要超过一半的节点还是online, 查询服务不会受到任何影响。这时，如果某个节点离线超过一定时长，系统可以自动补充新的节点，以保证在线的节点数在绝大部分时间是100%的。
|
||||
|
||||
对于特点3，完全可能发生，服务器确实持久化存储了某一数据更新操作，但客户端应用出了问题，认为操作不成功，它会重新发起操作。但对于Meta Data而言，没有关系，客户端可以再次发起同样的操作，不会有任何影响。
|
||||
|
||||
总结来看，只要quorum设置大于一，本数据复制的设计是能满足Meta Data的需求的。目前，还没有发现漏洞。
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
---
|
||||
sidebar_label: taosd 的设计
|
||||
title: taosd的设计
|
||||
---
|
||||
|
||||
逻辑上，TDengine 系统包含 dnode，taosc 和 App，dnode 是服务器侧执行代码 taosd 的一个运行实例，因此 taosd 是 TDengine 的核心，本文对 taosd 的设计做一简单的介绍，模块内的实现细节请见其他文档。
|
||||
|
||||
## 系统模块图
|
||||
|
||||
taosd 包含 rpc，dnode，vnode，tsdb，query，cq，sync，wal，mnode，http，monitor 等模块，具体如下图：
|
||||
|
||||
|
||||
|
||||
taosd 的启动入口是 dnode 模块，dnode 然后启动其他模块，包括可选配置的 http，monitor 模块。taosc 或 dnode 之间交互的消息都是通过 rpc 模块进行，dnode 模块根据接收到的消息类型，将消息分发到 vnode 或 mnode 的消息队列，或由 dnode 模块自己消费。dnode 的工作线程（worker）消费消息队列里的消息，交给 mnode 或 vnode 进行处理。下面对各个模块做简要说明。
|
||||
|
||||
## RPC 模块
|
||||
|
||||
该模块负责 taosd 与 taosc，以及其他数据节点之间的通讯。TDengine 没有采取标准的 HTTP 或 gRPC 等第三方工具，而是实现了自己的通讯模块 RPC。
|
||||
|
||||
考虑到物联网场景下，数据写入的包一般不大，因此除支持 TCP 连接之外，RPC 还支持 UDP 连接。当数据包小于 15K 时，RPC 将采用 UDP 方式进行连接，否则将采用 TCP 连接。对于查询类的消息，RPC 不管包的大小，总是采取 TCP 连接。对于 UDP 连接，RPC 实现了自己的超时、重传、顺序检查等机制，以保证数据可靠传输。
|
||||
|
||||
RPC 模块还提供数据压缩功能，如果数据包的字节数超过系统配置参数 compressMsgSize，RPC 在传输中将自动压缩数据，以节省带宽。
|
||||
|
||||
为保证数据的安全和数据的 integrity，RPC 模块采用 MD5 做数字签名，对数据的真实性和完整性进行认证。
|
||||
|
||||
## DNODE 模块
|
||||
|
||||
该模块是整个 taosd 的入口，它具体负责如下任务：
|
||||
|
||||
- 系统的初始化，包括
|
||||
- 从文件 taos.cfg 读取系统配置参数，从文件 dnodeCfg.json 读取数据节点的配置参数；
|
||||
- 启动 RPC 模块，并建立起与 taosc 通讯的 server 连接，与其他数据节点通讯的 server 连接；
|
||||
- 启动并初始化 dnode 的内部管理，该模块将扫描该数据节点已有的 vnode ，并打开它们；
|
||||
- 初始化可配置的模块，如 mnode，http，monitor 等。
|
||||
- 数据节点的管理，包括
|
||||
- 定时的向 mnode 发送 status 消息，报告自己的状态；
|
||||
- 根据 mnode 的指示，创建、改变、删除 vnode；
|
||||
- 根据 mnode 的指示，修改自己的配置参数；
|
||||
- 消息的分发、消费，包括
|
||||
- 为每一个 vnode 和 mnode 的创建并维护一个读队列、一个写队列；
|
||||
- 将从 taosc 或其他数据节点来的消息，根据消息类型，将其直接分发到不同的消息队列，或由自己的管理模块直接消费；
|
||||
- 维护一个读的线程池，消费读队列的消息，交给 vnode 或 mnode 处理。为支持高并发，一个读线程（worker）可以消费多个队列的消息，一个读队列可以由多个 worker 消费；
|
||||
- 维护一个写的线程池，消费写队列的消息，交给 vnode 或 mnode 处理。为保证写操作的序列化，一个写队列只能由一个写线程负责，但一个写线程可以负责多个写队列。
|
||||
|
||||
taosd 的消息消费由 dnode 通过读写线程池进行控制，是系统的中枢。该模块内的结构体图如下：
|
||||
|
||||
|
||||
|
||||
## VNODE 模块
|
||||
|
||||
vnode 是一独立的数据存储查询逻辑单元，但因为一个 vnode 只能容许一个 DB ，因此 vnode 内部没有 account，DB，user 等概念。为实现更好的模块化、封装以及未来的扩展，它有很多子模块，包括负责存储的 TSDB，负责查询的 query，负责数据复制的 sync，负责数据库日志的的 WAL，负责连续查询的 cq（continuous query），负责事件触发的流计算的 event 等模块，这些子模块只与 vnode 模块发生关系，与其他模块没有任何调用关系。模块图如下：
|
||||
|
||||
|
||||
|
||||
vnode 模块向下，与 dnodeVRead，dnodeVWrite 发生互动，向上，与子模块发生互动。它主要的功能有：
|
||||
|
||||
- 协调各个子模块的互动。各个子模块之间都不直接调用，都需要通过 vnode 模块进行；
|
||||
- 对于来自 taosc 或 mnode 的写操作，vnode 模块将其分解为写日志（WAL），转发（sync），本地存储（TSDB）子模块的操作；
|
||||
- 对于查询操作，分发到 query 模块进行。
|
||||
|
||||
一个数据节点里有多个 vnode，因此 vnode 模块是有多个运行实例的。每个运行实例是完全独立的。
|
||||
|
||||
vnode 与其子模块是通过 API 直接调用，而不是通过消息队列传递。而且各个子模块只与 vnode 模块有交互，不与 dnode，rpc 等模块发生任何直接关联。
|
||||
|
||||
## MNODE 模块
|
||||
|
||||
mnode 是整个系统的大脑，负责整个系统的资源调度，负责 meta data 的管理与存储。
|
||||
|
||||
一个运行的系统里，只有一个 mnode，但它有多个副本（由系统配置参数 numOfMnodes 控制）。这些副本分布在不同的 dnode 里，目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式，以确保数据的一致性，确保数据不会丢失。这些副本会自动选举一个 Master，其他副本是 slave。所有数据更新类的操作，都只能在 master 上进行，而查询类的可以在 slave 节点上进行。代码实现上，同步模块与 vnode 共享，但 mnode 被分配一个特殊的 vgroup ID: 1，而且 quorum 大于 1。整个集群系统是由多个 dnode 组成的，运行的 mnode 的副本数不可能超过 dnode 的个数，但不会超过配置的副本数。如果某个 mnode 副本宕机一段时间，只要超过半数的 mnode 副本仍在运行，运行的 mnode 会自动根据整个系统的资源情况，在其他 dnode 里再启动一个 mnode，以保证运行的副本数。
|
||||
|
||||
各个 dnode 通过信息交换，保存有 mnode 各个副本的 End Point 列表，并向其中的 master 节点定时（间隔由系统配置参数 statusInterval 控制）发送 status 消息，消息体里包含该 dnode 的 CPU、内存、剩余存储空间、vnode 个数，以及各个 vnode 的状态（存储空间、原始数据大小、记录条数、角色等）。这样 mnode 就了解整个系统的资源情况，如果用户创建新的表，就可以决定需要在哪个 dnode 创建；如果增加或删除 dnode，或者监测到某 dnode 数据过热、或离线太长，就可以决定需要挪动那些 vnode，以实现负载均衡。
|
||||
|
||||
mnode 里还负责 account，user，DB，stable，table，vgroup，dnode 的创建、删除与更新。mnode 不仅把这些 entity 的 meta data 保存在内存，还做持久化存储。但为节省内存，各个表的标签值不保存在 mnode（保存在 vnode），而且子表不维护自己的 schema，而是与 stable 共享。为减小 mnode 的查询压力，taosc 会缓存 table、stable 的 schema。对于查询类的操作，各个 slave mnode 也可以提供，以减轻 master 压力。
|
||||
|
||||
## TSDB 模块
|
||||
|
||||
TSDB 模块是 vnode 中的负责快速高并发地存储和读取属于该 vnode 的表的元数据及采集的时序数据的引擎。除此之外，TSDB 还提供了表结构的修改、表标签值的修改等功能。TSDB 提供 API 供 vnode 和 query 等模块调用。TSDB 中存储了两类数据，1：元数据信息；2：时序数据
|
||||
|
||||
### 元数据信息
|
||||
|
||||
TSDB 中存储的元数据包含属于其所在的 vnode 中表的类型，schema 的定义等。对于超级表和超级表下的子表而言，又包含了 tag 的 schema 定义以及子表的 tag 值等。对于元数据信息而言，TSDB 就相当于一个全内存的 KV 型数据库，属于该 vnode 的表对象全部在内存中，方便快速查询表的信息。除此之外，TSDB 还对其中的子表，按照 tag 的第一列取值做了全内存的索引，大大加快了对于标签的过滤查询。TSDB 中的元数据的最新状态在落盘时，会以追加（append-only）的形式，写入到 meta 文件中。meta 文件只进行追加操作，即便是元数据的删除，也会以一条记录的形式写入到文件末尾。TSDB 也提供了对于元数据的修改操作，如表 schema 的修改，tag schema 的修改以及 tag 值的修改等。
|
||||
|
||||
### 时序数据
|
||||
|
||||
每个 TSDB 在创建时，都会事先分配一定量的内存缓冲区，且内存缓冲区的大小可配可修改。表采集的时序数据，在写入 TSDB 时，首先以追加的方式写入到分配的内存缓冲区中，同时建立基于时间戳的内存索引，方便快速查询。当内存缓冲区的数据积累到一定的程度时（达到内存缓冲区总大小的 1/3），则会触发落盘操作，将缓冲区中的数据持久化到硬盘文件上。时序数据在内存缓冲区中是以行（row）的形式存储的。
|
||||
|
||||
而时序数据在写入到 TSDB 的数据文件时，是以列（column）的形式存储的。TSDB 中的数据文件包含多个数据文件组，每个数据文件组中又包含 .head、.data 和 .last 三个文件，如（v2f1801.head、v2f1801.data、v2f1801.last）数据文件组。TSDB 中的数据文件组是按照时间跨度进行分片的，默认是 10 天一个文件组，且可通过配置文件及建库选项进行配置。分片的数据文件组又按照编号递增排列，方便快速定位某一时间段的时序数据，高效定位数据文件组。时序数据在 TSDB 的数据文件中是以块的形式进行列式存储的，每个块中只包含一张表的数据，且数据在一个块中是按照时间顺序递增排列的。在一个数据文件组中，.head 文件负责存储数据块的索引及统计信息，如每个块的位置，压缩算法，时间戳范围等。存储在 .head 文件中一张表的索引信息是按照数据块中存储的数据的时间递增排列的，方便进行折半查找等工作。.head 和 .last 文件是存储真实数据块的文件，若数据块中的数据累计到一定程度，则会写入 .data 文件中，否则，会写入 .last 文件中，等待下次落盘时合并数据写入 .data 文件中，从而大大减少文件中块的个数，避免数据的过度碎片化。
|
||||
|
||||
## Query 模块
|
||||
|
||||
该模块负责整体系统的查询处理。客户端调用该该模块进行 SQL 语法解析，并将查询或写入请求发送到 vnode ，同时负责针对超级表的查询进行二阶段的聚合操作。在 vnode 端，该模块调用 TSDB 模块读取系统中存储的数据进行查询处理。query 模块还定义了系统能够支持的全部查询函数，查询函数的实现机制与查询框架无耦合，可以在不修改查询流程的情况下动态增加查询函数。详细的设计请参见《TDengine 2.0 查询模块设计》。
|
||||
|
||||
## SYNC 模块
|
||||
|
||||
该模块实现数据的多副本复制，包括 vnode 与 mnode 的数据复制，支持异步和同步两种复制方式，以满足 meta data 与时序数据不同复制的需求。因为它为 mnode 与 vnode 共享，系统为 mnode 副本预留了一个特殊的 vgroup ID:1。因此 vnode group 的 ID 是从 2 开始的。
|
||||
|
||||
每个 vnode/mnode 模块实例会有一对应的 sync 模块实例，他们是一一对应的。详细设计请见[TDengine 2.0 数据复制模块设计](/tdinternal/replica/)
|
||||
|
||||
## WAL 模块
|
||||
|
||||
该模块负责将新插入的数据写入 write ahead log（WAL），为 vnode，mnode 共享。以保证服务器 crash 或其他故障，能从 WAL 中恢复数据。
|
||||
|
||||
每个 vnode/mnode 模块实例会有一对应的 WAL 模块实例，是完全一一对应的。WAL 的落盘操作由两个参数 walLevel，fsync 控制。看具体场景，如果要 100% 保证数据不会丢失，需要将 walLevel 配置为 2，fsync 设置为 0，每条数据插入请求，都会实时落盘后，才会给应用确认
|
||||
|
||||
## HTTP 模块
|
||||
|
||||
该模块负责处理系统对外的 RESTful 接口，可以通过配置，由 dnode 启动或停止 。（仅 2.2 及之前的版本中存在）
|
||||
|
||||
该模块将接收到的 RESTful 请求，做了各种合法性检查后，将其变成标准的 SQL 语句，通过 taosc 的异步接口，将请求发往整个系统中的任一 dnode 。收到处理后的结果后，再翻译成 HTTP 协议，返回给应用。
|
||||
|
||||
如果 HTTP 模块启动，就意味着启动了一个 taosc 的实例。任一一个 dnode 都可以启动该模块，以实现对 RESTful 请求的分布式处理。
|
||||
|
||||
## Monitor 模块
|
||||
|
||||
该模块负责检测一个 dnode 的运行状态，可以通过配置，由 dnode 启动或停止。原则上，每个 dnode 都应该启动一个 monitor 实例。
|
||||
|
||||
Monitor 采集 TDengine 里的关键操作，比如创建、删除、更新账号、表、库等，而且周期性的收集 CPU、内存、网络等资源的使用情况（采集周期由系统配置参数 monitorInterval 控制）。获得这些数据后，monitor 模块将采集的数据写入系统的日志库（DB 名字由系统配置参数 monitorDbName 控制）。
|
||||
|
||||
Monitor 模块使用 taosc 来将采集的数据写入系统，因此每个 monitor 实例，都有一个 taosc 运行实例。
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
---
|
||||
title: TSZ 压缩算法
|
||||
---
|
||||
|
||||
TSZ 压缩算法是 TDengine 为浮点数据类型提供更加丰富的压缩功能，可以实现浮点数的有损至无损全状态压缩，相比原来在 TDengine 中原有压缩算法，TSZ 压缩算法压缩选项更丰富，压缩率更高，即使切到无损状态下对浮点数压缩，压缩率也会比原来的压缩算法高一倍。
|
||||
|
||||
## 适合场景
|
||||
|
||||
TSZ 压缩算法压缩率比原来的要高，但压缩时间会更长，即开启 TSZ 压缩算法写入速度会有一些下降，通常情况下会有 20% 左右的下降。影响写入速度是因为需要更多的 CPU 计算，所以从原始数据到压缩好数据的交付时间变长，导致写入速度变慢。如果您的服务器 CPU 配置很高的话，这个影响会变小甚至没有。
|
||||
|
||||
另外如果设备产生了大量的高精度浮点数，存储占用的空间非常庞大，但实际使用并不需要那么高的精度时，可以通过 TSZ 压缩的有损压缩功能，把精度压缩至指定的长度，节约存储空间。
|
||||
|
||||
总结：采集到了大量浮点数，存储时占用空间过大或出有存储空间不足，需要超高压缩率的场景。
|
||||
|
||||
## 使用步骤
|
||||
|
||||
- 检查版本支持，2.4.0.10 及之后 TDengine 的版本都支持此功能
|
||||
|
||||
- 配置选项开启功能，在 TDengine 的配置文件 taos.cfg 增加一行以下内容，打开 TSZ 功能
|
||||
|
||||
```TSZ
|
||||
lossyColumns float|double
|
||||
```
|
||||
|
||||
- 根据自己需要配置其它选项，如果不配置都会按默认值处理。
|
||||
|
||||
- 重启服务，配置生效。
|
||||
- 确认功能已开启，在服务启动过程中输出的信息如果有前面配置的内容，表明功能已生效：
|
||||
|
||||
```TSZ Test
|
||||
02/22 10:49:27.607990 00002933 UTL lossyColumns float|double
|
||||
```
|
||||
|
||||
## 注意事项
|
||||
|
||||
- 确认版本是否支持
|
||||
|
||||
- 除了服务器启动时的输出的配置成功信息外，不再会有其它的信息输出是使用的哪种压缩算法，可以通过配置前后数据库文件大小来比较效果
|
||||
|
||||
- 如果浮点数类型列较少，看整体数据文件大小效果会不太明显
|
||||
|
||||
- 此压缩产生的数据文件中浮点数据部分将不能被 2.4.0.10 以下的版本解析，即不向下兼容，使用时避免更换回旧版本，以免数据不能被读取出来。
|
||||
|
||||
- 在使用过程中允许反复开启和关闭 TSZ 压缩选项的操作，前后两种压缩算法产生的数据都能正常读取。
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
---
|
||||
title: 物联网大数据
|
||||
description: "物联网、工业互联网大数据的特点；物联网大数据平台应具备的功能和特点；通用大数据架构为什么不适合处理物联网数据；物联网、车联网、工业互联网大数据平台，为什么推荐使用 TDengine"
|
||||
---
|
||||
|
||||
- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html)
|
||||
- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html)
|
||||
- [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/107.html)
|
||||
- [物联网、车联网、工业互联网大数据平台，为什么推荐使用 TDengine？](https://www.taosdata.com/blog/2019/07/09/109.html)
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
---
|
||||
title: 视频教程
|
||||
---
|
||||
|
||||
## 技术公开课
|
||||
|
||||
- [技术公开课：开源、高效的物联网大数据平台，TDengine 内核技术剖析](https://www.taosdata.com/blog/2020/12/25/2126.html)
|
||||
|
||||
## 视频教程
|
||||
|
||||
- [TDengine 视频教程 - 快速上手](https://www.taosdata.com/blog/2020/11/11/1941.html)
|
||||
- [TDengine 视频教程 - 数据建模](https://www.taosdata.com/blog/2020/11/11/1945.html)
|
||||
- [TDengine 视频教程 - 集群搭建](https://www.taosdata.com/blog/2020/11/11/1961.html)
|
||||
- [TDengine 视频教程 - Go Connector](https://www.taosdata.com/blog/2020/11/11/1951.html)
|
||||
- [TDengine 视频教程 - JDBC Connector](https://www.taosdata.com/blog/2020/11/11/1955.html)
|
||||
- [TDengine 视频教程 - Node.js Connector](https://www.taosdata.com/blog/2020/11/11/1957.html)
|
||||
- [TDengine 视频教程 - Python Connector](https://www.taosdata.com/blog/2020/11/11/1963.html)
|
||||
- [TDengine 视频教程 - RESTful Connector](https://www.taosdata.com/blog/2020/11/11/1965.html)
|
||||
- [TDengine 视频教程 - “零”代码运维监控](https://www.taosdata.com/blog/2020/11/11/1959.html)
|
||||
|
||||
## 微课堂
|
||||
|
||||
关注 TDengine 视频号， 有精心制作的微课堂。
|
||||
|
||||
<img src="/img/shi-pin-hao.png" width={350} />
|
||||
|
|
@ -4,24 +4,24 @@ sidebar_label: Documentation Home
|
|||
slug: /
|
||||
---
|
||||
|
||||
TDengine is a [high-performance](https://tdengine.com/fast), [scalable](https://tdengine.com/scalable) time series database with [SQL support](https://tdengine.com/sql-support). This document is the TDengine user manual. It introduces the basic concepts, installation, features, SQL, APIs, operation, maintenance, kernel design, etc. It’s written mainly for architects, developers and system administrators.
|
||||
TDengine is a [high-performance](https://tdengine.com/fast), [scalable](https://tdengine.com/scalable) time series database with [SQL support](https://tdengine.com/sql-support). This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators.
|
||||
|
||||
To get a global view about TDengine, like feature list, benchmarks, and competitive advantages, please browse through section [Introduction](./intro).
|
||||
To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section.
|
||||
|
||||
TDengine makes full use of the characteristics of time series data, proposes the concepts of "one table for one data collection point" and "super table", and designs an innovative storage engine, which greatly improves the efficiency of data ingestion, querying and storage. To understand the new concepts and use TDengine in the right way, please read [“Concepts”](./concept) thoroughly.
|
||||
TDengine greatly improves the efficiency of data ingestion, querying and storage by exploiting the characteristics of time series data, introducing the novel concepts of "one table for one data collection point" and "super table", and designing an innovative storage engine. To understand the new concepts in TDengine and make full use of the features and capabilities of TDengine, please read [“Concepts”](./concept) thoroughly.
|
||||
|
||||
If you are a developer, please read the [“Developer Guide”](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined function, etc. in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, make a few changes to accommodate your application, and it will work.
|
||||
If you are a developer, please read the [“Developer Guide”](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, make a few changes to accommodate your application, and it will work.
|
||||
|
||||
We live in the era of big data, and scale-up is unable to meet the growing business needs. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. The TDengine team has not only developed the cluster feature, they also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to ["Cluster"](./cluster).
|
||||
We live in the era of big data, and scale-up is unable to meet the growing needs of business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to ["cluster"](./cluster).
|
||||
|
||||
TDengine uses SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to support time series data scenarios better, such as roll up, interpolation, time weighted average, etc. The ["SQL Reference"](./taos-sql) chapter describes the SQL syntax in detail, and lists the various supported commands and functions.
|
||||
TDengine uses ubiquitious SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll up, interpolation and time weighted average, among many others. The ["SQL Reference"](./taos-sql) chapter describes the SQL syntax in detail, and lists the various supported commands and functions.
|
||||
|
||||
If you are a system administrator who cares about installation, upgrade, fault tolerance, disaster recovery, data import, data export, system configuration, how to monitor whether TDengine is running healthily, and how to improve system performance, please refer to the ["Administration"](./operation) thoroughly.
|
||||
If you are a system administrator who cares about installation, upgrade, fault tolerance, disaster recovery, data import, data export, system configuration, how to monitor whether TDengine is running healthily, and how to improve system performance, please refer to, and thoroughly read the ["Administration"](./operation) section.
|
||||
|
||||
If you want to know more about TDengine tools, REST API, and connectors for various programming languages, please see the ["Reference"](./reference) chapter.
|
||||
If you want to know more about TDengine tools, the REST API, and connectors for various programming languages, please see the ["Reference"](./reference) chapter.
|
||||
|
||||
If you are very interested in the internal design of TDengine, please read the chapter ["Inside TDengine”](./tdinternal), which introduces the cluster design, data partitioning, sharding, writing, and reading processes in detail. If you want to study TDengine code or even contribute code, please read this chapter carefully.
|
||||
|
||||
TDengine is an open source database, you are welcome to be a part of TDengine. If you find any errors in the documentation, or the description is not clear, please click "Edit this page" at the bottom of each page to edit it directly.
|
||||
TDengine is an open source database, and we would love for you to be a part of TDengine. If you find any errors in the documentation, or see parts where more clarity or elaboration is needed, please click "Edit this page" at the bottom of each page to edit it directly.
|
||||
|
||||
Together, we make a difference.
|
||||
|
|
|
|||
|
|
@ -19,25 +19,24 @@ import InstallOnLinux from "../../14-reference/03-connector/\_windows_install.md
|
|||
import VerifyLinux from "../../14-reference/03-connector/\_verify_linux.mdx";
|
||||
import VerifyWindows from "../../14-reference/03-connector/\_verify_windows.mdx";
|
||||
|
||||
Any application programs running on any kind of platforms can access TDengine through the REST API provided by TDengine. For the details, please refer to [REST API](/reference/rest-api/). Besides, application programs can use the connectors of multiple programming languages to access TDengine, including C/C++, Java, Python, Go, Node.js, C#, and Rust. This chapter describes how to establish connection to TDengine and briefly introduce how to install and use connectors. For details about the connectors, please refer to [Connectors](/reference/connector/)
|
||||
Any application programs running on any kind of platforms can access TDengine through the REST API provided by TDengine. For the details, please refer to [REST API](/reference/rest-api/). Besides, application programs can use the connectors of multiple programming languages to access TDengine, including C/C++, Java, Python, Go, Node.js, C#, and Rust. This chapter describes how to establish connection to TDengine and briefly introduces how to install and use connectors. For details about the connectors, please refer to [Connectors](/reference/connector/)
|
||||
|
||||
## Establish Connection
|
||||
|
||||
There are two ways for a connector to establish connections to TDengine:
|
||||
|
||||
1. Connection through the REST API provided by taosAdapter component, this way is called "REST connection" hereinafter.
|
||||
1. Connection through the REST API provided by the taosAdapter component, this way is called "REST connection" hereinafter.
|
||||
2. Connection through the TDengine client driver (taosc), this way is called "Native connection" hereinafter.
|
||||
|
||||
Either way, same or similar APIs are provided by connectors to access database or execute SQL statements, no obvious difference can be observed.
|
||||
|
||||
Key differences：
|
||||
|
||||
1. With REST connection, it's not necessary to install TDengine client driver (taosc), it's more friendly for cross-platform with the cost of 30% performance downgrade. When taosc has an upgrade, application does not need to make changes.
|
||||
2. With native connection, full compatibility of TDengine can be utilized, like [Parameter Binding](/reference/connector/cpp#Parameter Binding-api), [Subscription](reference/connector/cpp#Subscription), etc. But taosc has to be installed, some platforms may not be supported.
|
||||
1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](/reference/connector/cpp#parameter-binding-api), [Subscription](/reference/connector/cpp#subscription-and-consumption-api), etc.
|
||||
2. The TDengine client driver (taosc) is not supported across all platforms, and applications built on taosc may need to be modified when updating taosc to newere versions.
|
||||
3. The REST connection is more accessible with cross-platform support, however it results in a 30% performance downgrade.
|
||||
|
||||
## Install Client Driver taosc
|
||||
|
||||
If choosing to use native connection and the application is not on the same host as TDengine server, TDengine client driver taosc needs to be installed on the host where the application is. If choosing to use REST connection or the application is on the same host as server side, this step can be skipped. It's better to use same version of taosc as the server.
|
||||
If you are choosing to use native connection and the application is not on the same host as TDengine server, the TDengine client driver taosc needs to be installed on the application host. If choosing to use the REST connection or the application is on the same host as TDengine server, this step can be skipped. It's better to use same version of taosc as the server.
|
||||
|
||||
### Install
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
title: Data Model
|
||||
---
|
||||
|
||||
The data model employed by TDengine is similar to relational database, you need to create databases and tables. For a specific application, the design of databases, STables (abbreviated for super table), and tables need to be considered. This chapter will explain the big picture without syntax details.
|
||||
The data model employed by TDengine is similar to a relational database, you need to create databases and tables. Design the data model based on your own application scenarios and you should design the STable (abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntax details.
|
||||
|
||||
## Create Database
|
||||
|
||||
The characteristics of data from different data collection points may be different, such as collection frequency, days to keep, number of replicas, data block size, whether it's allowed to update data, etc. For TDengine to operate with the best performance, it's strongly suggested to put the data with different characteristics into different databases because different storage policy can be set for each database. When creating a database, there are a lot of parameters that can be configured, such as the days to keep data, the number of replicas, the number of memory blocks, time precision, the minimum and maximum number of rows in each data block, compress or not, the time range of the data in single data file, etc. Below is an example of the SQL statement for creating a database.
|
||||
The characteristics of data from different data collection points may be different, such as collection frequency, days to keep, number of replicas, data block size, whether it's allowed to update data, etc. For TDengine to operate with the best performance, it's strongly suggested to put the data with different characteristics into different databases because different storage policies can be set for each database. When creating a database, there are a lot of parameters that can be configured, such as the days to keep data, the number of replicas, the number of memory blocks, time precision, the minimum and maximum number of rows in each data block, compress or not, the time range of the data in single data file, etc. Below is an example of the SQL statement for creating a database.
|
||||
|
||||
```sql
|
||||
CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 6 UPDATE 1;
|
||||
|
|
@ -14,7 +14,7 @@ CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 6 UPDATE 1;
|
|||
|
||||
In the above SQL statement, a database named "power" will be created, the data in it will be kept for 365 days, which means the data older than 365 days will be deleted automatically, a new data file will be created every 10 days, the number of memory blocks is 6, data is allowed to be updated. For more details please refer to [Database](/taos-sql/database).
|
||||
|
||||
After creating a database, the current database in use can be switched using SQL command `USE`, for example below SQL statement switches the current database to `power`. Without current database specified, table name must be preceded with the corresponding database name.
|
||||
After creating a database, the current database in use can be switched using SQL command `USE`, for example below SQL statement switches the current database to `power`. Without the current database specified, table name must be preceded with the corresponding database name.
|
||||
|
||||
```sql
|
||||
USE power;
|
||||
|
|
@ -23,14 +23,14 @@ USE power;
|
|||
:::note
|
||||
|
||||
- Any table or STable must belong to a database. To create a table or STable, the database it belongs to must be ready.
|
||||
- JOIN operation can't be performed tables from two different databases.
|
||||
- JOIN operations can't be performed on tables from two different databases.
|
||||
- Timestamp needs to be specified when inserting rows or querying historical rows.
|
||||
|
||||
:::
|
||||
|
||||
## Create STable
|
||||
|
||||
In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](/tdinternal/arch#model_table1), below SQL statement can be used to create the super table.
|
||||
In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](/tdinternal/arch#model_table1), the below SQL statement can be used to create the super table.
|
||||
|
||||
```sql
|
||||
CREATE STable meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);
|
||||
|
|
@ -41,11 +41,11 @@ If you are using versions prior to 2.0.15, the `STable` keyword needs to be repl
|
|||
|
||||
:::
|
||||
|
||||
Similar to creating a regular table, when creating a STable, name and schema need to be provided too. In the STable schema, the first column must be timestamp (like ts in the example), and other columns (like current, voltage and phase in the example) are the data collected. The type of a column can be integer, float, double, string ,etc. Besides, the schema for tags need to be provided, like location and groupId in the example. The type of a tag can be integer, float, string, etc. The static properties of a data collection point can be defined as tags, like the location, device type, device group ID, manager ID, etc. Tags in the schema can be added, removed or updated. Please refer to [STable](/taos-sql/stable) for more details.
|
||||
Similar to creating a regular table, when creating a STable, the name and schema need to be provided. In the STable schema, the first column must be timestamp (like ts in the example), and the other columns (like current, voltage and phase in the example) are the data collected. The column type can be integer, float, double, string ,etc. Besides, the schema for tags need to be provided, like location and groupId in the example. The tag type can be integer, float, string, etc. The static properties of a data collection point can be defined as tags, like the location, device type, device group ID, manager ID, etc. Tags in the schema can be added, removed or updated. Please refer to [STable](/taos-sql/stable) for more details.
|
||||
|
||||
For each kind of data collection points, a corresponding STable must be created. There may be many STables in an application. For electrical power system, we need to create a STable respectively for meters, transformers, busbars, switches. There may be multiple kinds of data collection points on a single device, for example there may be one data collection point for electrical data like current and voltage and another point for environmental data like temperature, humidity and wind direction, multiple STables are required for such kind of device.
|
||||
For each kind of data collection point, a corresponding STable must be created. There may be many STables in an application. For electrical power system, we need to create a STable respectively for meters, transformers, busbars, switches. There may be multiple kinds of data collection points on a single device, for example there may be one data collection point for electrical data like current and voltage and another point for environmental data like temperature, humidity and wind direction, multiple STables are required for such kind of device.
|
||||
|
||||
At most 4096 (or 1024 prior to version 2.1.7.0) columns are allowed in a STable. If there are more than 4096 of metrics to bo collected for a data collection point, multiple STables are required for such kind of data collection point. There can be multiple databases in system, while one or more STables can exist in a database.
|
||||
At most 4096 (or 1024 prior to version 2.1.7.0) columns are allowed in a STable. If there are more than 4096 of metrics to be collected for a data collection point, multiple STables are required. There can be multiple databases in a system, while one or more STables can exist in a database.
|
||||
|
||||
## Create Table
|
||||
|
||||
|
|
@ -57,7 +57,7 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);
|
|||
|
||||
In the above SQL statement, "d1001" is the table name, "meters" is the STable name, followed by the value of tag "Location" and the value of tag "groupId", which are "Beijing.Chaoyang" and "2" respectively in the example. The tag values can be updated after the table is created. Please refer to [Tables](/taos-sql/table) for details.
|
||||
|
||||
In TDengine system, it's recommended to create a table for a data collection point via STable. Table created via STable is called subtable in some parts of TDengine document. All SQL commands applied on regular table can be applied on subtable.
|
||||
In TDengine system, it's recommended to create a table for a data collection point via STable. A table created via STable is called subtable in some parts of the TDengine documentation. All SQL commands applied on regular tables can be applied on subtables.
|
||||
|
||||
:::warning
|
||||
It's not recommended to create a table in a database while using a STable from another database as template.
|
||||
|
|
@ -67,7 +67,7 @@ It's suggested to use the global unique ID of a data collection point as the tab
|
|||
|
||||
## Create Table Automatically
|
||||
|
||||
In some circumstances, it's not sure whether the table already exists when inserting rows. The table can be created automatically using the SQL statement below, and nothing will happen if the table already exist.
|
||||
In some circumstances, it's unknown whether the table already exists when inserting rows. The table can be created automatically using the SQL statement below, and nothing will happen if the table already exist.
|
||||
|
||||
```sql
|
||||
INSERT INTO d1001 USING meters TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);
|
||||
|
|
@ -79,6 +79,6 @@ For more details please refer to [Create Table Automatically](/taos-sql/insert#a
|
|||
|
||||
## Single Column vs Multiple Column
|
||||
|
||||
Multiple columns data model is supported in TDengine. As long as multiple metrics are collected by same data collection point at same time, i.e. the timestamp are identical, these metrics can be put in single stable as columns. However, there is another kind of design, i.e. single column data model, a table is created for each metric, which means a STable is required for each kind of metric. For example, 3 STables are required for current, voltage and phase.
|
||||
A multiple columns data model is supported in TDengine. As long as multiple metrics are collected by the same data collection point at the same time, i.e. the timestamp are identical, these metrics can be put in a single STable as columns. However, there is another kind of design, i.e. single column data model, a table is created for each metric, which means a STable is required for each kind of metric. For example, 3 STables are required for current, voltage and phase.
|
||||
|
||||
It's recommended to use multiple column data model as much as possible because it's better in the performance of inserting or querying rows. In some cases, however, the metrics to be collected vary frequently and correspondingly the STable schema needs to be changed frequently too. In such case, it's more convenient to use single column data model.
|
||||
It's recommended to use a multiple column data model as much as possible because it's better in the performance of inserting or querying rows. In some cases, however, the metrics to be collected vary frequently and correspondingly the STable schema needs to be changed frequently too. In such case, it's more convenient to use single column data model.
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
|
|||
- quorum: [Description](/reference/config/#quorum)
|
||||
- maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb)
|
||||
- comp: [Description](/reference/config/#comp)
|
||||
- precision: [Description](reference/config/#precision)
|
||||
- precision: [Description](/reference/config/#precision)
|
||||
6. Please be noted that all of the parameters mentioned in this section can be configured in configuration file `taosd.cfg` at server side and used by default, can be override if they are specified in `create database` statement.
|
||||
|
||||
:::
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
title: Performance Optimization
|
||||
---
|
||||
|
||||
After a TDengine cluster has been running for long enough time, because of updating data, deleting tables and deleting expired data, there may be fragments in data files and query performance may be impacted. To resolve the problem of fragments, from version 2.1.3.0 a new SQL command `COMPACT` can be used to defragment the data files.
|
||||
|
||||
```sql
|
||||
COMPACT VNODES IN (vg_id1, vg_id2, ...)
|
||||
```
|
||||
|
||||
`COMPACT` can be used to defragment one or more vgroups. The defragmentation work will be put in task queue for scheduling execution by TDengine. `SHOW VGROUPS` command can be used to get the vgroup ids to be used in `COMPACT` command. There is a column `compacting` in the output of `SHOW GROUPS` to indicate the compacting status of the vgroup: 2 means the vgroup is waiting in task queue for compacting, 1 means compacting is in progress, and 0 means the vgroup has nothing to do with compacting.
|
||||
|
||||
Please be noted that a lot of disk I/O is required for defragementation operation, during which the performance may be impacted significantly for data insertion and query, data insertion may be blocked shortly in extreme cases.
|
||||
|
||||
## Optimize Storage Parameters
|
||||
|
||||
The data in different use cases may have different characteristics, such as the days to keep, number of replicas, collection interval, record size, number of collection points, compression or not, etc. To achieve best efficiency in storage, the parameters in below table can be used, all of them can be either configured in `taos.cfg` as default configuration or in the command `create database`. For detailed definition of these parameters please refer to [Configuration Parameters](/reference/config/).
|
||||
|
||||
| # | Parameter | Unit | Definition | **Value Range** | **Default Value** |
|
||||
| --- | --------- | ---- | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------- | ----------------- |
|
||||
| 1 | days | Day | The time range of the data stored in a single data file | 1-3650 | 10 |
|
||||
| 2 | keep | Day | The number of days the data is kept in the database | 1-36500 | 3650 |
|
||||
| 3 | cache | MB | The size of each memory block | 1-128 | 16 |
|
||||
| 4 | blocks | None | The number of memory blocks used by each vnode | 3-10000 | 6 |
|
||||
| 5 | quorum | None | The number of required confirmation in case of multiple replicas | 1-2 | 1 |
|
||||
| 6 | minRows | None | The minimum number of rows in a data file | 10-1000 | 100 |
|
||||
| 7 | maxRows | None | The maximum number of rows in a daa file | 200-10000 | 4096 |
|
||||
| 8 | comp | None | Whether to compress the data | 0：uncompressed; 1: One Phase compression; 2: Two Phase compression | 2 |
|
||||
| 9 | walLevel | None | wal sync level (named as "wal" in create database ) | 1：wal enabled without fsync; 2：wal enabled with fsync | 1 |
|
||||
| 10 | fsync | ms | The time to wait for invoking fsync when walLevel is set to 2; 0 means no wait | 3000 |
|
||||
| 11 | replica | none | The number of replications | 1-3 | 1 |
|
||||
| 12 | precision | none | Time precision | ms: millisecond; us: microsecond;ns: nanosecond | ms |
|
||||
| 13 | update | none | Whether to allow updating data | 0: not allowed; 1: a row must be updated as whole; 2: a part of columns in a row can be updated | 0 |
|
||||
| 14 | cacheLast | none | Whether the latest data of a table is cached in memory | 0: not cached; 1: the last row is cached; 2: the latest non-NULL value of each column is cached | 0 |
|
||||
|
||||
For a specific use case, there may be multiple kinds of data with different characteristics, it's best to put data with same characteristics in same database. So there may be multiple databases in a system while each database can be configured with different storage parameters to achieve best performance. The above parameters can be used when creating a database to override the default setting in configuration file.
|
||||
|
||||
```sql
|
||||
CREATE DATABASE demo DAYS 10 CACHE 32 BLOCKS 8 REPLICA 3 UPDATE 1;
|
||||
```
|
||||
|
||||
The above SQL statement creates a database named as `demo`, in which each data file stores data across 10 days, the size of each memory block is 32 MB and each vnode is allocated with 8 blocks, the replica is set to 3, update operation is allowed, and all other parameters not specified in the command follow the default configuration in `taos.cfg`.
|
||||
|
||||
Once a database is created, only some parameters can be changed and be effective immediately while others are can't.
|
||||
|
||||
| **Parameter** | **Alterable** | **Value Range** | **Syntax** |
|
||||
| ------------- | ------------- | ---------------- | -------------------------------------- |
|
||||
| name | | | |
|
||||
| create time | | | |
|
||||
| ntables | | | |
|
||||
| vgroups | | | |
|
||||
| replica | **YES** | 1-3 | ALTER DATABASE <dbname\> REPLICA _n_ |
|
||||
| quorum | **YES** | 1-2 | ALTER DATABASE <dbname\> QUORUM _n_ |
|
||||
| days | | | |
|
||||
| keep | **YES** | days-365000 | ALTER DATABASE <dbname\> KEEP _n_ |
|
||||
| cache | | | |
|
||||
| blocks | **YES** | 3-1000 | ALTER DATABASE <dbname\> BLOCKS _n_ |
|
||||
| minrows | | | |
|
||||
| maxrows | | | |
|
||||
| wal | | | |
|
||||
| fsync | | | |
|
||||
| comp | **YES** | 0-2 | ALTER DATABASE <dbname\> COMP _n_ |
|
||||
| precision | | | |
|
||||
| status | | | |
|
||||
| update | | | |
|
||||
| cachelast | **YES** | 0 \| 1 \| 2 \| 3 | ALTER DATABASE <dbname\> CACHELAST _n_ |
|
||||
|
||||
**Explanation：** Prior to version 2.1.3.0, `taosd` server process needs to be restarted for these parameters to take in effect if they are changed using `ALTER DATABASE`.
|
||||
|
||||
When trying to join a new dnode into a running TDengine cluster, all the parameters related to cluster in the new dnode configuration must be consistent with the cluster, otherwise it can't join the cluster. The parameters that are checked when joining a dnode are as below. For detailed definition of these parameters please refer to [Configuration Parameters](/reference/config/).
|
||||
|
||||
- numOfMnodes
|
||||
- mnodeEqualVnodeNum
|
||||
- offlineThreshold
|
||||
- statusInterval
|
||||
- maxTablesPerVnode
|
||||
- maxVgroupsPerDb
|
||||
- arbitrator
|
||||
- timezone
|
||||
- balance
|
||||
- flowctrl
|
||||
- slaveQuery
|
||||
- adjustMaster
|
||||
|
||||
For the convenience of debugging, the log setting of a dnode can be changed temporarily. The temporary change will be lost once the server is restarted.
|
||||
|
||||
```sql
|
||||
ALTER DNODE <dnode_id> <config>
|
||||
```
|
||||
|
||||
- dnode_id: from output of "SHOW DNODES"
|
||||
- config: the parameter to be changed, as below
|
||||
- resetlog: close the old log file and create the new on
|
||||
- debugFlag: 131 (INFO/ERROR/WARNING), 135 (DEBUG), 143 (TRACE)
|
||||
|
||||
For example
|
||||
|
||||
```
|
||||
alter dnode 1 debugFlag 135;
|
||||
```
|
||||
|
|
@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
|
|||
|
||||
`TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data.
|
||||
|
||||
The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) documentation.
|
||||
The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation.
|
||||
|
||||
This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying.
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ Manually install the following tools.
|
|||
- Install [Python](https://www.python.org/downloads/) 2.7 (`v3.x.x` is not supported) and execute `npm config set python python2.7`.
|
||||
- Go to the `cmd` command-line interface, `npm config set msvs_version 2017`
|
||||
|
||||
Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows- environment. md#compiling-native-addon-modules).
|
||||
Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules).
|
||||
|
||||
If using ARM64 Node.js on Windows 10 ARM, you must add "Visual C++ compilers and libraries for ARM64" and "Visual C++ ATL for ARM64".
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
---
|
||||
title: IoT Big Data
|
||||
description: "Characteristics of IoT Big Data, why general big data platform does not work well for IoT? The required features for an IoT Big Data Platform"
|
||||
---
|
||||
|
||||
- [Characteristics of IoT Big Data](https://tdengine.com/2019/07/09/86.html)
|
||||
- [Why don’t General Big Data Platforms Fit IoT Scenarios?](https://tdengine.com/2019/07/09/92.html)
|
||||
- [Why TDengine is the Best Choice for IoT Big Data Processing?](https://tdengine.com/2019/07/09/94.html)
|
||||
- [Why Redis, Kafka, Spark aren’t Needed if TDengine is Used in the IoT Platform?](https://tdengine.com/2019/07/09/96.html)
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ We will explain how to migrate OpenTSDB applications to TDengine quickly, secure
|
|||
The following figure (Figure 1) shows the system's overall architecture for a typical DevOps application scenario.
|
||||
|
||||
**Figure 1. Typical architecture in a DevOps scenario**
|
||||
Figure 1. [IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg "Figure 1. Typical architecture in a DevOps scenario")
|
||||
|
||||
|
||||
In this application scenario, there are Agent tools deployed in the application environment to collect machine metrics, network metrics, and application metrics. Data collectors to aggregate information collected by agents, systems for persistent data storage and management, and tools for monitoring data visualization (e.g., Grafana, etc.).
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ After writing the data to TDengine properly, you can adapt Grafana to visualize
|
|||
TDengine provides two sets of Dashboard templates by default, and users only need to import the templates from the Grafana directory into Grafana to activate their use.
|
||||
|
||||
**Importing Grafana Templates** Figure 2.
|
||||
! [](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg "Figure 2. Importing a Grafana Template")
|
||||
|
||||
|
||||
After the above steps, you completed the migration to replace OpenTSDB with TDengine. You can see that the whole process is straightforward, there is no need to write any code, and only some configuration files need to be adjusted to meet the migration work.
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ In most DevOps scenarios, if you have a small OpenTSDB cluster (3 or fewer nodes
|
|||
Suppose your application is particularly complex, or the application domain is not a DevOps scenario. You can continue reading subsequent chapters for a more comprehensive and in-depth look at the advanced topics of migrating an OpenTSDB application to TDengine.
|
||||
|
||||
**Figure 3. System architecture after migration**
|
||||
! [IT-DevOps-Solutions-Immigrate-TDengine-Arch](/img/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg "Figure 3. System architecture after migration completion")
|
||||
|
||||
|
||||
## Migration evaluation and strategy for other scenarios
|
||||
|
||||
|
|
@ -96,7 +96,7 @@ Suppose your application is particularly complex, or the application domain is n
|
|||
|
||||
This chapter describes the differences between OpenTSDB and TDengine at the system functionality level. After reading this chapter, you can fully evaluate whether you can migrate some complex OpenTSDB-based applications to TDengine, and what you should pay attention to after migration.
|
||||
|
||||
TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github) .com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly.
|
||||
TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github.com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly.
|
||||
|
||||
TDengine version 2.3.0.x only supports collectd and StatsD as data collection aggregation software but will provide more data collection aggregation software in the future. If you use other data aggregators on the collection side, your application needs to be ported to these two data aggregation systems to write data correctly.
|
||||
In addition to the two data aggregator software protocols mentioned above, TDengine also supports writing data directly via InfluxDB's line protocol and OpenTSDB's data writing protocol, JSON format. You can rewrite the logic on the data push side to write data using the line protocols supported by TDengine.
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ tmq_t* build_consumer() {
|
|||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
assert(tmq);
|
||||
tmq_conf_destroy(conf);
|
||||
return tmq;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -82,9 +82,7 @@ int32_t create_stream() {
|
|||
/*const char* sql = "select sum(k) from tu1 interval(10m)";*/
|
||||
/*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
|
||||
pRes = taos_query(
|
||||
pConn,
|
||||
"create stream stream1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
|
||||
"from tu1 interval(10m)");
|
||||
pConn, "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from tu1 interval(10m)");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
|
||||
return -1;
|
||||
|
|
|
|||
|
|
@ -86,11 +86,17 @@ typedef enum {
|
|||
TSDB_RETENTION_MAX = 3
|
||||
} ERetentionLevel;
|
||||
|
||||
typedef enum {
|
||||
TSDB_BITMODE_DEFAULT = 0, // 2 bits
|
||||
TSDB_BITMODE_ONE_BIT = 1, // 1 bit
|
||||
} EBitmapMode;
|
||||
|
||||
extern char *qtypeStr[];
|
||||
|
||||
#define TSDB_PORT_HTTP 11
|
||||
|
||||
#undef TD_DEBUG_PRINT_ROW
|
||||
#undef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -313,8 +313,9 @@ typedef struct {
|
|||
SDataCol *cols;
|
||||
} SDataCols;
|
||||
|
||||
static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; }
|
||||
static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; }
|
||||
static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; }
|
||||
static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; }
|
||||
static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; }
|
||||
|
||||
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
|
||||
#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data
|
||||
|
|
|
|||
|
|
@ -128,6 +128,7 @@ extern bool tsStartUdfd;
|
|||
|
||||
// schemaless
|
||||
extern char tsSmlChildTableName[];
|
||||
extern char tsSmlTagName[];
|
||||
extern bool tsSmlDataFormat;
|
||||
|
||||
// internal
|
||||
|
|
|
|||
|
|
@ -244,12 +244,12 @@ typedef struct {
|
|||
const void* pMsg;
|
||||
} SSubmitMsgIter;
|
||||
|
||||
int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
|
||||
int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter);
|
||||
int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
|
||||
int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
|
||||
STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
|
||||
// for debug
|
||||
int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema);
|
||||
int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema);
|
||||
|
||||
typedef struct {
|
||||
int32_t code;
|
||||
|
|
@ -1210,9 +1210,10 @@ typedef struct {
|
|||
} SRetrieveMetaTableRsp;
|
||||
|
||||
typedef struct SExplainExecInfo {
|
||||
uint64_t startupCost;
|
||||
uint64_t totalCost;
|
||||
double startupCost;
|
||||
double totalCost;
|
||||
uint64_t numOfRows;
|
||||
uint32_t verboseLen;
|
||||
void* verboseInfo;
|
||||
} SExplainExecInfo;
|
||||
|
||||
|
|
@ -1221,6 +1222,18 @@ typedef struct {
|
|||
SExplainExecInfo* subplanInfo;
|
||||
} SExplainRsp;
|
||||
|
||||
typedef struct STableScanAnalyzeInfo {
|
||||
uint64_t totalRows;
|
||||
uint64_t totalCheckedRows;
|
||||
uint32_t totalBlocks;
|
||||
uint32_t loadBlocks;
|
||||
uint32_t loadBlockStatis;
|
||||
uint32_t skipBlocks;
|
||||
uint32_t filterOutBlocks;
|
||||
double elapsedTime;
|
||||
uint64_t filterTime;
|
||||
} STableScanAnalyzeInfo;
|
||||
|
||||
int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
|
||||
int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
|
||||
|
||||
|
|
@ -1697,7 +1710,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam);
|
|||
|
||||
// TDMT_VND_CREATE_STB ==============
|
||||
typedef struct SVCreateStbReq {
|
||||
const char* name;
|
||||
char* name;
|
||||
tb_uid_t suid;
|
||||
int8_t rollup;
|
||||
SSchemaWrapper schema;
|
||||
|
|
@ -1710,8 +1723,8 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq);
|
|||
|
||||
// TDMT_VND_DROP_STB ==============
|
||||
typedef struct SVDropStbReq {
|
||||
const char* name;
|
||||
tb_uid_t suid;
|
||||
char* name;
|
||||
tb_uid_t suid;
|
||||
} SVDropStbReq;
|
||||
|
||||
int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq);
|
||||
|
|
@ -1720,16 +1733,16 @@ int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq);
|
|||
// TDMT_VND_CREATE_TABLE ==============
|
||||
#define TD_CREATE_IF_NOT_EXISTS 0x1
|
||||
typedef struct SVCreateTbReq {
|
||||
int32_t flags;
|
||||
tb_uid_t uid;
|
||||
int64_t ctime;
|
||||
const char* name;
|
||||
int32_t ttl;
|
||||
int8_t type;
|
||||
int32_t flags;
|
||||
tb_uid_t uid;
|
||||
int64_t ctime;
|
||||
char* name;
|
||||
int32_t ttl;
|
||||
int8_t type;
|
||||
union {
|
||||
struct {
|
||||
tb_uid_t suid;
|
||||
const uint8_t* pTag;
|
||||
tb_uid_t suid;
|
||||
uint8_t* pTag;
|
||||
} ctb;
|
||||
struct {
|
||||
SSchemaWrapper schema;
|
||||
|
|
@ -1777,8 +1790,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc
|
|||
|
||||
// TDMT_VND_DROP_TABLE =================
|
||||
typedef struct {
|
||||
const char* name;
|
||||
int8_t igNotExists;
|
||||
char* name;
|
||||
int8_t igNotExists;
|
||||
} SVDropTbReq;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -1809,9 +1822,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
|
|||
|
||||
// TDMT_VND_ALTER_TABLE =====================
|
||||
typedef struct {
|
||||
const char* tbName;
|
||||
int8_t action;
|
||||
const char* colName;
|
||||
char* tbName;
|
||||
int8_t action;
|
||||
char* colName;
|
||||
// TSDB_ALTER_TABLE_ADD_COLUMN
|
||||
int8_t type;
|
||||
int8_t flags;
|
||||
|
|
@ -1820,17 +1833,17 @@ typedef struct {
|
|||
// TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
|
||||
int32_t colModBytes;
|
||||
// TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
|
||||
const char* colNewName;
|
||||
char* colNewName;
|
||||
// TSDB_ALTER_TABLE_UPDATE_TAG_VAL
|
||||
const char* tagName;
|
||||
int8_t isNull;
|
||||
uint32_t nTagVal;
|
||||
const uint8_t* pTagVal;
|
||||
char* tagName;
|
||||
int8_t isNull;
|
||||
uint32_t nTagVal;
|
||||
uint8_t* pTagVal;
|
||||
// TSDB_ALTER_TABLE_UPDATE_OPTIONS
|
||||
int8_t updateTTL;
|
||||
int32_t newTTL;
|
||||
int8_t updateComment;
|
||||
const char* newComment;
|
||||
int8_t updateTTL;
|
||||
int32_t newTTL;
|
||||
int8_t updateComment;
|
||||
char* newComment;
|
||||
} SVAlterTbReq;
|
||||
|
||||
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
|
||||
|
|
@ -1987,19 +2000,16 @@ static FORCE_INLINE void tFreeClientHbReq(void* pReq) {
|
|||
if (req->info) {
|
||||
tFreeReqKvHash(req->info);
|
||||
taosHashCleanup(req->info);
|
||||
req->info = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq);
|
||||
int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq);
|
||||
|
||||
static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) {
|
||||
static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) {
|
||||
SClientHbBatchReq* req = (SClientHbBatchReq*)pReq;
|
||||
if (deep) {
|
||||
taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
|
||||
} else {
|
||||
taosArrayDestroy(req->reqs);
|
||||
}
|
||||
taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
|
||||
taosMemoryFree(pReq);
|
||||
}
|
||||
|
||||
|
|
@ -2023,6 +2033,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) {
|
|||
|
||||
int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp);
|
||||
int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp);
|
||||
void tFreeSClientHbBatchRsp(SClientHbBatchRsp* pBatchRsp);
|
||||
|
||||
static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) {
|
||||
if (tEncodeI32(pEncoder, pKv->key) < 0) return -1;
|
||||
|
|
@ -2257,20 +2268,20 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
|
|||
int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int8_t version; // for compatibility(default 0)
|
||||
int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
|
||||
int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
|
||||
int8_t timezoneInt; // sma data expired if timezone changes.
|
||||
char indexName[TSDB_INDEX_NAME_LEN];
|
||||
int32_t exprLen;
|
||||
int32_t tagsFilterLen;
|
||||
int64_t indexUid;
|
||||
tb_uid_t tableUid; // super/child/common table uid
|
||||
int64_t interval;
|
||||
int64_t offset; // use unit by precision of DB
|
||||
int64_t sliding;
|
||||
const char* expr; // sma expression
|
||||
const char* tagsFilter;
|
||||
int8_t version; // for compatibility(default 0)
|
||||
int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
|
||||
int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
|
||||
int8_t timezoneInt; // sma data expired if timezone changes.
|
||||
char indexName[TSDB_INDEX_NAME_LEN];
|
||||
int32_t exprLen;
|
||||
int32_t tagsFilterLen;
|
||||
int64_t indexUid;
|
||||
tb_uid_t tableUid; // super/child/common table uid
|
||||
int64_t interval;
|
||||
int64_t offset; // use unit by precision of DB
|
||||
int64_t sliding;
|
||||
char* expr; // sma expression
|
||||
char* tagsFilter;
|
||||
} STSma; // Time-range-wise SMA
|
||||
|
||||
typedef STSma SVCreateTSmaReq;
|
||||
|
|
@ -2523,11 +2534,9 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
|
|||
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
|
||||
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
|
||||
buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
|
||||
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
|
||||
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
if (pRsp->blockNum != 0) {
|
||||
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
|
||||
buf = taosDecodeFixedI8(buf, &pRsp->withTag);
|
||||
|
|
@ -2540,14 +2549,20 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
|
|||
taosArrayPush(pRsp->blockDataLen, &bLen);
|
||||
taosArrayPush(pRsp->blockData, &data);
|
||||
if (pRsp->withSchema) {
|
||||
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
|
||||
buf = taosDecodeSSchemaWrapper(buf, pSW);
|
||||
taosArrayPush(pRsp->blockSchema, &pSW);
|
||||
} else {
|
||||
pRsp->blockSchema = NULL;
|
||||
}
|
||||
if (pRsp->withTbName) {
|
||||
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
|
||||
char* name = NULL;
|
||||
buf = taosDecodeString(buf, &name);
|
||||
taosArrayPush(pRsp->blockTbName, &name);
|
||||
} else {
|
||||
pRsp->blockTbName = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2594,12 +2609,12 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
|
|||
|
||||
#define TD_AUTO_CREATE_TABLE 0x1
|
||||
typedef struct {
|
||||
int64_t suid;
|
||||
int64_t uid;
|
||||
int32_t sver;
|
||||
uint32_t nData;
|
||||
const uint8_t* pData;
|
||||
SVCreateTbReq cTbReq;
|
||||
int64_t suid;
|
||||
int64_t uid;
|
||||
int32_t sver;
|
||||
uint32_t nData;
|
||||
uint8_t* pData;
|
||||
SVCreateTbReq cTbReq;
|
||||
} SVSubmitBlk;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -158,6 +158,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "mnode-drop-index", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "mnode-get-index", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply-msg", NULL, NULL)
|
||||
|
||||
// Requests handled by VNODE
|
||||
TD_NEW_MSG_SEG(TDMT_VND_MSG)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ typedef uint64_t TDRowVerT;
|
|||
typedef int16_t col_id_t;
|
||||
typedef int8_t col_type_t;
|
||||
typedef int32_t col_bytes_t;
|
||||
typedef uint16_t schema_ver_t;
|
||||
typedef int32_t schema_ver_t;
|
||||
typedef int32_t func_id_t;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
|
|||
* @return int32_t 0 for success, -1 for failure.
|
||||
*/
|
||||
int32_t mndProcessMsg(SRpcMsg *pMsg);
|
||||
int32_t mndProcessSyncMsg(SRpcMsg *pMsg);
|
||||
|
||||
/**
|
||||
* @brief Generate machine code
|
||||
|
|
|
|||
|
|
@ -304,13 +304,16 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type);
|
|||
int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type);
|
||||
|
||||
/**
|
||||
* @brief Update the version of sdb
|
||||
* @brief Update the index of sdb
|
||||
*
|
||||
* @param pSdb The sdb object.
|
||||
* @param val The update value of the version.
|
||||
* @return int32_t The current version of sdb
|
||||
* @param index The update value of the apply index.
|
||||
* @return int32_t The current index of sdb
|
||||
*/
|
||||
int64_t sdbUpdateVer(SSdb *pSdb, int32_t val);
|
||||
void sdbSetApplyIndex(SSdb *pSdb, int64_t index);
|
||||
int64_t sdbGetApplyIndex(SSdb *pSdb);
|
||||
void sdbSetApplyTerm(SSdb *pSdb, int64_t term);
|
||||
int64_t sdbGetApplyTerm(SSdb *pSdb);
|
||||
|
||||
SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen);
|
||||
void sdbFreeRaw(SSdbRaw *pRaw);
|
||||
|
|
@ -339,6 +342,7 @@ typedef struct SSdb {
|
|||
char *tmpDir;
|
||||
int64_t lastCommitVer;
|
||||
int64_t curVer;
|
||||
int64_t curTerm;
|
||||
int64_t tableVer[SDB_MAX];
|
||||
int64_t maxId[SDB_MAX];
|
||||
EKeyType keyTypes[SDB_MAX];
|
||||
|
|
|
|||
|
|
@ -72,7 +72,6 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad);
|
|||
* @param pMsg The request message
|
||||
*/
|
||||
int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg);
|
||||
int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,24 +46,34 @@ typedef enum {
|
|||
AUTH_TYPE_OTHER,
|
||||
} AUTH_TYPE;
|
||||
|
||||
typedef struct SUserAuthInfo {
|
||||
char user[TSDB_USER_LEN];
|
||||
char dbFName[TSDB_DB_FNAME_LEN];
|
||||
AUTH_TYPE type;
|
||||
} SUserAuthInfo;
|
||||
|
||||
typedef struct SCatalogReq {
|
||||
SArray *pTableName; // element is SNAME
|
||||
SArray *pUdf; // udf name
|
||||
SArray *pTableMeta; // element is SNAME
|
||||
SArray *pDbVgroup; // element is db full name
|
||||
SArray *pTableHash; // element is SNAME
|
||||
SArray *pUdf; // element is udf name
|
||||
SArray *pDbCfg; // element is db full name
|
||||
SArray *pIndex; // element is index name
|
||||
SArray *pUser; // element is SUserAuthInfo
|
||||
bool qNodeRequired; // valid qnode
|
||||
} SCatalogReq;
|
||||
|
||||
typedef struct SMetaData {
|
||||
SArray *pTableMeta; // STableMeta array
|
||||
SArray *pVgroupInfo; // SVgroupInfo list
|
||||
SArray *pUdfList; // udf info list
|
||||
SArray *pQnodeList; // qnode list, SArray<SQueryNodeAddr>
|
||||
SArray *pTableMeta; // SArray<STableMeta>
|
||||
SArray *pDbVgroup; // SArray<SArray<SVgroupInfo>*>
|
||||
SArray *pTableHash; // SArray<SVgroupInfo>
|
||||
SArray *pUdfList; // SArray<SFuncInfo>
|
||||
SArray *pDbCfg; // SArray<SDbCfgInfo>
|
||||
SArray *pIndex; // SArray<SIndexInfo>
|
||||
SArray *pUser; // SArray<bool>
|
||||
SArray *pQnodeList; // SArray<SQueryNodeAddr>
|
||||
} SMetaData;
|
||||
|
||||
typedef struct STbSVersion {
|
||||
char* tbFName;
|
||||
int32_t sver;
|
||||
} STbSVersion;
|
||||
|
||||
typedef struct SCatalogCfg {
|
||||
uint32_t maxTblCacheNum;
|
||||
uint32_t maxDBCacheNum;
|
||||
|
|
@ -88,6 +98,11 @@ typedef struct SDbVgVersion {
|
|||
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
|
||||
} SDbVgVersion;
|
||||
|
||||
typedef struct STbSVersion {
|
||||
char* tbFName;
|
||||
int32_t sver;
|
||||
} STbSVersion;
|
||||
|
||||
typedef struct SUserAuthVersion {
|
||||
char user[TSDB_USER_LEN];
|
||||
int32_t version;
|
||||
|
|
@ -96,6 +111,8 @@ typedef struct SUserAuthVersion {
|
|||
typedef SDbCfgRsp SDbCfgInfo;
|
||||
typedef SUserIndexRsp SIndexInfo;
|
||||
|
||||
typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code);
|
||||
|
||||
int32_t catalogInit(SCatalogCfg *cfg);
|
||||
|
||||
/**
|
||||
|
|
@ -131,7 +148,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d
|
|||
|
||||
int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
|
||||
|
||||
int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName);
|
||||
int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName);
|
||||
|
||||
int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
|
||||
|
||||
|
|
@ -241,9 +258,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_
|
|||
|
||||
int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg);
|
||||
|
||||
int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
|
||||
int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
|
||||
|
||||
int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo);
|
||||
int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo);
|
||||
|
||||
int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass);
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
|
|||
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
|
||||
int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs);
|
||||
int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp);
|
||||
int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp);
|
||||
int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp);
|
||||
void qExplainFreeCtx(SExplainCtx *pCtx);
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -146,7 +146,8 @@ bool fmIsBuiltinFunc(const char* pFunc);
|
|||
|
||||
bool fmIsAggFunc(int32_t funcId);
|
||||
bool fmIsScalarFunc(int32_t funcId);
|
||||
bool fmIsNonstandardSQLFunc(int32_t funcId);
|
||||
bool fmIsVectorFunc(int32_t funcId);
|
||||
bool fmIsIndefiniteRowsFunc(int32_t funcId);
|
||||
bool fmIsStringFunc(int32_t funcId);
|
||||
bool fmIsDatetimeFunc(int32_t funcId);
|
||||
bool fmIsSelectFunc(int32_t funcId);
|
||||
|
|
|
|||
|
|
@ -16,9 +16,11 @@
|
|||
#ifndef _TD_INDEX_H_
|
||||
#define _TD_INDEX_H_
|
||||
|
||||
#include "nodes.h"
|
||||
#include "os.h"
|
||||
#include "taoserror.h"
|
||||
#include "tarray.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -189,6 +191,12 @@ void indexTermDestroy(SIndexTerm* p);
|
|||
*/
|
||||
void indexInit();
|
||||
|
||||
/* index filter */
|
||||
typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
|
||||
|
||||
SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
|
||||
|
||||
int32_t doFilterTag(const SNode* pFilterNode, SArray* result);
|
||||
/*
|
||||
* destory index env
|
||||
*
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ extern "C" {
|
|||
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
|
||||
(NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
|
||||
|
||||
#define DESTORY_LIST(list) \
|
||||
do { \
|
||||
nodesDestroyList(list); \
|
||||
list = NULL; \
|
||||
#define DESTORY_LIST(list) \
|
||||
do { \
|
||||
nodesDestroyList((list)); \
|
||||
(list) = NULL; \
|
||||
} while (0)
|
||||
|
||||
typedef enum ENodeType {
|
||||
|
|
@ -96,6 +96,7 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_EXPLAIN_OPTIONS,
|
||||
QUERY_NODE_STREAM_OPTIONS,
|
||||
QUERY_NODE_TOPIC_OPTIONS,
|
||||
QUERY_NODE_LEFT_VALUE,
|
||||
|
||||
// Statement nodes are used in parser and planner module.
|
||||
QUERY_NODE_SET_OPERATOR,
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ typedef struct SScanLogicNode {
|
|||
int64_t sliding;
|
||||
int8_t intervalUnit;
|
||||
int8_t slidingUnit;
|
||||
SNode* pTagCond;
|
||||
} SScanLogicNode;
|
||||
|
||||
typedef struct SJoinLogicNode {
|
||||
|
|
@ -343,6 +344,7 @@ typedef struct SSubplan {
|
|||
SNodeList* pParents; // the data destination subplan, get data from current subplan
|
||||
SPhysiNode* pNode; // physical plan of current subplan
|
||||
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
|
||||
SNode* pTagCond;
|
||||
} SSubplan;
|
||||
|
||||
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ typedef struct SValueNode {
|
|||
char* literal;
|
||||
bool isDuration;
|
||||
bool translate;
|
||||
bool notReserved;
|
||||
int16_t placeholderNo;
|
||||
union {
|
||||
bool b;
|
||||
|
|
@ -93,6 +94,10 @@ typedef struct SValueNode {
|
|||
char unit;
|
||||
} SValueNode;
|
||||
|
||||
typedef struct SLeftValueNode {
|
||||
ENodeType type;
|
||||
} SLeftValueNode;
|
||||
|
||||
typedef struct SOperatorNode {
|
||||
SExprNode node; // QUERY_NODE_OPERATOR
|
||||
EOperatorType opType;
|
||||
|
|
@ -236,7 +241,7 @@ typedef struct SSelectStmt {
|
|||
bool isTimeOrderQuery;
|
||||
bool hasAggFuncs;
|
||||
bool hasRepeatScanFuncs;
|
||||
bool hasNonstdSQLFunc;
|
||||
bool hasIndefiniteRowsFunc;
|
||||
} SSelectStmt;
|
||||
|
||||
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
|
||||
|
|
|
|||
|
|
@ -198,6 +198,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
#define NEED_CLIENT_HANDLE_ERROR(_code) \
|
||||
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
|
||||
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
|
||||
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \
|
||||
|| (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB)
|
||||
|
||||
#define NEED_SCHEDULER_RETRY_ERROR(_code) \
|
||||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
|
||||
|
|
|
|||
|
|
@ -114,17 +114,12 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit)
|
|||
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
|
||||
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
|
||||
|
||||
typedef struct {
|
||||
void* inputHandle;
|
||||
void* executor;
|
||||
} SStreamRunner;
|
||||
|
||||
typedef struct {
|
||||
int8_t parallelizable;
|
||||
char* qmsg;
|
||||
// followings are not applicable to encoder and decoder
|
||||
int8_t numOfRunners;
|
||||
SStreamRunner* runners;
|
||||
void* inputHandle;
|
||||
void* executor;
|
||||
} STaskExec;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -320,17 +315,15 @@ int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
|
|||
int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
|
||||
int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
|
||||
|
||||
int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
|
||||
|
||||
int32_t streamTaskRun(SStreamTask* pTask);
|
||||
|
||||
int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
|
||||
|
||||
int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb);
|
||||
int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);
|
||||
int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
|
||||
int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);
|
||||
int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
|
||||
int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ typedef struct SFsmCbMeta {
|
|||
int32_t code;
|
||||
ESyncState state;
|
||||
uint64_t seqNum;
|
||||
SyncTerm term;
|
||||
SyncTerm currentTerm;
|
||||
} SFsmCbMeta;
|
||||
|
||||
typedef struct SSyncFSM {
|
||||
|
|
@ -85,6 +87,7 @@ typedef struct SSyncFSM {
|
|||
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
|
||||
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
|
||||
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
|
||||
void (*FpRestoreFinish)(struct SSyncFSM* pFsm);
|
||||
int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
|
||||
int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
|
||||
} SSyncFSM;
|
||||
|
|
@ -117,7 +120,6 @@ typedef struct SSyncLogStore {
|
|||
|
||||
} SSyncLogStore;
|
||||
|
||||
|
||||
typedef struct SSyncInfo {
|
||||
SyncGroupId vgId;
|
||||
SSyncCfg syncCfg;
|
||||
|
|
@ -144,6 +146,7 @@ int32_t syncGetVgId(int64_t rid);
|
|||
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
|
||||
bool syncEnvIsStart();
|
||||
const char* syncStr(ESyncState state);
|
||||
bool syncIsRestoreFinish(int64_t rid);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ extern "C" {
|
|||
#define TAOS_CONN_CLIENT 1
|
||||
#define IsReq(pMsg) (pMsg->msgType & 1U)
|
||||
|
||||
extern int tsRpcHeadSize;
|
||||
extern int32_t tsRpcHeadSize;
|
||||
|
||||
typedef struct {
|
||||
uint32_t clientIp;
|
||||
|
|
@ -69,10 +69,10 @@ typedef struct SRpcInit {
|
|||
char localFqdn[TSDB_FQDN_LEN];
|
||||
uint16_t localPort; // local port
|
||||
char * label; // for debug purpose
|
||||
int numOfThreads; // number of threads to handle connections
|
||||
int sessions; // number of sessions allowed
|
||||
int32_t numOfThreads; // number of threads to handle connections
|
||||
int32_t sessions; // number of sessions allowed
|
||||
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
|
||||
int idleTime; // milliseconds, 0 means idle timer is disabled
|
||||
int32_t idleTime; // milliseconds, 0 means idle timer is disabled
|
||||
|
||||
// the following is for client app ecurity only
|
||||
char *user; // user name
|
||||
|
|
@ -108,9 +108,9 @@ int32_t rpcInit();
|
|||
void rpcCleanup();
|
||||
void * rpcOpen(const SRpcInit *pRpc);
|
||||
void rpcClose(void *);
|
||||
void * rpcMallocCont(int contLen);
|
||||
void * rpcMallocCont(int32_t contLen);
|
||||
void rpcFreeCont(void *pCont);
|
||||
void * rpcReallocCont(void *ptr, int contLen);
|
||||
void * rpcReallocCont(void *ptr, int32_t contLen);
|
||||
|
||||
// Because taosd supports multi-process mode
|
||||
// These functions should not be used on the server side
|
||||
|
|
@ -121,10 +121,10 @@ void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
|
|||
void rpcReleaseHandle(void *handle, int8_t type); // just release client conn to rpc instance, no close sock
|
||||
|
||||
// These functions will not be called in the child process
|
||||
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
|
||||
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
|
||||
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
|
||||
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
|
||||
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
|
||||
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
|
||||
int32_t rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
|
||||
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifdef WINDOWS
|
||||
#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\"
|
||||
#else
|
||||
#define TD_TMP_DIR_PATH "/tmp/"
|
||||
#endif
|
||||
|
||||
typedef struct TdDir *TdDirPtr;
|
||||
typedef struct TdDirEntry *TdDirEntryPtr;
|
||||
|
||||
|
|
|
|||
|
|
@ -129,9 +129,8 @@ int32_t* taosGetErrno();
|
|||
// mnode-common
|
||||
#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300)
|
||||
#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301)
|
||||
#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302)
|
||||
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303)
|
||||
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304)
|
||||
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0302)
|
||||
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0303)
|
||||
|
||||
// mnode-show
|
||||
#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310)
|
||||
|
|
@ -254,6 +253,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2)
|
||||
#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3)
|
||||
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4)
|
||||
#define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D5)
|
||||
|
||||
// mnode-mq
|
||||
#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0)
|
||||
|
|
@ -420,6 +420,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09)
|
||||
#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A)
|
||||
#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B)
|
||||
#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0A0B)
|
||||
|
||||
// wal
|
||||
#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000)
|
||||
|
|
@ -637,6 +638,8 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_PAR_COMMENT_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x264E)
|
||||
#define TSDB_CODE_PAR_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x264F)
|
||||
#define TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY TAOS_DEF_ERROR_CODE(0, 0x2650)
|
||||
#define TSDB_CODE_PAR_INVALID_DROP_COL TAOS_DEF_ERROR_CODE(0, 0x2651)
|
||||
#define TSDB_CODE_PAR_INVALID_COL_JSON TAOS_DEF_ERROR_CODE(0, 0x2652)
|
||||
|
||||
//planner
|
||||
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ typedef enum EOperatorType {
|
|||
OP_TYPE_MOD,
|
||||
// unary arithmetic operator
|
||||
OP_TYPE_MINUS,
|
||||
OP_TYPE_ASSIGN,
|
||||
|
||||
// bit operator
|
||||
OP_TYPE_BIT_AND,
|
||||
|
|
@ -233,6 +234,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_MAX_TAG_CONDITIONS 1024
|
||||
|
||||
#define TSDB_MAX_JSON_TAG_LEN 16384
|
||||
#define TSDB_MAX_JSON_KEY_LEN 256
|
||||
|
||||
#define TSDB_AUTH_LEN 16
|
||||
#define TSDB_PASSWORD_LEN 32
|
||||
|
|
|
|||
|
|
@ -39,11 +39,11 @@ typedef struct {
|
|||
} SEncoder;
|
||||
|
||||
typedef struct {
|
||||
const uint8_t* data;
|
||||
uint32_t size;
|
||||
uint32_t pos;
|
||||
SCoderMem* mList;
|
||||
SDecoderNode* dStack;
|
||||
uint8_t* data;
|
||||
uint32_t size;
|
||||
uint32_t pos;
|
||||
SCoderMem* mList;
|
||||
SDecoderNode* dStack;
|
||||
} SDecoder;
|
||||
|
||||
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
|
||||
|
|
@ -82,7 +82,7 @@ typedef struct {
|
|||
do { \
|
||||
SEncoder coder = {0}; \
|
||||
tEncoderInit(&coder, NULL, 0); \
|
||||
if ((E)(&coder, S) == 0) { \
|
||||
if ((E)(&coder, S) >= 0) { \
|
||||
SIZE = coder.pos; \
|
||||
RET = 0; \
|
||||
} else { \
|
||||
|
|
@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le
|
|||
static int32_t tEncodeCStr(SEncoder* pCoder, const char* val);
|
||||
|
||||
/* ------------------------ DECODE ------------------------ */
|
||||
void tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size);
|
||||
void tDecoderInit(SDecoder* pCoder, uint8_t* data, uint32_t size);
|
||||
void tDecoderClear(SDecoder* SDecoder);
|
||||
int32_t tStartDecode(SDecoder* pCoder);
|
||||
void tEndDecode(SDecoder* pCoder);
|
||||
|
|
@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val);
|
|||
static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val);
|
||||
static int32_t tDecodeFloat(SDecoder* pCoder, float* val);
|
||||
static int32_t tDecodeDouble(SDecoder* pCoder, double* val);
|
||||
static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len);
|
||||
static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len);
|
||||
static int32_t tDecodeCStr(SDecoder* pCoder, const char** val);
|
||||
static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len);
|
||||
static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len);
|
||||
static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
|
||||
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
|
||||
|
||||
/* ------------------------ IMPL ------------------------ */
|
||||
|
|
@ -317,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
|
|||
if (tDecodeU16v(pCoder, &tval) < 0) {
|
||||
return -1;
|
||||
}
|
||||
*val = ZIGZAGD(int16_t, tval);
|
||||
if (val) *val = ZIGZAGD(int16_t, tval);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -331,7 +331,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
|
|||
if (tDecodeU32v(pCoder, &tval) < 0) {
|
||||
return -1;
|
||||
}
|
||||
*val = ZIGZAGD(int32_t, tval);
|
||||
if (val) *val = ZIGZAGD(int32_t, tval);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -345,7 +345,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
|
|||
if (tDecodeU64v(pCoder, &tval) < 0) {
|
||||
return -1;
|
||||
}
|
||||
*val = ZIGZAGD(int64_t, tval);
|
||||
if (val) *val = ZIGZAGD(int64_t, tval);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) {
|
||||
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
|
||||
if (tDecodeU32v(pCoder, len) < 0) return -1;
|
||||
|
||||
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1;
|
||||
|
|
@ -389,20 +389,20 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) {
|
||||
if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1;
|
||||
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
|
||||
if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1;
|
||||
(*len) -= 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) {
|
||||
static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
|
||||
uint32_t len;
|
||||
return tDecodeCStrAndLen(pCoder, val, &len);
|
||||
}
|
||||
|
||||
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
|
||||
const char* pStr;
|
||||
uint32_t len;
|
||||
char* pStr;
|
||||
uint32_t len;
|
||||
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
|
||||
|
||||
memcpy(val, pStr, len + 1);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -17,6 +17,7 @@ serverName="taosd"
|
|||
clientName="taos"
|
||||
uninstallScript="rmtaos"
|
||||
configFile="taos.cfg"
|
||||
tarName="taos.tar.gz"
|
||||
|
||||
osType=Linux
|
||||
pagMode=full
|
||||
|
|
@ -242,6 +243,11 @@ function install_examples() {
|
|||
|
||||
function update_TDengine() {
|
||||
# Start to update
|
||||
if [ ! -e ${tarName} ]; then
|
||||
echo "File ${tarName} does not exist"
|
||||
exit 1
|
||||
fi
|
||||
tar -zxf ${tarName}
|
||||
echo -e "${GREEN}Start to update ${productName} client...${NC}"
|
||||
# Stop the client shell if running
|
||||
if pidof ${clientName} &> /dev/null; then
|
||||
|
|
@ -264,42 +270,49 @@ function update_TDengine() {
|
|||
|
||||
echo
|
||||
echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}"
|
||||
|
||||
rm -rf $(tar -tf ${tarName})
|
||||
}
|
||||
|
||||
function install_TDengine() {
|
||||
# Start to install
|
||||
echo -e "${GREEN}Start to install ${productName} client...${NC}"
|
||||
# Start to install
|
||||
if [ ! -e ${tarName} ]; then
|
||||
echo "File ${tarName} does not exist"
|
||||
exit 1
|
||||
fi
|
||||
tar -zxf ${tarName}
|
||||
echo -e "${GREEN}Start to install ${productName} client...${NC}"
|
||||
|
||||
install_main_path
|
||||
install_log
|
||||
install_header
|
||||
install_lib
|
||||
install_jemalloc
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
install_connector
|
||||
fi
|
||||
install_examples
|
||||
install_bin
|
||||
install_config
|
||||
install_main_path
|
||||
install_log
|
||||
install_header
|
||||
install_lib
|
||||
install_jemalloc
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
install_connector
|
||||
fi
|
||||
install_examples
|
||||
install_bin
|
||||
install_config
|
||||
|
||||
echo
|
||||
echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}"
|
||||
echo
|
||||
echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}"
|
||||
|
||||
rm -rf $(tar -tf ${tarName})
|
||||
rm -rf $(tar -tf ${tarName})
|
||||
}
|
||||
|
||||
|
||||
## ==============================Main program starts from here============================
|
||||
# Install or updata client and client
|
||||
# if server is already install, don't install client
|
||||
if [ -e ${bin_dir}/${serverName} ]; then
|
||||
echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}"
|
||||
exit 0
|
||||
fi
|
||||
if [ -e ${bin_dir}/${serverName} ]; then
|
||||
echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -x ${bin_dir}/${clientName} ]; then
|
||||
update_flag=1
|
||||
update_TDengine
|
||||
else
|
||||
install_TDengine
|
||||
fi
|
||||
if [ -x ${bin_dir}/${clientName} ]; then
|
||||
update_flag=1
|
||||
update_TDengine
|
||||
else
|
||||
install_TDengine
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ struct SAppInstInfo {
|
|||
SCorEpSet mgmtEp;
|
||||
SInstanceSummary summary;
|
||||
SList* pConnList; // STscObj linked list
|
||||
int64_t clusterId;
|
||||
uint64_t clusterId;
|
||||
void* pTransporter;
|
||||
SAppHbMgr* pAppHbMgr;
|
||||
};
|
||||
|
|
@ -286,6 +286,8 @@ void initMsgHandleFp();
|
|||
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
|
||||
uint16_t port, int connType);
|
||||
|
||||
SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen);
|
||||
|
||||
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
|
||||
|
||||
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ static void registerRequest(SRequestObj *pRequest) {
|
|||
static void deregisterRequest(SRequestObj *pRequest) {
|
||||
assert(pRequest != NULL);
|
||||
|
||||
STscObj * pTscObj = pRequest->pTscObj;
|
||||
STscObj *pTscObj = pRequest->pTscObj;
|
||||
SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
|
||||
|
||||
int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
|
||||
|
|
@ -313,7 +313,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
SConfig * pCfg = taosGetCfg();
|
||||
SConfig *pCfg = taosGetCfg();
|
||||
SConfigItem *pItem = NULL;
|
||||
|
||||
switch (option) {
|
||||
|
|
|
|||
|
|
@ -310,6 +310,8 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
|
|||
taosArrayDestroy(desc.subDesc);
|
||||
desc.subDesc = NULL;
|
||||
}
|
||||
} else {
|
||||
desc.subDesc = NULL;
|
||||
}
|
||||
|
||||
releaseRequest(*rid);
|
||||
|
|
@ -394,6 +396,10 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
|
|||
|
||||
tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen);
|
||||
|
||||
if (NULL == req->info) {
|
||||
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
|
||||
}
|
||||
|
||||
taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -429,6 +435,10 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
|
|||
|
||||
tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen);
|
||||
|
||||
if (NULL == req->info) {
|
||||
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
|
||||
}
|
||||
|
||||
taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -463,6 +473,10 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
|
|||
|
||||
tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen);
|
||||
|
||||
if (NULL == req->info) {
|
||||
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
|
||||
}
|
||||
|
||||
taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -511,16 +525,6 @@ static FORCE_INLINE void hbMgrInitHandle() {
|
|||
hbMgrInitMqHbHandle();
|
||||
}
|
||||
|
||||
void hbFreeReq(void *req) {
|
||||
SClientHbReq *pReq = (SClientHbReq *)req;
|
||||
tFreeReqKvHash(pReq->info);
|
||||
}
|
||||
|
||||
void hbClearClientHbReq(SClientHbReq *pReq) {
|
||||
pReq->query = NULL;
|
||||
pReq->info = NULL;
|
||||
}
|
||||
|
||||
SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
|
||||
SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq));
|
||||
if (pBatchReq == NULL) {
|
||||
|
|
@ -535,6 +539,8 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
|
|||
while (pIter != NULL) {
|
||||
SClientHbReq *pOneReq = pIter;
|
||||
|
||||
pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq);
|
||||
|
||||
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey));
|
||||
if (info) {
|
||||
code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, info->param, pOneReq);
|
||||
|
|
@ -544,7 +550,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
|
|||
}
|
||||
}
|
||||
|
||||
taosArrayPush(pBatchReq->reqs, pOneReq);
|
||||
//hbClearClientHbReq(pOneReq);
|
||||
|
||||
pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
|
||||
|
|
@ -601,8 +606,8 @@ static void *hbThreadFunc(void *param) {
|
|||
void *buf = taosMemoryMalloc(tlen);
|
||||
if (buf == NULL) {
|
||||
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
tFreeClientHbBatchReq(pReq, false);
|
||||
hbClearReqInfo(pAppHbMgr);
|
||||
tFreeClientHbBatchReq(pReq);
|
||||
//hbClearReqInfo(pAppHbMgr);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -611,8 +616,8 @@ static void *hbThreadFunc(void *param) {
|
|||
|
||||
if (pInfo == NULL) {
|
||||
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
tFreeClientHbBatchReq(pReq, false);
|
||||
hbClearReqInfo(pAppHbMgr);
|
||||
tFreeClientHbBatchReq(pReq);
|
||||
//hbClearReqInfo(pAppHbMgr);
|
||||
taosMemoryFree(buf);
|
||||
break;
|
||||
}
|
||||
|
|
@ -628,8 +633,8 @@ static void *hbThreadFunc(void *param) {
|
|||
int64_t transporterId = 0;
|
||||
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
|
||||
asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo);
|
||||
tFreeClientHbBatchReq(pReq, false);
|
||||
hbClearReqInfo(pAppHbMgr);
|
||||
tFreeClientHbBatchReq(pReq);
|
||||
//hbClearReqInfo(pAppHbMgr);
|
||||
|
||||
atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
|
||||
}
|
||||
|
|
@ -721,8 +726,7 @@ void appHbMgrCleanup(void) {
|
|||
void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
|
||||
while (pIter != NULL) {
|
||||
SClientHbReq *pOneReq = pIter;
|
||||
hbFreeReq(pOneReq);
|
||||
taosHashCleanup(pOneReq->info);
|
||||
tFreeClientHbReq(pOneReq);
|
||||
pIter = taosHashIterate(pTarget->activeInfo, pIter);
|
||||
}
|
||||
taosHashCleanup(pTarget->activeInfo);
|
||||
|
|
@ -782,7 +786,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
|
|||
}
|
||||
SClientHbReq hbReq = {0};
|
||||
hbReq.connKey = connKey;
|
||||
hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
|
||||
//hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
|
||||
|
||||
taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq));
|
||||
|
||||
|
|
@ -823,8 +827,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in
|
|||
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
|
||||
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
|
||||
if (pReq) {
|
||||
hbFreeReq(pReq);
|
||||
taosHashCleanup(pReq->info);
|
||||
tFreeClientHbReq(pReq);
|
||||
taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -345,6 +345,10 @@ int32_t validateSversion(SRequestObj* pRequest, void* res) {
|
|||
|
||||
for (int32_t i = 0; i < pRsp->nBlocks; ++i) {
|
||||
SSubmitBlkRsp* blk = pRsp->pBlocks + i;
|
||||
if (NULL == blk->tblFName || 0 == blk->tblFName[0]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver};
|
||||
taosArrayPush(pArray, &tbSver);
|
||||
}
|
||||
|
|
@ -383,14 +387,14 @@ _return:
|
|||
}
|
||||
|
||||
void freeRequestRes(SRequestObj* pRequest, void* res) {
|
||||
if (NULL == res) {
|
||||
if (NULL == pRequest || NULL == res) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (TDMT_VND_SUBMIT == pRequest->type) {
|
||||
tFreeSSubmitRsp((SSubmitRsp*)res);
|
||||
} else if (TDMT_VND_QUERY == pRequest->type) {
|
||||
taosArrayDestroy((SArray *)res);
|
||||
taosArrayDestroy((SArray*)res);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -431,12 +435,13 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
|
|||
|
||||
if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
|
||||
pRequest->code = terrno;
|
||||
freeRequestRes(pRequest, pRes);
|
||||
pRes = NULL;
|
||||
}
|
||||
|
||||
if (res) {
|
||||
*res = pRes;
|
||||
} else {
|
||||
freeRequestRes(pRequest, pRes);
|
||||
pRes = NULL;
|
||||
}
|
||||
|
||||
return pRequest;
|
||||
|
|
@ -499,6 +504,23 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
|
||||
SCatalog* pCatalog = NULL;
|
||||
int32_t tbNum = taosArrayGetSize(tbList);
|
||||
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < tbNum; ++i) {
|
||||
SName* pTbName = taosArrayGet(tbList, i);
|
||||
catalogRemoveTableMeta(pCatalog, pTbName);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
|
||||
SRequestObj* pRequest = NULL;
|
||||
int32_t retryNum = 0;
|
||||
|
|
@ -518,6 +540,10 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
|
|||
}
|
||||
} while (retryNum++ < REQUEST_MAX_TRY_TIMES);
|
||||
|
||||
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
|
||||
removeMeta(pTscObj, pRequest->tableList);
|
||||
}
|
||||
|
||||
return pRequest;
|
||||
}
|
||||
|
||||
|
|
@ -840,8 +866,7 @@ static char* parseTagDatatoJson(void* p) {
|
|||
if (j == 0) {
|
||||
if (*val == TSDB_DATA_TYPE_NULL) {
|
||||
string = taosMemoryCalloc(1, 8);
|
||||
sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L);
|
||||
varDataSetLen(string, strlen(varDataVal(string)));
|
||||
sprintf(string, "%s", TSDB_DATA_NULL_STR_L);
|
||||
goto end;
|
||||
}
|
||||
continue;
|
||||
|
|
@ -977,7 +1002,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
|
|||
length = 0;
|
||||
}
|
||||
varDataSetLen(dst, length + CHAR_BYTES * 2);
|
||||
*(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
|
||||
*(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double jsonVd = *(double*)(jsonInnerData);
|
||||
sprintf(varDataVal(dst), "%.9lf", jsonVd);
|
||||
|
|
|
|||
|
|
@ -146,10 +146,10 @@ void taos_free_result(TAOS_RES *res) {
|
|||
SMqRspObj *pRsp = (SMqRspObj *)res;
|
||||
if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
|
||||
if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
|
||||
if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema);
|
||||
if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName);
|
||||
if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
|
||||
if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
|
||||
if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
|
||||
if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
|
||||
pRsp->resInfo.pRspMsg = NULL;
|
||||
doFreeReqResultInfo(&pRsp->resInfo);
|
||||
}
|
||||
|
|
@ -565,10 +565,32 @@ const char *taos_get_server_info(TAOS *taos) {
|
|||
|
||||
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
|
||||
if (taos == NULL || sql == NULL) {
|
||||
// todo directly call fp
|
||||
fp(param, NULL, TSDB_CODE_INVALID_PARA);
|
||||
return;
|
||||
}
|
||||
|
||||
taos_query_l(taos, sql, (int32_t)strlen(sql));
|
||||
SRequestObj* pRequest = NULL;
|
||||
int32_t retryNum = 0;
|
||||
int32_t code = 0;
|
||||
|
||||
size_t sqlLen = strlen(sql);
|
||||
|
||||
while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
|
||||
pRequest = launchQuery(taos, sql, sqlLen);
|
||||
if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
|
||||
break;
|
||||
}
|
||||
|
||||
code = refreshMeta(taos, pRequest);
|
||||
if (code) {
|
||||
pRequest->code = code;
|
||||
break;
|
||||
}
|
||||
|
||||
destroyRequest(pRequest);
|
||||
}
|
||||
|
||||
fp(param, pRequest, code);
|
||||
}
|
||||
|
||||
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
|
||||
|
|
|
|||
|
|
@ -125,10 +125,10 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
struct SCatalog* pCatalog = NULL;
|
||||
|
||||
if (usedbRsp.vgVersion >= 0) {
|
||||
int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
|
||||
uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId;
|
||||
int32_t code1 = catalogGetHandle(clusterId, &pCatalog);
|
||||
if (code1 != TSDB_CODE_SUCCESS) {
|
||||
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
|
||||
tstrerror(code1));
|
||||
tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId, tstrerror(code1));
|
||||
} else {
|
||||
catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid);
|
||||
}
|
||||
|
|
@ -158,7 +158,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash);
|
||||
taosMemoryFreeClear(output.dbVgroup);
|
||||
|
||||
tscError("failed to build use db output since %s", terrstr());
|
||||
tscError("0x%" PRIx64" failed to build use db output since %s", pRequest->requestId, terrstr());
|
||||
} else if (output.dbVgroup) {
|
||||
struct SCatalog* pCatalog = NULL;
|
||||
|
||||
|
|
|
|||
|
|
@ -63,10 +63,6 @@ for (int i = 1; i < keyLen; ++i) { \
|
|||
|
||||
#define TS "_ts"
|
||||
#define TS_LEN 3
|
||||
#define TAG "_tag"
|
||||
#define TAG_LEN 4
|
||||
#define TAG_VALUE "NULL"
|
||||
#define TAG_VALUE_LEN 4
|
||||
#define VALUE "value"
|
||||
#define VALUE_LEN 5
|
||||
|
||||
|
|
@ -263,7 +259,7 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi
|
|||
memcpy(tname, field->key, field->keyLen);
|
||||
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
|
||||
int32_t bytes = field->length > CHAR_SAVE_LENGTH ? (2*field->length) : CHAR_SAVE_LENGTH;
|
||||
int out = snprintf(buf, bufSize,"`%s` %s(%d)",
|
||||
int out = snprintf(buf, bufSize, "`%s` %s(%d)",
|
||||
tname, tDataTypes[field->type].name, bytes);
|
||||
*outBytes = out;
|
||||
} else {
|
||||
|
|
@ -400,6 +396,12 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
|
|||
pos += outBytes; freeBytes -= outBytes;
|
||||
*pos = ','; ++pos; --freeBytes;
|
||||
}
|
||||
if(taosArrayGetSize(cols) == 0){
|
||||
outBytes = snprintf(pos, freeBytes,"`%s` %s(%d)",
|
||||
tsSmlTagName, tDataTypes[TSDB_DATA_TYPE_NCHAR].name, CHAR_SAVE_LENGTH);
|
||||
pos += outBytes; freeBytes -= outBytes;
|
||||
*pos = ','; ++pos; --freeBytes;
|
||||
}
|
||||
pos--; ++freeBytes;
|
||||
outBytes = snprintf(pos, freeBytes, ")");
|
||||
TAOS_RES* res = taos_query(info->taos, result);
|
||||
|
|
@ -724,9 +726,6 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
|
|||
if(value + len != endPtr){
|
||||
return -1;
|
||||
}
|
||||
if(tsInt64 == 0){
|
||||
return taosGetTimestampNs();
|
||||
}
|
||||
double ts = tsInt64;
|
||||
switch (type) {
|
||||
case TSDB_TIME_PRECISION_HOURS:
|
||||
|
|
@ -792,8 +791,8 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) {
|
|||
}
|
||||
|
||||
static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){
|
||||
if(len == 0){
|
||||
return taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
|
||||
if(len == 0 || (len == 1 && data[0] == '0')){
|
||||
return taosGetTimestampNs();
|
||||
}
|
||||
|
||||
int8_t tsType = smlGetTsTypeByPrecision(info->precision);
|
||||
|
|
@ -815,6 +814,9 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t
|
|||
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL);
|
||||
return -1;
|
||||
}
|
||||
if(len == 1 && data[0] == '0'){
|
||||
return taosGetTimestampNs();
|
||||
}
|
||||
int8_t tsType = smlGetTsTypeByLen(len);
|
||||
if (tsType == -1) {
|
||||
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data);
|
||||
|
|
@ -1112,14 +1114,6 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable
|
|||
|
||||
static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){
|
||||
if(isTag && len == 0){
|
||||
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
|
||||
if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
kv->key = TAG;
|
||||
kv->keyLen = TAG_LEN;
|
||||
kv->value = TAG_VALUE;
|
||||
kv->length = TAG_VALUE_LEN;
|
||||
kv->type = TSDB_DATA_TYPE_NCHAR;
|
||||
if(cols) taosArrayPush(cols, &kv);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,8 +47,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
|
|||
}
|
||||
break;
|
||||
case STMT_EXECUTE:
|
||||
if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) {
|
||||
code = TSDB_CODE_TSC_STMT_API_ERROR;
|
||||
if (STMT_TYPE_QUERY == pStmt->sql.type) {
|
||||
if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL)) {
|
||||
code = TSDB_CODE_TSC_STMT_API_ERROR;
|
||||
}
|
||||
} else {
|
||||
if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) {
|
||||
code = TSDB_CODE_TSC_STMT_API_ERROR;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
|
|
@ -794,6 +800,7 @@ int stmtExec(TAOS_STMT* stmt) {
|
|||
if (code) {
|
||||
pStmt->exec.pRequest->code = code;
|
||||
} else {
|
||||
tFreeSSubmitRsp(pRsp);
|
||||
STMT_ERR_RET(stmtResetStmt(pStmt));
|
||||
STMT_ERR_RET(TSDB_CODE_NEED_RETRY);
|
||||
}
|
||||
|
|
@ -811,12 +818,14 @@ _return:
|
|||
if (TSDB_CODE_SUCCESS == code && autoCreateTbl) {
|
||||
if (NULL == pRsp) {
|
||||
tscError("no submit resp got for auto create table");
|
||||
STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
|
||||
code = TSDB_CODE_TSC_APP_ERROR;
|
||||
} else {
|
||||
code = stmtUpdateTableUid(pStmt, pRsp);
|
||||
}
|
||||
|
||||
STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp));
|
||||
}
|
||||
|
||||
tFreeSSubmitRsp(pRsp);
|
||||
|
||||
++pStmt->sql.runTimes;
|
||||
|
||||
STMT_RET(code);
|
||||
|
|
|
|||
|
|
@ -202,7 +202,12 @@ tmq_conf_t* tmq_conf_new() {
|
|||
}
|
||||
|
||||
void tmq_conf_destroy(tmq_conf_t* conf) {
|
||||
if (conf) taosMemoryFree(conf);
|
||||
if (conf) {
|
||||
if (conf->ip) taosMemoryFree(conf->ip);
|
||||
if (conf->user) taosMemoryFree(conf->user);
|
||||
if (conf->pass) taosMemoryFree(conf->pass);
|
||||
taosMemoryFree(conf);
|
||||
}
|
||||
}
|
||||
|
||||
tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) {
|
||||
|
|
@ -497,6 +502,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
|
|||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
taosFreeQitem(pTaskType);
|
||||
}
|
||||
taosFreeQall(qall);
|
||||
return 0;
|
||||
|
|
@ -954,8 +960,12 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
SMqClientVg* pVg = pParam->pVg;
|
||||
SMqClientTopic* pTopic = pParam->pTopic;
|
||||
tmq_t* tmq = pParam->tmq;
|
||||
int32_t vgId = pParam->vgId;
|
||||
int32_t epoch = pParam->epoch;
|
||||
taosMemoryFree(pParam);
|
||||
if (code != 0) {
|
||||
tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code);
|
||||
tscWarn("msg discard from vg %d, epoch %d, code:%x", vgId, epoch, code);
|
||||
if (pMsg->pData) taosMemoryFree(pMsg->pData);
|
||||
goto CREATE_MSG_FAIL;
|
||||
}
|
||||
|
||||
|
|
@ -963,19 +973,21 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
|
||||
if (msgEpoch < tmqEpoch) {
|
||||
// do not write into queue since updating epoch reset
|
||||
tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
|
||||
tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch,
|
||||
tmqEpoch);
|
||||
tsem_post(&tmq->rspSem);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (msgEpoch != tmqEpoch) {
|
||||
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
|
||||
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch);
|
||||
}
|
||||
|
||||
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
|
||||
if (pRspWrapper == NULL) {
|
||||
tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch);
|
||||
goto CREATE_MSG_FAIL;
|
||||
}
|
||||
|
||||
|
|
@ -986,6 +998,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
|
||||
|
||||
tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
|
||||
tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
|
||||
pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
|
||||
|
|
@ -995,7 +1008,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
return 0;
|
||||
CREATE_MSG_FAIL:
|
||||
if (pParam->epoch == tmq->epoch) {
|
||||
if (epoch == tmq->epoch) {
|
||||
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
|
||||
}
|
||||
tsem_post(&tmq->rspSem);
|
||||
|
|
@ -1088,6 +1101,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
|
|||
int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
|
||||
tmq_t* tmq = pParam->tmq;
|
||||
int8_t async = pParam->async;
|
||||
pParam->code = code;
|
||||
if (code != 0) {
|
||||
tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async);
|
||||
|
|
@ -1104,7 +1118,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
goto END;
|
||||
}
|
||||
|
||||
if (!pParam->async) {
|
||||
if (!async) {
|
||||
SMqAskEpRsp rsp;
|
||||
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
|
||||
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
|
||||
|
|
@ -1125,13 +1139,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
taosWriteQitem(tmq->mqueue, pWrapper);
|
||||
tsem_post(&tmq->rspSem);
|
||||
taosMemoryFree(pParam);
|
||||
}
|
||||
|
||||
END:
|
||||
/*atomic_store_8(&tmq->epStatus, 0);*/
|
||||
if (!pParam->async) {
|
||||
if (!async) {
|
||||
tsem_post(&pParam->rspSem);
|
||||
} else {
|
||||
taosMemoryFree(pParam);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
@ -1279,7 +1294,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
|
|||
setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
|
||||
}
|
||||
|
||||
taosFreeQitem(pWrapper);
|
||||
return pRspObj;
|
||||
}
|
||||
|
||||
|
|
@ -1401,6 +1415,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) {
|
|||
}
|
||||
// build rsp
|
||||
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
|
||||
taosFreeQitem(pollRspWrapper);
|
||||
return pRsp;
|
||||
} else {
|
||||
/*printf("epoch mismatch\n");*/
|
||||
|
|
|
|||
|
|
@ -567,7 +567,6 @@ TEST(testCase, insert_test) {
|
|||
taos_free_result(pRes);
|
||||
taos_close(pConn);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(testCase, projection_query_tables) {
|
||||
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
|
|
@ -606,7 +605,7 @@ TEST(testCase, projection_query_tables) {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
for(int32_t i = 0; i < 100000; i += 20) {
|
||||
for(int32_t i = 0; i < 1000000; i += 20) {
|
||||
char sql[1024] = {0};
|
||||
sprintf(sql,
|
||||
"insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
|
||||
|
|
@ -626,7 +625,7 @@ TEST(testCase, projection_query_tables) {
|
|||
|
||||
printf("start to insert next table\n");
|
||||
|
||||
for(int32_t i = 0; i < 100000; i += 20) {
|
||||
for(int32_t i = 0; i < 1000000; i += 20) {
|
||||
char sql[1024] = {0};
|
||||
sprintf(sql,
|
||||
"insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
|
||||
|
|
@ -693,6 +692,8 @@ TEST(testCase, projection_query_stables) {
|
|||
taos_close(pConn);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
TEST(testCase, agg_query_tables) {
|
||||
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
|
||||
ASSERT_NE(pConn, nullptr);
|
||||
|
|
@ -705,7 +706,7 @@ TEST(testCase, agg_query_tables) {
|
|||
}
|
||||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(pConn, "select tbname from st1");
|
||||
pRes = taos_query(pConn, "explain analyze select count(*) from tu interval(1s)");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
|
||||
taos_free_result(pRes);
|
||||
|
|
|
|||
|
|
@ -269,16 +269,7 @@ TEST(testCase, smlParseCols_tag_Test) {
|
|||
ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf);
|
||||
ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
|
||||
size = taosArrayGetSize(cols);
|
||||
ASSERT_EQ(size, 1);
|
||||
|
||||
// nchar
|
||||
kv = (SSmlKv *)taosArrayGetP(cols, 0);
|
||||
ASSERT_EQ(strncasecmp(kv->key, TAG, TAG_LEN), 0);
|
||||
ASSERT_EQ(kv->keyLen, TAG_LEN);
|
||||
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
|
||||
ASSERT_EQ(kv->length, TAG_LEN);
|
||||
ASSERT_EQ(strncasecmp(kv->value, TAG_VALUE, TAG_VALUE_LEN), 0);
|
||||
taosMemoryFree(kv);
|
||||
ASSERT_EQ(size, 0);
|
||||
|
||||
taosArrayDestroy(cols);
|
||||
taosHashCleanup(dumplicateKey);
|
||||
|
|
@ -1207,7 +1198,8 @@ TEST(testCase, sml_TD15662_Test) {
|
|||
ASSERT_NE(info, nullptr);
|
||||
|
||||
const char *sql[] = {
|
||||
"hetrey,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64",
|
||||
"hetrey c0=f,c1=127i8 1626006833639",
|
||||
"hetrey,t1=r c0=f,c1=127i8 1626006833640",
|
||||
};
|
||||
int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0]));
|
||||
ASSERT_EQ(ret, 0);
|
||||
|
|
|
|||
|
|
@ -1538,7 +1538,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
|
|||
int32_t msgLen = sizeof(SSubmitReq);
|
||||
int32_t numOfBlks = 0;
|
||||
SRowBuilder rb = {0};
|
||||
tdSRowInit(&rb, 0); // TODO: use the latest version
|
||||
tdSRowInit(&rb, pTSchema->version); // TODO: use the latest version
|
||||
|
||||
for (int32_t i = 0; i < sz; ++i) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
|
||||
|
|
|
|||
|
|
@ -855,7 +855,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
|
|||
pCols->maxCols = maxCols;
|
||||
pCols->numOfRows = 0;
|
||||
pCols->numOfCols = 0;
|
||||
// pCols->bitmapMode = 0; // calloc already set 0
|
||||
pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
|
||||
|
||||
if (maxCols > 0) {
|
||||
pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol));
|
||||
|
|
@ -899,7 +899,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
|
|||
#endif
|
||||
|
||||
pCols->numOfRows = 0;
|
||||
pCols->bitmapMode = 0;
|
||||
pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
|
||||
pCols->numOfCols = schemaNCols(pSchema);
|
||||
|
||||
for (i = 0; i < schemaNCols(pSchema); ++i) {
|
||||
|
|
@ -1077,7 +1077,7 @@ void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {
|
|||
|
||||
SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
|
||||
int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size;
|
||||
if (tlen == 0) return NULL;
|
||||
// if (tlen == 0) return NULL; // nCols == 0 means no tags
|
||||
|
||||
tlen += TD_KV_ROW_HEAD_SIZE;
|
||||
|
||||
|
|
@ -1087,8 +1087,10 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
|
|||
kvRowSetNCols(row, pBuilder->nCols);
|
||||
kvRowSetLen(row, tlen);
|
||||
|
||||
memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols);
|
||||
memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size);
|
||||
if(pBuilder->nCols > 0){
|
||||
memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols);
|
||||
memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size);
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
|
|||
uint16_t tsTelemPort = 80;
|
||||
|
||||
// schemaless
|
||||
char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
|
||||
char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; //user defined child table name can be specified in tag value.
|
||||
//If set to empty system will generate table name using MD5 hash.
|
||||
bool tsSmlDataFormat = true; // true means that the name and order of cols in each line are the same(only for influx protocol)
|
||||
|
|
@ -326,6 +327,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
|
||||
|
||||
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
|
||||
|
|
@ -522,6 +524,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
}
|
||||
|
||||
tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
|
||||
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN);
|
||||
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
|
||||
|
||||
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
#undef TD_MSG_SEG_CODE_
|
||||
#include "tmsgdef.h"
|
||||
|
||||
int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
|
||||
int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
|
||||
if (pMsg == NULL) {
|
||||
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
|
||||
return -1;
|
||||
|
|
@ -102,7 +102,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) {
|
||||
int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) {
|
||||
SSubmitMsgIter msgIter = {0};
|
||||
if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1;
|
||||
while (true) {
|
||||
|
|
@ -3318,9 +3318,11 @@ int32_t tSerializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
|
|||
if (tEncodeI32(&encoder, pRsp->numOfPlans) < 0) return -1;
|
||||
for (int32_t i = 0; i < pRsp->numOfPlans; ++i) {
|
||||
SExplainExecInfo *info = &pRsp->subplanInfo[i];
|
||||
if (tEncodeU64(&encoder, info->startupCost) < 0) return -1;
|
||||
if (tEncodeU64(&encoder, info->totalCost) < 0) return -1;
|
||||
if (tEncodeDouble(&encoder, info->startupCost) < 0) return -1;
|
||||
if (tEncodeDouble(&encoder, info->totalCost) < 0) return -1;
|
||||
if (tEncodeU64(&encoder, info->numOfRows) < 0) return -1;
|
||||
if (tEncodeU32(&encoder, info->verboseLen) < 0) return -1;
|
||||
if (tEncodeBinary(&encoder, info->verboseInfo, info->verboseLen) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
|
@ -3341,9 +3343,11 @@ int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
|
|||
if (pRsp->subplanInfo == NULL) return -1;
|
||||
}
|
||||
for (int32_t i = 0; i < pRsp->numOfPlans; ++i) {
|
||||
if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].startupCost) < 0) return -1;
|
||||
if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1;
|
||||
if (tDecodeDouble(&decoder, &pRsp->subplanInfo[i].startupCost) < 0) return -1;
|
||||
if (tDecodeDouble(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1;
|
||||
if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].numOfRows) < 0) return -1;
|
||||
if (tDecodeU32(&decoder, &pRsp->subplanInfo[i].verboseLen) < 0) return -1;
|
||||
if (tDecodeBinary(&decoder, (uint8_t**) &pRsp->subplanInfo[i].verboseInfo, &pRsp->subplanInfo[i].verboseLen) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
|
@ -3817,7 +3821,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
|
|||
|
||||
STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
|
||||
STSchemaBuilder schemaBuilder = {0};
|
||||
if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
|
||||
if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -308,13 +308,10 @@ static int compareKv(const void* p1, const void* p2) {
|
|||
* use stable name and tags to grearate child table name
|
||||
*/
|
||||
void buildChildTableName(RandTableName* rName) {
|
||||
int32_t size = taosArrayGetSize(rName->tags);
|
||||
ASSERT(size > 0);
|
||||
taosArraySort(rName->tags, compareKv);
|
||||
|
||||
SStringBuilder sb = {0};
|
||||
taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen);
|
||||
for (int j = 0; j < size; ++j) {
|
||||
taosArraySort(rName->tags, compareKv);
|
||||
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
|
||||
SSmlKv* tagKv = taosArrayGetP(rName->tags, j);
|
||||
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
|
||||
if(IS_VAR_DATA_TYPE(tagKv->type)){
|
||||
|
|
|
|||
|
|
@ -341,18 +341,19 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8
|
|||
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
|
||||
int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
|
||||
uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
|
||||
uint8_t *qBitmap = (uint8_t*)pBitmap;
|
||||
for (int i = 0; i < nBytes; ++i) {
|
||||
if (*((uint8_t *)pBitmap) != vTypeByte) {
|
||||
if (*qBitmap != vTypeByte) {
|
||||
return false;
|
||||
}
|
||||
pBitmap = POINTER_SHIFT(pBitmap, i);
|
||||
qBitmap = (uint8_t *)POINTER_SHIFT(pBitmap, i);
|
||||
}
|
||||
|
||||
int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
|
||||
|
||||
for (int j = 0; j < nLeft; ++j) {
|
||||
uint8_t vType;
|
||||
tdGetBitmapValType(pBitmap, j, &vType, bitmapMode);
|
||||
tdGetBitmapValType(qBitmap, j, &vType, bitmapMode);
|
||||
if (vType != TD_VTYPE_NORM) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -923,7 +924,7 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) {
|
|||
STSRowIter iter = {0};
|
||||
tdSTSRowIterInit(&iter, pSchema);
|
||||
tdSTSRowIterReset(&iter, row);
|
||||
printf("%s >>>", tag);
|
||||
printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row));
|
||||
for (int i = 0; i < pSchema->numOfCols; ++i) {
|
||||
STColumn *stCol = pSchema->columns + i;
|
||||
SCellVal sVal = {255, NULL};
|
||||
|
|
|
|||
|
|
@ -47,10 +47,8 @@ static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) {
|
|||
|
||||
static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SBnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
int32_t code = -1;
|
||||
dTrace("msg:%p, get from bnode-monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = pMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->msgType == TDMT_MON_BM_INFO) {
|
||||
code = bmProcessGetMonBmInfoReq(pMgmt, pMsg);
|
||||
|
|
@ -58,13 +56,13 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (IsReq(pMsg)) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
bmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
static void *dmStatusThreadFp(void *param) {
|
||||
SDnodeMgmt *pMgmt = param;
|
||||
int64_t lastTime = taosGetTimestampMs();
|
||||
|
||||
setThreadName("dnode-status");
|
||||
|
||||
while (1) {
|
||||
|
|
@ -40,7 +39,6 @@ static void *dmStatusThreadFp(void *param) {
|
|||
static void *dmMonitorThreadFp(void *param) {
|
||||
SDnodeMgmt *pMgmt = param;
|
||||
int64_t lastTime = taosGetTimestampMs();
|
||||
|
||||
setThreadName("dnode-monitor");
|
||||
|
||||
while (1) {
|
||||
|
|
@ -103,11 +101,9 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
|
|||
static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SDnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
int32_t code = -1;
|
||||
tmsg_t msgType = pMsg->msgType;
|
||||
bool isRequest = msgType & 1u;
|
||||
dTrace("msg:%p, will be processed in dnode-mgmt queue, type:%s", pMsg, TMSG_INFO(msgType));
|
||||
dTrace("msg:%p, will be processed in dnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
|
||||
switch (msgType) {
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_DND_CONFIG_DNODE:
|
||||
code = dmProcessConfigReq(pMgmt, pMsg);
|
||||
break;
|
||||
|
|
@ -149,7 +145,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
break;
|
||||
}
|
||||
|
||||
if (isRequest) {
|
||||
if (IsReq(pMsg)) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
SRpcMsg rsp = {
|
||||
.code = code,
|
||||
|
|
|
|||
|
|
@ -24,19 +24,22 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
typedef struct SMnodeMgmt {
|
||||
SDnodeData *pData;
|
||||
SMnode *pMnode;
|
||||
SMsgCb msgCb;
|
||||
const char *path;
|
||||
const char *name;
|
||||
SSingleWorker queryWorker;
|
||||
SSingleWorker readWorker;
|
||||
SSingleWorker writeWorker;
|
||||
SSingleWorker syncWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
SReplica replicas[TSDB_MAX_REPLICA];
|
||||
int8_t replica;
|
||||
int8_t selfIndex;
|
||||
SDnodeData *pData;
|
||||
SMnode *pMnode;
|
||||
SMsgCb msgCb;
|
||||
const char *path;
|
||||
const char *name;
|
||||
SSingleWorker queryWorker;
|
||||
SSingleWorker readWorker;
|
||||
SSingleWorker writeWorker;
|
||||
SSingleWorker syncWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
SReplica replicas[TSDB_MAX_REPLICA];
|
||||
int8_t replica;
|
||||
int8_t selfIndex;
|
||||
bool stopped;
|
||||
int32_t refCount;
|
||||
TdThreadRwlock lock;
|
||||
} SMnodeMgmt;
|
||||
|
||||
// mmFile.c
|
||||
|
|
@ -45,6 +48,8 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed);
|
|||
|
||||
// mmInt.c
|
||||
int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pMsg);
|
||||
int32_t mmAcquire(SMnodeMgmt *pMgmt);
|
||||
void mmRelease(SMnodeMgmt *pMgmt);
|
||||
|
||||
// mmHandle.c
|
||||
SArray *mmGetMsgHandles();
|
||||
|
|
|
|||
|
|
@ -237,6 +237,16 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ static void mmClose(SMnodeMgmt *pMgmt) {
|
|||
if (pMgmt->pMnode != NULL) {
|
||||
mmStopWorker(pMgmt);
|
||||
mndClose(pMgmt->pMnode);
|
||||
taosThreadRwlockDestroy(&pMgmt->lock);
|
||||
pMgmt->pMnode = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -122,6 +123,11 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (syncInit() != 0) {
|
||||
dError("failed to init sync since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt));
|
||||
if (pMgmt == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
|
@ -137,6 +143,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
|
|||
pMgmt->msgCb.queueFps[WRITE_QUEUE] = (PutToQueueFp)mmPutRpcMsgToWriteQueue;
|
||||
pMgmt->msgCb.queueFps[SYNC_QUEUE] = (PutToQueueFp)mmPutRpcMsgToSyncQueue;
|
||||
pMgmt->msgCb.mgmt = pMgmt;
|
||||
taosThreadRwlockInit(&pMgmt->lock, NULL);
|
||||
|
||||
bool deployed = false;
|
||||
if (mmReadFile(pMgmt, &deployed) != 0) {
|
||||
|
|
@ -206,3 +213,22 @@ SMgmtFunc mmGetMgmtFunc() {
|
|||
|
||||
return mgmtFunc;
|
||||
}
|
||||
|
||||
int32_t mmAcquire(SMnodeMgmt *pMgmt) {
|
||||
int32_t code = 0;
|
||||
|
||||
taosThreadRwlockRdlock(&pMgmt->lock);
|
||||
if (pMgmt->stopped) {
|
||||
code = -1;
|
||||
} else {
|
||||
atomic_add_fetch_32(&pMgmt->refCount, 1);
|
||||
}
|
||||
taosThreadRwlockUnlock(&pMgmt->lock);
|
||||
return code;
|
||||
}
|
||||
|
||||
void mmRelease(SMnodeMgmt *pMgmt) {
|
||||
taosThreadRwlockRdlock(&pMgmt->lock);
|
||||
atomic_sub_fetch_32(&pMgmt->refCount, 1);
|
||||
taosThreadRwlockUnlock(&pMgmt->lock);
|
||||
}
|
||||
|
|
@ -46,7 +46,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
code = mndProcessMsg(pMsg);
|
||||
}
|
||||
|
||||
if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
mmSendRsp(pMsg, code);
|
||||
}
|
||||
|
|
@ -56,22 +56,12 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SMnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
int32_t code = -1;
|
||||
tmsg_t msgType = pMsg->msgType;
|
||||
bool isRequest = msgType & 1U;
|
||||
dTrace("msg:%p, get from mnode-query queue", pMsg);
|
||||
dTrace("msg:%p, get from mnode-sync queue", pMsg);
|
||||
|
||||
pMsg->info.node = pMgmt->pMnode;
|
||||
code = mndProcessMsg(pMsg);
|
||||
|
||||
if (isRequest) {
|
||||
if (pMsg->info.handle != NULL && code != 0) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
mmSendRsp(pMsg, code);
|
||||
}
|
||||
}
|
||||
int32_t code = mndProcessSyncMsg(pMsg);
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
|
|
@ -127,7 +117,17 @@ int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg);
|
||||
int32_t code = -1;
|
||||
if (mmAcquire(pMgmt) == 0) {
|
||||
code = mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg);
|
||||
mmRelease(pMgmt);
|
||||
}
|
||||
|
||||
if (code != 0) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
||||
|
|
@ -135,7 +135,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
|||
.min = tsNumOfMnodeQueryThreads,
|
||||
.max = tsNumOfMnodeQueryThreads,
|
||||
.name = "mnode-query",
|
||||
.fp = (FItem)mmProcessQueryQueue,
|
||||
.fp = (FItem)mmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
|
||||
|
|
@ -171,7 +171,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
|||
.min = 1,
|
||||
.max = 1,
|
||||
.name = "mnode-sync",
|
||||
.fp = (FItem)mmProcessQueue,
|
||||
.fp = (FItem)mmProcessSyncQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
|
||||
|
|
@ -196,6 +196,11 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
|||
}
|
||||
|
||||
void mmStopWorker(SMnodeMgmt *pMgmt) {
|
||||
taosThreadRwlockWrlock(&pMgmt->lock);
|
||||
pMgmt->stopped = 1;
|
||||
taosThreadRwlockUnlock(&pMgmt->lock);
|
||||
while (pMgmt->refCount > 0) taosMsleep(10);
|
||||
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->queryWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->readWorker);
|
||||
|
|
|
|||
|
|
@ -19,70 +19,39 @@
|
|||
static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {
|
||||
.code = code,
|
||||
.info = pMsg->info,
|
||||
.pCont = pMsg->info.rsp,
|
||||
.contLen = pMsg->info.rspLen,
|
||||
.info = pMsg->info,
|
||||
};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
int32_t code = -1;
|
||||
dTrace("msg:%p, get from qnode queue", pMsg);
|
||||
|
||||
dTrace("msg:%p, get from qnode-monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = pMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->msgType == TDMT_MON_QM_INFO) {
|
||||
code = qmProcessGetMonitorInfoReq(pMgmt, pMsg);
|
||||
} else {
|
||||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_MON_QM_INFO:
|
||||
code = qmProcessGetMonitorInfoReq(pMgmt, pMsg);
|
||||
break;
|
||||
default:
|
||||
code = qndProcessQueryMsg(pMgmt->pQnode, pMsg);
|
||||
break;
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (IsReq(pMsg) && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
qmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
dTrace("msg:%p, get from qnode-query queue", pMsg);
|
||||
SRpcMsg *pRpc = pMsg;
|
||||
int32_t code = qndProcessQueryMsg(pMgmt->pQnode, pRpc);
|
||||
|
||||
if (pRpc->msgType & 1U && code != 0) {
|
||||
qmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
dTrace("msg:%p, get from qnode-fetch queue", pMsg);
|
||||
SRpcMsg *pRpc = pMsg;
|
||||
int32_t code = qndProcessFetchMsg(pMgmt->pQnode, pRpc);
|
||||
|
||||
if (pRpc->msgType & 1U && code != 0) {
|
||||
qmSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
|
||||
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
|
||||
dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -101,9 +70,7 @@ int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
|
||||
static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
|
||||
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
|
||||
if (pMsg == NULL) {
|
||||
return -1;
|
||||
}
|
||||
if (pMsg == NULL) return -1;
|
||||
|
||||
dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
|
|
@ -141,7 +108,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
|||
.min = tsNumOfVnodeQueryThreads,
|
||||
.max = tsNumOfVnodeQueryThreads,
|
||||
.name = "qnode-query",
|
||||
.fp = (FItem)qmProcessQueryQueue,
|
||||
.fp = (FItem)qmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
|
||||
|
|
@ -154,7 +121,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
|||
.min = tsNumOfQnodeFetchThreads,
|
||||
.max = tsNumOfQnodeFetchThreads,
|
||||
.name = "qnode-fetch",
|
||||
.fp = (FItem)qmProcessFetchQueue,
|
||||
.fp = (FItem)qmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
|
||||
|
|
@ -167,7 +134,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
|||
.min = 1,
|
||||
.max = 1,
|
||||
.name = "qnode-monitor",
|
||||
.fp = (FItem)qmProcessMonitorQueue,
|
||||
.fp = (FItem)qmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
|
|
|
|||
|
|
@ -28,10 +28,8 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) {
|
|||
|
||||
static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SSnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
int32_t code = -1;
|
||||
dTrace("msg:%p, get from snode-monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = pMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->msgType == TDMT_MON_SM_INFO) {
|
||||
code = smProcessGetMonitorInfoReq(pMgmt, pMsg);
|
||||
|
|
@ -39,13 +37,13 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (IsReq(pMsg)) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
smSendRsp(pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
|
|||
pCfg->dbId = pCreate->dbUid;
|
||||
pCfg->szPage = pCreate->pageSize * 1024;
|
||||
pCfg->szCache = pCreate->pages;
|
||||
pCfg->szBuf = pCreate->buffer * 1024 * 1024;
|
||||
pCfg->szBuf = (uint64_t)pCreate->buffer * 1024 * 1024;
|
||||
pCfg->isWeak = true;
|
||||
pCfg->tsdbCfg.compression = pCreate->compression;
|
||||
pCfg->tsdbCfg.precision = pCreate->precision;
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
|
|||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
||||
SVnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
int32_t code = -1;
|
||||
dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
|
|
@ -92,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
|
||||
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SVnodeObj *pVnode = pInfo->ahandle;
|
||||
SArray * pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
|
||||
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
|
||||
if (pArray == NULL) {
|
||||
dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr());
|
||||
return;
|
||||
|
|
@ -113,6 +113,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
|
|||
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
|
||||
SRpcMsg rsp = {.info = pMsg->info};
|
||||
|
||||
vnodePreprocessReq(pVnode->pImpl, pMsg);
|
||||
|
||||
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false);
|
||||
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
|
||||
dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId);
|
||||
|
|
@ -222,8 +224,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
|
|||
}
|
||||
|
||||
static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
|
||||
SRpcMsg * pRpc = pMsg;
|
||||
SMsgHead *pHead = pRpc->pCont;
|
||||
SMsgHead *pHead = pMsg->pCont;
|
||||
int32_t code = 0;
|
||||
|
||||
pHead->contLen = ntohl(pHead->contLen);
|
||||
|
|
@ -237,23 +238,23 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType
|
|||
|
||||
switch (qtype) {
|
||||
case QUERY_QUEUE:
|
||||
dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
|
||||
dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pVnode->pQueryQ, pMsg);
|
||||
break;
|
||||
case FETCH_QUEUE:
|
||||
dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
|
||||
dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pVnode->pFetchQ, pMsg);
|
||||
break;
|
||||
case WRITE_QUEUE:
|
||||
dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
|
||||
dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pVnode->pWriteQ, pMsg);
|
||||
break;
|
||||
case SYNC_QUEUE:
|
||||
dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
|
||||
dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pVnode->pSyncQ, pMsg);
|
||||
break;
|
||||
case MERGE_QUEUE:
|
||||
dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
|
||||
dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
|
||||
taosWriteQitem(pVnode->pMergeQ, pMsg);
|
||||
break;
|
||||
default:
|
||||
|
|
@ -301,7 +302,7 @@ int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) {
|
||||
SMsgHead * pHead = pRpc->pCont;
|
||||
SMsgHead *pHead = pRpc->pCont;
|
||||
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
|
||||
if (pVnode == NULL) return -1;
|
||||
|
||||
|
|
@ -469,7 +470,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
|
|||
.min = 1,
|
||||
.max = 1,
|
||||
.name = "vnode-mgmt",
|
||||
.fp = (FItem)vmProcessMgmtMonitorQueue,
|
||||
.fp = (FItem)vmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) {
|
||||
|
|
@ -481,7 +482,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
|
|||
.min = 1,
|
||||
.max = 1,
|
||||
.name = "vnode-monitor",
|
||||
.fp = (FItem)vmProcessMgmtMonitorQueue,
|
||||
.fp = (FItem)vmProcessQueue,
|
||||
.param = pMgmt,
|
||||
};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
|
|||
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
|
||||
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg);
|
||||
void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg);
|
||||
void dmProcessFetchRsp(SRpcMsg *pMsg);
|
||||
|
||||
// dmNodes.c
|
||||
int32_t dmOpenNode(SMgmtWrapper *pWrapper);
|
||||
|
|
|
|||
|
|
@ -314,8 +314,3 @@ void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) {
|
|||
rpcSendResponse(&rsp);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
}
|
||||
|
||||
void dmProcessFetchRsp(SRpcMsg *pMsg) {
|
||||
qWorkerProcessFetchRsp(NULL, NULL, pMsg);
|
||||
// rpcFreeCont(pMsg->pCont);
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "dmMgmt.h"
|
||||
#include "qworker.h"
|
||||
|
||||
static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet);
|
||||
static void dmSendRsp(SRpcMsg *pMsg);
|
||||
|
|
@ -61,7 +62,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
dmProcessNetTestReq(pDnode, pRpc);
|
||||
return;
|
||||
} else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) {
|
||||
dmProcessFetchRsp(pRpc);
|
||||
qWorkerProcessFetchRsp(NULL, NULL, pRpc);
|
||||
return;
|
||||
} else {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
class DndTestBnode : public ::testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() {
|
||||
test.Init("/tmp/dbnodeTest", 9112);
|
||||
test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112);
|
||||
taosMsleep(1100);
|
||||
}
|
||||
static void TearDownTestSuite() { test.Cleanup(); }
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ target_link_libraries(
|
|||
dmnodeTest sut
|
||||
)
|
||||
|
||||
add_test(
|
||||
NAME dmnodeTest
|
||||
COMMAND dmnodeTest
|
||||
)
|
||||
#add_test(
|
||||
# NAME dmnodeTest
|
||||
# COMMAND dmnodeTest
|
||||
#)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
class DndTestMnode : public ::testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() { test.Init("/tmp/dmnodeTest", 9114); }
|
||||
static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dmnodeTest", 9114); }
|
||||
static void TearDownTestSuite() { test.Cleanup(); }
|
||||
|
||||
static Testbase test;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
class DndTestQnode : public ::testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() { test.Init("/tmp/dqnodeTest", 9111); }
|
||||
static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dqnodeTest", 9111); }
|
||||
static void TearDownTestSuite() { test.Cleanup(); }
|
||||
|
||||
static Testbase test;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
class DndTestSnode : public ::testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() { test.Init("/tmp/dsnodeTest", 9113); }
|
||||
static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dsnodeTest", 9113); }
|
||||
static void TearDownTestSuite() { test.Cleanup(); }
|
||||
|
||||
static Testbase test;
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ void Testbase::Init(const char* path, int16_t port) {
|
|||
strcpy(tsDataDir, path);
|
||||
taosRemoveDir(path);
|
||||
taosMkDir(path);
|
||||
InitLog("/tmp/td");
|
||||
InitLog(TD_TMP_DIR_PATH "td");
|
||||
|
||||
server.Start();
|
||||
client.Init("root", "taosdata");
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
class DndTestVnode : public ::testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() { test.Init("/tmp/dvnodeTest", 9115); }
|
||||
static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dvnodeTest", 9115); }
|
||||
static void TearDownTestSuite() { test.Cleanup(); }
|
||||
|
||||
static Testbase test;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "mndDef.h"
|
||||
|
||||
#include "sdb.h"
|
||||
#include "syncTools.h"
|
||||
#include "tcache.h"
|
||||
#include "tdatablock.h"
|
||||
#include "tglobal.h"
|
||||
|
|
@ -31,12 +32,14 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
// clang-format off
|
||||
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
|
||||
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
|
||||
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
|
||||
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
|
||||
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
|
||||
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
|
||||
// clang-format on
|
||||
|
||||
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
|
|
@ -72,10 +75,11 @@ typedef struct {
|
|||
} STelemMgmt;
|
||||
|
||||
typedef struct {
|
||||
int32_t errCode;
|
||||
sem_t syncSem;
|
||||
SWal *pWal;
|
||||
SSyncNode *pSyncNode;
|
||||
int32_t errCode;
|
||||
bool restored;
|
||||
sem_t syncSem;
|
||||
int64_t sync;
|
||||
ESyncState state;
|
||||
} SSyncMgmt;
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ int32_t mndInitSync(SMnode *pMnode);
|
|||
void mndCleanupSync(SMnode *pMnode);
|
||||
bool mndIsMaster(SMnode *pMnode);
|
||||
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw);
|
||||
void mndSyncStart(SMnode *pMnode);
|
||||
void mndSyncStop(SMnode *pMnode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
|
|||
|
||||
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
|
||||
|
||||
int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
|
||||
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -304,10 +304,10 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateBnode(pMnode, pReq, pDnode, &createReq);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -414,10 +414,10 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropBnode(pMnode, pReq, pObj);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -419,7 +419,9 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
SMqTopicObj topicObj = {0};
|
||||
memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
|
||||
topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
|
||||
if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
|
||||
mInfo("subscribe topic %s by consumer %ld cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup,
|
||||
topicObj.refConsumerCnt);
|
||||
if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
|
||||
|
||||
mndReleaseTopic(pMnode, pTopic);
|
||||
}
|
||||
|
|
@ -511,7 +513,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
|
||||
}
|
||||
|
||||
code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
SUBSCRIBE_OVER:
|
||||
mndTransDrop(pTrans);
|
||||
|
|
|
|||
|
|
@ -525,7 +525,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
|
|||
|
||||
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
|
||||
dbObj.cfg.pRetensions = pCreate->pRetensions;
|
||||
pCreate->pRetensions = NULL;
|
||||
|
||||
mndSetDefaultDbCfg(&dbObj.cfg);
|
||||
|
||||
|
|
@ -605,10 +604,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateDb(pMnode, pReq, &createReq, pUser);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("db:%s, failed to create since %s", createReq.db, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -839,10 +838,10 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
|
|||
dbObj.cfgVersion++;
|
||||
dbObj.updateTime = taosGetTimestampMs();
|
||||
code = mndAlterDb(pMnode, pReq, pDb, &dbObj);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("db:%s, failed to alter since %s", alterReq.db, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -1110,10 +1109,10 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropDb(pMnode, pReq, pDb);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("db:%s, failed to drop since %s", dropReq.db, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -448,13 +448,13 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
|
|||
}
|
||||
mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
|
||||
|
||||
SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj);
|
||||
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
|
||||
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
|
||||
SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj);
|
||||
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
|
||||
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) {
|
||||
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
|
||||
|
|
@ -504,10 +504,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateDnode(pMnode, pReq, &createReq);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
CREATE_DNODE_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -524,13 +524,13 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) {
|
|||
}
|
||||
mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
|
||||
|
||||
SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode);
|
||||
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
|
||||
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
|
||||
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
|
||||
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
|
||||
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) {
|
||||
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
|
||||
|
|
@ -585,10 +585,10 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropDnode(pMnode, pReq, pDnode);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
DROP_DNODE_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -330,10 +330,10 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateFunc(pMnode, pReq, &createReq);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("func:%s, failed to create since %s", createReq.name, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -386,10 +386,10 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropFunc(pMnode, pReq, pFunc);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("func:%s, failed to drop since %s", dropReq.name, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -402,10 +402,10 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateMnode(pMnode, pReq, pDnode, &createReq);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -574,10 +574,10 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropMnode(pMnode, pReq, pObj);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -153,6 +153,7 @@ int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicNa
|
|||
return -1;
|
||||
}
|
||||
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
|
||||
// commit log or redo log?
|
||||
if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -188,7 +189,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
|
|||
pOffsetObj->offset = pOffset->offset;
|
||||
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
|
||||
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
|
||||
mndTransAppendRedolog(pTrans, pOffsetRaw);
|
||||
mndTransAppendCommitlog(pTrans, pOffsetRaw);
|
||||
if (create) {
|
||||
taosMemoryFree(pOffsetObj);
|
||||
} else {
|
||||
|
|
@ -205,7 +206,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
mndTransDrop(pTrans);
|
||||
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
return TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
}
|
||||
|
||||
static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) {
|
||||
|
|
|
|||
|
|
@ -128,7 +128,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType
|
|||
}
|
||||
|
||||
static void mndFreeConn(SConnObj *pConn) {
|
||||
taosMemoryFreeClear(pConn->pQueries);
|
||||
taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc);
|
||||
|
||||
mTrace("conn:%u, is destroyed, data:%p", pConn->id, pConn);
|
||||
}
|
||||
|
||||
|
|
@ -396,6 +397,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
|
|||
if (NULL == hbRsp.info) {
|
||||
mError("taosArrayInit %d rsp kv failed", kvNum);
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
tFreeClientHbRsp(&hbRsp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -453,6 +455,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
|
|||
|
||||
SClientHbBatchReq batchReq = {0};
|
||||
if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) {
|
||||
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -479,18 +482,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
|
|||
void *buf = rpcMallocCont(tlen);
|
||||
tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp);
|
||||
|
||||
int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps);
|
||||
for (int32_t i = 0; i < rspNum; ++i) {
|
||||
SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i);
|
||||
int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0;
|
||||
for (int32_t n = 0; n < kvNum; ++n) {
|
||||
SKv *kv = taosArrayGet(rsp->info, n);
|
||||
taosMemoryFreeClear(kv->value);
|
||||
}
|
||||
taosArrayDestroy(rsp->info);
|
||||
}
|
||||
|
||||
taosArrayDestroy(batchRsp.rsps);
|
||||
tFreeClientHbBatchRsp(&batchRsp);
|
||||
pReq->info.rspLen = tlen;
|
||||
pReq->info.rsp = buf;
|
||||
|
||||
|
|
|
|||
|
|
@ -306,10 +306,10 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndCreateQnode(pMnode, pReq, pDnode, &createReq);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
@ -416,10 +416,10 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
code = mndDropQnode(pMnode, pReq, pObj);
|
||||
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
_OVER:
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
|
||||
mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,37 +18,35 @@
|
|||
#include "mndMnode.h"
|
||||
#include "qworker.h"
|
||||
|
||||
int32_t mndProcessQueryMsg(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
|
||||
int32_t code = -1;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb};
|
||||
|
||||
mTrace("msg:%p, in query queue is processing", pReq);
|
||||
switch (pReq->msgType) {
|
||||
case TDMT_VND_QUERY:
|
||||
return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pReq);
|
||||
case TDMT_VND_QUERY_CONTINUE:
|
||||
return qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pReq);
|
||||
default:
|
||||
mError("unknown msg type:%d in query queue", pReq->msgType);
|
||||
return TSDB_CODE_VND_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t mndProcessFetchMsg(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
mTrace("msg:%p, in fetch queue is processing", pMsg);
|
||||
|
||||
mTrace("msg:%p, in query queue is processing", pMsg);
|
||||
switch (pMsg->msgType) {
|
||||
case TDMT_VND_QUERY:
|
||||
code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg);
|
||||
break;
|
||||
case TDMT_VND_QUERY_CONTINUE:
|
||||
code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg);
|
||||
break;
|
||||
case TDMT_VND_FETCH:
|
||||
return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
break;
|
||||
case TDMT_VND_DROP_TASK:
|
||||
return qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
break;
|
||||
case TDMT_VND_QUERY_HEARTBEAT:
|
||||
return qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg);
|
||||
break;
|
||||
default:
|
||||
mError("unknown msg type:%d in fetch queue", pMsg->msgType);
|
||||
return TSDB_CODE_VND_APP_ERROR;
|
||||
terrno = TSDB_CODE_VND_APP_ERROR;
|
||||
mError("unknown msg type:%d in query queue", pMsg->msgType);
|
||||
}
|
||||
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t mndInitQuery(SMnode *pMnode) {
|
||||
|
|
@ -59,9 +57,9 @@ int32_t mndInitQuery(SMnode *pMnode) {
|
|||
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessFetchMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessFetchMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessFetchMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessQueryMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessQueryMsg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue