huili
GitHub
commit
208978bd15
|
|
@ -40,14 +40,15 @@ def pre_test(){
|
|||
sh '''
|
||||
|
||||
cd ${WKC}
|
||||
rm -rf *
|
||||
git checkout develop
|
||||
git pull
|
||||
git fetch
|
||||
git checkout ${CHANGE_BRANCH}
|
||||
git merge develop
|
||||
cd ${WK}
|
||||
git reset --hard
|
||||
git checkout develop
|
||||
git pull
|
||||
cd ${WKC}
|
||||
rm -rf *
|
||||
mv ${WORKSPACE}/* .
|
||||
cd ${WK}
|
||||
export TZ=Asia/Harbin
|
||||
date
|
||||
|
|
@ -78,14 +79,25 @@ pipeline {
|
|||
changeRequest()
|
||||
}
|
||||
parallel {
|
||||
stage('python') {
|
||||
agent{label 'pytest'}
|
||||
stage('python_1') {
|
||||
agent{label 'p1'}
|
||||
steps {
|
||||
|
||||
pre_test()
|
||||
sh '''
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh pytest
|
||||
./test-all.sh p1
|
||||
date'''
|
||||
}
|
||||
}
|
||||
stage('python_2') {
|
||||
agent{label 'p2'}
|
||||
steps {
|
||||
|
||||
pre_test()
|
||||
sh '''
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh p2
|
||||
date'''
|
||||
}
|
||||
}
|
||||
|
|
@ -95,7 +107,7 @@ pipeline {
|
|||
pre_test()
|
||||
sh '''
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh b1
|
||||
./test-all.sh b1fq
|
||||
date'''
|
||||
}
|
||||
}
|
||||
|
|
@ -119,7 +131,7 @@ pipeline {
|
|||
sh '''
|
||||
date
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh b2
|
||||
./test-all.sh b2fq
|
||||
date
|
||||
'''
|
||||
}
|
||||
|
|
@ -140,7 +152,7 @@ pipeline {
|
|||
sh '''
|
||||
date
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh b3
|
||||
./test-all.sh b3fq
|
||||
date'''
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ go 1.14
|
|||
require (
|
||||
github.com/jmoiron/sqlx v1.2.0
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible
|
||||
github.com/taosdata/driver-go v0.0.0-20200727182616-1a3b1941c206
|
||||
github.com/taosdata/driver-go v0.0.0-20201113094317-050667e5b4d0
|
||||
go.uber.org/zap v1.14.1
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ PROJECT(TDengine)
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "2.0.11.0")
|
||||
SET(TD_VER_NUMBER "2.0.12.0")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ SELECT function<field_name>,…
|
|||
|
||||
超级表聚合查询，TDengine目前支持以下聚合\选择函数：sum、count、avg、first、last、min、max、top、bottom，以及针对全部或部分列的投影操作，使用方式与单表查询的计算过程相同。暂不支持其他类型的聚合计算和四则运算。当前所有的函数及计算过程均不支持嵌套的方式进行执行。
|
||||
|
||||
不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY <tag_name> 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY <tag_name> 语句指定，每个分组内部，时间序列是单调递增的。
|
||||
不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY <tag_name> 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY <tag_name> 语句指定，每个分组内部，时间序列是单调递增的。
|
||||
|
||||
使用SLIMIT/SOFFSET语句指定组间分页，即指定结果集中输出的最大组数以及对组起始的位置。使用LIMIT/OFFSET语句指定组内分页，即指定结果集中每个组内最多输出多少条记录以及记录起始的位置。
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ CREATE TABLE thermometer (ts timestamp, degree double)
|
|||
TAGS(location binary(20), type int)
|
||||
```
|
||||
|
||||
假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下：
|
||||
假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下：
|
||||
|
||||
```mysql
|
||||
CREATE TABLE therm1 USING thermometer TAGS (’beijing’, 1);
|
||||
|
|
|
|||
|
|
@ -77,15 +77,13 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
SHOW VARIABLES;
|
||||
```
|
||||
|
||||
|
||||
- **使用数据库**
|
||||
|
||||
|
||||
```mysql
|
||||
USE db_name;
|
||||
```
|
||||
使用/切换数据库
|
||||
|
||||
|
||||
- **删除数据库**
|
||||
```mysql
|
||||
DROP DATABASE [IF EXISTS] db_name;
|
||||
|
|
@ -120,7 +118,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
|
||||
**Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。
|
||||
|
||||
|
||||
- **显示系统所有数据库**
|
||||
```mysql
|
||||
SHOW DATABASES;
|
||||
|
|
@ -128,7 +125,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
|
||||
## 表管理
|
||||
- **创建数据表**
|
||||
|
||||
|
||||
```mysql
|
||||
CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]);
|
||||
```
|
||||
|
|
@ -153,7 +150,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
|
||||
显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。
|
||||
|
||||
|
||||
- **在线修改显示字符宽度**
|
||||
|
||||
```mysql
|
||||
|
|
@ -184,18 +180,18 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
|
||||
## 超级表STable管理
|
||||
- **创建超级表**
|
||||
|
||||
|
||||
```mysql
|
||||
CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]);
|
||||
```
|
||||
创建STable, 与创建表的SQL语法相似，但需指定TAGS字段的名称和类型
|
||||
|
||||
|
||||
说明：
|
||||
1) TAGS 列的数据类型不能是timestamp类型；
|
||||
2) TAGS 列名不能与其他列名相同；
|
||||
3) TAGS 列名不能为预留关键字；
|
||||
4) TAGS 最多允许128个，可以0个，总长度不超过16k个字符
|
||||
|
||||
|
||||
- **删除超级表**
|
||||
|
||||
```mysql
|
||||
|
|
@ -215,7 +211,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
```mysql
|
||||
DESCRIBE stb_name;
|
||||
```
|
||||
|
||||
|
||||
- **超级表增加列**
|
||||
|
||||
```mysql
|
||||
|
|
@ -230,11 +226,11 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
|
||||
## 超级表 STable 中 TAG 管理
|
||||
- **添加标签**
|
||||
|
||||
|
||||
```mysql
|
||||
ALTER TABLE stb_name ADD TAG new_tag_name tag_type;
|
||||
```
|
||||
为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符.
|
||||
为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符。
|
||||
|
||||
- **删除标签**
|
||||
|
||||
|
|
@ -265,36 +261,31 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
|
|||
```
|
||||
向表tb_name中插入一条记录
|
||||
|
||||
|
||||
- **插入一条记录，数据对应到指定的列**
|
||||
```mysql
|
||||
INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...)
|
||||
```
|
||||
向表tb_name中插入一条记录，数据对应到指定的列。SQL语句中没有出现的列，数据库将自动填充为NULL。主键（时间戳）不能为NULL。
|
||||
|
||||
|
||||
- **插入多条记录**
|
||||
```mysql
|
||||
INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...;
|
||||
```
|
||||
向表tb_name中插入多条记录
|
||||
|
||||
|
||||
- **按指定的列插入多条记录**
|
||||
```mysql
|
||||
INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
|
||||
```
|
||||
向表tb_name中按指定的列插入多条记录
|
||||
|
||||
|
||||
- **向多个表插入多条记录**
|
||||
```mysql
|
||||
INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)...
|
||||
INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)...
|
||||
tb2_name VALUES (field1_value1, ...)(field1_value2, ...)...;
|
||||
```
|
||||
同时向表tb1_name和tb2_name中分别插入多条记录
|
||||
|
||||
|
||||
- **同时向多个表按列插入多条记录**
|
||||
```mysql
|
||||
INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...)
|
||||
|
|
@ -320,7 +311,7 @@ SELECT select_expr [, select_expr ...]
|
|||
[FILL fill_val]
|
||||
[SLIDING fill_val]
|
||||
[GROUP BY col_list]
|
||||
[ORDER BY col_list { DESC | ASC }]
|
||||
[ORDER BY col_list { DESC | ASC }]
|
||||
[SLIMIT limit_val [, SOFFSET offset_val]]
|
||||
[LIMIT limit_val [, OFFSET offset_val]]
|
||||
[>> export_file]
|
||||
|
|
@ -382,7 +373,6 @@ taos> SELECT * FROM meters;
|
|||
Query OK, 9 row(s) in set (0.002022s)
|
||||
```
|
||||
|
||||
|
||||
通配符支持表名前缀，以下两个SQL语句均为返回全部的列：
|
||||
```mysql
|
||||
SELECT * FROM d1001;
|
||||
|
|
@ -592,11 +582,11 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中记录行数或某列的非空值个数。
|
||||
返回结果数据类型：长整型INT64。
|
||||
应用字段：应用全部字段。
|
||||
适用于：表、超级表。
|
||||
说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。
|
||||
功能说明：统计表/超级表中记录行数或某列的非空值个数。
|
||||
返回结果数据类型：长整型INT64。
|
||||
应用字段：应用全部字段。
|
||||
适用于：表、超级表。
|
||||
说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。
|
||||
|
||||
示例：
|
||||
```mysql
|
||||
|
|
@ -613,16 +603,15 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
Query OK, 1 row(s) in set (0.001075s)
|
||||
```
|
||||
|
||||
|
||||
- **AVG**
|
||||
```mysql
|
||||
SELECT AVG(field_name) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的平均值。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool字段。
|
||||
适用于：表、超级表。
|
||||
|
||||
功能说明：统计表/超级表中某列的平均值。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool字段。
|
||||
适用于：表、超级表。
|
||||
|
||||
示例：
|
||||
```mysql
|
||||
taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters;
|
||||
|
|
@ -642,8 +631,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT TWA(field_name) FROM tb_name WHERE clause;
|
||||
```
|
||||
功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：时间加权平均(time weighted average, TWA）查询需要指定查询时间段的 _开始时间_ 和 _结束时间_ 。
|
||||
适用于：表、超级表。
|
||||
|
|
@ -652,10 +641,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT SUM(field_name) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的和。
|
||||
返回结果数据类型：双精度浮点数Double和长整型INT64。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
适用于：表、超级表。
|
||||
功能说明：统计表/超级表中某列的和。
|
||||
返回结果数据类型：双精度浮点数Double和长整型INT64。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
适用于：表、超级表。
|
||||
|
||||
示例：
|
||||
```mysql
|
||||
|
|
@ -676,9 +665,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT STDDEV(field_name) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表中某列的均方差。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表中某列的均方差。
|
||||
返回结果数据类型：双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
适用于：表。
|
||||
|
||||
示例：
|
||||
|
|
@ -694,10 +683,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。
|
||||
返回结果数据类型：字符串表达式（斜率, 截距）。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：自变量是时间戳，因变量是该列的值。
|
||||
功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。
|
||||
返回结果数据类型：字符串表达式（斜率, 截距）。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：自变量是时间戳，因变量是该列的值。
|
||||
适用于：表。
|
||||
|
||||
示例：
|
||||
|
|
@ -715,8 +704,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的值最小值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
功能说明：统计表/超级表中某列的值最小值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
|
||||
示例：
|
||||
|
|
@ -738,8 +727,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的值最大值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
功能说明：统计表/超级表中某列的值最大值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
|
||||
示例：
|
||||
|
|
@ -757,14 +746,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
Query OK, 1 row(s) in set (0.000987s)
|
||||
```
|
||||
|
||||
|
||||
- **FIRST**
|
||||
```mysql
|
||||
SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的值最先写入的非NULL值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
功能说明：统计表/超级表中某列的值最先写入的非NULL值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
说明：1）如果要返回各个列的首个（时间戳最小）非NULL值，可以使用FIRST(*)；2) 如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；3) 如果结果集中所有列全部为NULL值，则不返回结果。
|
||||
|
||||
示例：
|
||||
|
|
@ -786,9 +774,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的值最后写入的非NULL值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
功能说明：统计表/超级表中某列的值最后写入的非NULL值。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
说明：1）如果要返回各个列的最后（时间戳最大）一个非NULL值，可以使用LAST(*)；2）如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；如果结果集中所有列全部为NULL值，则不返回结果。
|
||||
|
||||
示例：
|
||||
|
|
@ -810,10 +798,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。
|
||||
功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。
|
||||
|
||||
示例：
|
||||
```mysql
|
||||
|
|
@ -837,9 +825,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。
|
||||
|
||||
示例：
|
||||
|
|
@ -863,9 +851,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表中某列的值百分比分位数。
|
||||
返回结果数据类型： 双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表中某列的值百分比分位数。
|
||||
返回结果数据类型： 双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
|
||||
|
||||
示例：
|
||||
|
|
@ -881,9 +869,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。
|
||||
返回结果数据类型： 双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。
|
||||
返回结果数据类型： 双精度浮点数Double。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
|
||||
```mysql
|
||||
taos> SELECT APERCENTILE(current, 20) FROM d1001;
|
||||
|
|
@ -897,9 +885,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT LAST_ROW(field_name) FROM { tb_name | stb_name };
|
||||
```
|
||||
功能说明：返回表（超级表）的最后一条记录。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
功能说明：返回表（超级表）的最后一条记录。
|
||||
返回结果数据类型：同应用的字段。
|
||||
应用字段：所有字段。
|
||||
说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。
|
||||
|
||||
示例：
|
||||
|
|
@ -922,11 +910,11 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
```mysql
|
||||
SELECT DIFF(field_name) FROM tb_name [WHERE clause];
|
||||
```
|
||||
功能说明：统计表中某列的值与前一行对应值的差。
|
||||
返回结果数据类型： 同应用字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表中某列的值与前一行对应值的差。
|
||||
返回结果数据类型： 同应用字段。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：输出结果行数是范围内总行数减一，第一行没有结果输出。
|
||||
|
||||
|
||||
示例：
|
||||
```mysql
|
||||
taos> SELECT DIFF(current) FROM d1001;
|
||||
|
|
@ -937,14 +925,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
Query OK, 2 row(s) in set (0.001162s)
|
||||
```
|
||||
|
||||
|
||||
- **SPREAD**
|
||||
```mysql
|
||||
SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列的最大值和最小值之差。
|
||||
返回结果数据类型： 双精度浮点数。
|
||||
应用字段：不能应用在binary、nchar、bool类型字段。
|
||||
功能说明：统计表/超级表中某列的最大值和最小值之差。
|
||||
返回结果数据类型： 双精度浮点数。
|
||||
应用字段：不能应用在binary、nchar、bool类型字段。
|
||||
说明：可用于TIMESTAMP字段，此时表示记录的时间覆盖范围。
|
||||
|
||||
示例：
|
||||
|
|
@ -962,15 +949,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
Query OK, 1 row(s) in set (0.000836s)
|
||||
```
|
||||
|
||||
|
||||
- **四则运算**
|
||||
|
||||
```mysql
|
||||
SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name } [WHERE clause];
|
||||
```
|
||||
功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。
|
||||
返回结果数据类型：双精度浮点数。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。
|
||||
返回结果数据类型：双精度浮点数。
|
||||
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
|
||||
说明：1）支持两列或多列之间进行计算，可使用括号控制计算优先级;2）NULL字段不参与计算，如果参与计算的某行中包含NULL，该行的计算结果为NULL。
|
||||
|
||||
```mysql
|
||||
|
|
@ -987,12 +973,12 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
|
|||
TDengine支持按时间段进行聚合，可以将表中数据按照时间段进行切割后聚合生成结果，比如温度传感器每秒采集一次数据，但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下：
|
||||
|
||||
```mysql
|
||||
SELECT function_list FROM tb_name
|
||||
SELECT function_list FROM tb_name
|
||||
[WHERE where_condition]
|
||||
INTERVAL (interval [, offset])
|
||||
[FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
|
||||
|
||||
SELECT function_list FROM stb_name
|
||||
SELECT function_list FROM stb_name
|
||||
[WHERE where_condition]
|
||||
INTERVAL (interval [, offset])
|
||||
[FILL ({ VALUE | PREV | NULL | LINEAR})]
|
||||
|
|
@ -1000,18 +986,17 @@ SELECT function_list FROM stb_name
|
|||
```
|
||||
|
||||
- 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。
|
||||
- WHERE语句可以指定查询的起止时间和其他过滤条件
|
||||
- WHERE语句可以指定查询的起止时间和其他过滤条件
|
||||
- FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种：
|
||||
1. 不进行填充：NONE(默认填充模式)。
|
||||
|
||||
2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。
|
||||
|
||||
3. NULL填充：使用NULL填充数据。例如：fill(null)。
|
||||
|
||||
4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。
|
||||
|
||||
|
||||
说明：
|
||||
2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。
|
||||
|
||||
3. NULL填充：使用NULL填充数据。例如：fill(null)。
|
||||
|
||||
4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。
|
||||
|
||||
说明：
|
||||
1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。
|
||||
2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
|
||||
3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。
|
||||
|
|
@ -1040,8 +1025,6 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER
|
|||
- SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M
|
||||
- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制
|
||||
|
||||
|
||||
|
||||
## TAOS SQL其他约定
|
||||
|
||||
**group by的限制**
|
||||
|
|
@ -1054,6 +1037,4 @@ TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持
|
|||
|
||||
**is not null与不为空的表达式适用范围**
|
||||
|
||||
is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。
|
||||
|
||||
|
||||
is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
# TDengine 2.0 错误码以及对应的十进制码
|
||||
|
||||
|
||||
| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） |
|
||||
|-----------------------| :---: | :---------: | :------------------------ | ---------------- |
|
||||
|TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647|
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb)
|
||||
```
|
||||
|
||||
示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。
|
||||
示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。
|
||||
|
||||
实际运行的系统往往会根据数据特点的不同，将数据存放在不同的DB里。因此做规划时，也需要考虑。
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景
|
|||
Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
|
||||
```
|
||||
|
||||
示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。
|
||||
示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。
|
||||
|
||||
用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。
|
||||
|
||||
|
|
@ -121,7 +121,7 @@ taosd -C
|
|||
- replica：副本个数，取值范围：1-3。单位为个，默认值：1
|
||||
- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms
|
||||
|
||||
对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL：
|
||||
对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL：
|
||||
|
||||
```
|
||||
create database demo days 10 cache 32 blocks 8 replica 3;
|
||||
|
|
@ -148,8 +148,8 @@ ALTER DNODE <dnode_id> <config>
|
|||
```
|
||||
|
||||
- dnode_id: 可以通过SQL语句"SHOW DNODES"命令获取
|
||||
- config: 要调整的日志参数，在如下列表中取值
|
||||
> resetlog 截断旧日志文件，创建一个新日志文件
|
||||
- config: 要调整的日志参数，在如下列表中取值
|
||||
> resetlog 截断旧日志文件，创建一个新日志文件
|
||||
> debugFlag < 131 | 135 | 143 > 设置debugFlag为131、135或者143
|
||||
|
||||
例如：
|
||||
|
|
@ -157,9 +157,9 @@ ALTER DNODE <dnode_id> <config>
|
|||
alter dnode 1 debugFlag 135;
|
||||
```
|
||||
|
||||
## 客户端配置
|
||||
## 客户端配置
|
||||
|
||||
TDengine系统的前台交互客户端应用程序为taos，以及应用驱动，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见[Shell命令行程序](https://www.taosdata.com/cn/documentation/administrator/#_TDengine_Shell命令行程序)。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。
|
||||
TDengine系统的前台交互客户端应用程序为taos，以及应用驱动，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见<a href="https://www.taosdata.com/cn/documentation/administrator/#_TDengine_Shell命令行程序">Shell命令行程序</a>。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。
|
||||
|
||||
**2.0.10.0 之后版本支持命令行以下参数显示当前客户端参数的配置**
|
||||
|
||||
|
|
@ -176,7 +176,7 @@ taos -C 或 taos --dump-config
|
|||
- locale
|
||||
|
||||
默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
|
||||
|
||||
|
||||
TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是，编码正确性是客户端来保证。因此，如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。
|
||||
|
||||
客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。
|
||||
|
|
@ -186,7 +186,7 @@ taos -C 或 taos --dump-config
|
|||
- charset
|
||||
|
||||
默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
|
||||
|
||||
|
||||
如果配置文件中不设置charset，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，则中断启动过程。
|
||||
|
||||
在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如：
|
||||
|
|
@ -242,12 +242,11 @@ taos -C 或 taos --dump-config
|
|||
为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
|
||||
|
||||
启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。
|
||||
|
||||
|
||||
- maxBinaryDisplayWidth
|
||||
|
||||
Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。
|
||||
|
||||
|
||||
## 用户管理
|
||||
|
||||
系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下：
|
||||
|
|
@ -280,7 +279,7 @@ ALTER USER <user_name> PRIVILEGE <super|write|read>;
|
|||
SHOW USERS;
|
||||
```
|
||||
|
||||
显示所有用户
|
||||
显示所有用户
|
||||
|
||||
**注意：**SQL 语法中，< >表示需要用户输入的部分，但请不要输入< >本身
|
||||
|
||||
|
|
@ -428,8 +427,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
|
|||
|
||||
您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
|
||||
|
||||
|
||||
|
||||
## TDengine参数限制与保留关键字
|
||||
|
||||
- 数据库名：不能包含“.”以及特殊字符，不能超过32个字符
|
||||
|
|
@ -448,8 +445,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
|
|||
- 库的个数：仅受节点个数限制
|
||||
- 单个库上虚拟节点个数：不能超过64个
|
||||
|
||||
|
||||
|
||||
目前TDengine有将近200个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下：
|
||||
|
||||
| 关键字列表 | | | | |
|
||||
|
|
@ -489,5 +484,4 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
|
|||
| COMP | GE | METRIC | SELECT | VIEW |
|
||||
| CONCAT | GLOB | METRICS | SEMI | WAVG |
|
||||
| CONFIGS | GRANTS | MIN | SET | WHERE |
|
||||
| CONFLICT | GROUP | | | |
|
||||
|
||||
| CONFLICT | GROUP | | | |
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#数据模型和整体架构
|
||||
# 数据模型和整体架构
|
||||
|
||||
## 数据模型
|
||||
### 物联网典型场景
|
||||
|
|
@ -102,7 +102,7 @@
|
|||
|
||||
每一条记录都有设备ID，时间戳，采集的物理量(如上图中的电流、电压、相位），还有与每个设备相关的静态标签（如上述表一中的位置Location和分组groupId）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
|
||||
|
||||
### 数据特征
|
||||
### 数据特征
|
||||
除时序特征外，仔细研究发现，物联网、车联网、运维监测类数据还具有很多其他明显的特征：
|
||||
|
||||
1. 数据高度结构化；
|
||||
|
|
@ -121,7 +121,7 @@
|
|||
### 关系型数据库模型
|
||||
因为采集的数据一般是结构化数据，同时为降低学习门槛，TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库，然后创建表，之后才能插入或查询数据。TDengine采用的是结构化存储，而不是NoSQL的key-value存储。
|
||||
|
||||
### 一个数据采集点一张表
|
||||
### 一个数据采集点一张表
|
||||
为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的d1001, d1002, d1003, d1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点：
|
||||
|
||||
1. 能保证一个采集点的数据在存储介质上是以块为单位连续存储的。如果读取一个时间段的数据，它能大幅减少随机读取操作，成数量级的提升读取和查询速度。
|
||||
|
|
@ -150,7 +150,7 @@ TDengine 分布式架构的逻辑结构图如下：
|
|||
<center> 图 1 TDengine架构示意图 </center>
|
||||
一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
|
||||
|
||||
**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。
|
||||
**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">《一篇文章说清楚TDengine的FQDN》</a>。
|
||||
|
||||
**数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。
|
||||
|
||||
|
|
@ -356,7 +356,7 @@ SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成
|
|||
|
||||
客户端在获取查询结果的时候，dnode的查询执行队列中的工作线程会等待vnode执行线程执行完成，才能将查询结果返回到请求的客户端。
|
||||
|
||||
### 按时间轴聚合、降采样、插值
|
||||
### 按时间轴聚合、降采样、插值
|
||||
|
||||
时序数据有别于普通数据的显著特征是每条记录均具有时间戳，因此针对具有时间戳数据在时间轴上进行聚合是不同于普通数据库的重要功能。从这点上来看，与流计算引擎的窗口查询有相似的地方。
|
||||
|
||||
|
|
|
|||
|
|
@ -1,41 +1,162 @@
|
|||
# 连接器
|
||||
|
||||
TDengine提供了丰富的应用程序开发接口，其中包括C/C++、C# 、Java、Python、Go、Node.js、RESTful 等，便于用户快速开发应用。
|
||||
TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用。
|
||||
|
||||
|
||||
|
||||
目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下：
|
||||
目前TDengine的连接器可支持的平台广泛，包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下：
|
||||
|
||||
| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** |
|
||||
| ----------- | --------------- | --------------- | --------------- | --------------- | --------- | --------- | --------------- | ---------------- | -------------- |
|
||||
| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** |
|
||||
| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
|
||||
| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
|
||||
| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● |
|
||||
| **C/C++** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ |
|
||||
| **JDBC** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ |
|
||||
| **Python** | ● | ● | ● | ○ | ● | ● | ○ | -- | ○ |
|
||||
| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- |
|
||||
| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- |
|
||||
| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- |
|
||||
| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● |
|
||||
| **RESTful** | ● | ● | ● | ● | ● | ● | ○ | ○ | ○ |
|
||||
|
||||
其中 ● 表示经过官方测试验证， ○ 表示非官方测试验证。
|
||||
|
||||
注意：
|
||||
|
||||
* 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。
|
||||
* 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
|
||||
* 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。
|
||||
|
||||
## 安装连接器驱动步骤
|
||||
|
||||
服务器应该已经安装TDengine服务端安装包。连接器驱动安装步骤如下：
|
||||
|
||||
**Linux**
|
||||
|
||||
**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载**
|
||||
|
||||
* X64硬件环境：TDengine-client-2.x.x.x-Linux-x64.tar.gz
|
||||
|
||||
* ARM64硬件环境：TDengine-client-2.x.x.x-Linux-aarch64.tar.gz
|
||||
|
||||
* ARM32硬件环境：TDengine-client-2.x.x.x-Linux-aarch32.tar.gz
|
||||
|
||||
**2. 解压缩软件包**
|
||||
|
||||
将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令：
|
||||
|
||||
`tar -xzvf TDengine-client-xxxxxxxxx.tar.gz`
|
||||
|
||||
其中xxxxxxx需要替换为实际版本的字符串。
|
||||
|
||||
**3. 执行安装脚本**
|
||||
|
||||
解压软件包之后，会在解压目录下看到以下文件(目录)：
|
||||
|
||||
*install_client.sh*：安装脚本，用于应用驱动程序
|
||||
*taos.tar.gz*：应用驱动安装包
|
||||
*driver*：TDengine应用驱动driver
|
||||
*connector*: 各种编程语言连接器（go/grafanaplugin/nodejs/python/JDBC）
|
||||
*examples*: 各种编程语言的示例程序(c/C#/go/JDBC/matlab/python/R)
|
||||
|
||||
运行install_client.sh进行安装
|
||||
|
||||
**4. 配置taos.cfg**
|
||||
|
||||
编辑taos.cfg文件(默认路径/etc/taos/taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030
|
||||
|
||||
**提示： 如本机没有部署TDengine服务，仅安装了应用驱动，则taos.cfg中仅需配置firstEP，无需配置FQDN。**
|
||||
|
||||
**Windows x64/x86**
|
||||
|
||||
**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载 ：**
|
||||
|
||||
* X64硬件环境：TDengine-client-2.X.X.X-Windows-x64.exe
|
||||
|
||||
* X86硬件环境：TDengine-client-2.X.X.X-Windows-x86.exe
|
||||
|
||||
**2. 执行安装程序，按提示选择默认值，完成安装**
|
||||
|
||||
**3. 安装路径**
|
||||
|
||||
默认安装路径为：C:\TDengine，其中包括以下文件(目录)：
|
||||
|
||||
*taos.exe*：taos shell命令行程序
|
||||
|
||||
*cfg* : 配置文件目录
|
||||
*driver*: 应用驱动动态链接库
|
||||
*examples*: 示例程序 bash/C/C#/go/JDBC/Python/Node.js
|
||||
*include*: 头文件
|
||||
*log* : 日志文件
|
||||
*unins000.exe*: 卸载程序
|
||||
|
||||
**4. 配置taos.cfg**
|
||||
|
||||
编辑taos.cfg文件(默认路径C:\TDengine\cfg\taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030
|
||||
|
||||
**提示：**
|
||||
|
||||
**1. 如利用FQDN连接服务器，必须确认本机网络环境DNS已配置好，或在hosts文件中添加FQDN寻址记录，如编辑C:\Windows\system32\drivers\etc\hosts，添加如下的记录：** **192.168.1.99 h1.taos.com**
|
||||
|
||||
**2．卸载：运行unins000.exe可卸载TDengine应用驱动。**
|
||||
|
||||
**安装验证**
|
||||
|
||||
以上安装和配置完成后，并确认TDengine服务已经正常启动运行，此时可以执行taos客户端进行登录。
|
||||
|
||||
**Linux环境：**
|
||||
|
||||
在linux shell下直接执行 taos，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下：
|
||||
|
||||
```mysql
|
||||
$ taos
|
||||
Welcome to the TDengine shell from Linux, Client Version:2.0.5.0
|
||||
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
|
||||
taos> show databases;
|
||||
name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB)| blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
|
||||
=========================================================================================================================================================================================================================
|
||||
test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16| 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
|
||||
log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1| 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
|
||||
Query OK, 2 row(s) in set (0.001198s)
|
||||
taos>
|
||||
```
|
||||
|
||||
**Windows（x64/x86）环境：**
|
||||
|
||||
在cmd下进入到c:\TDengine目录下直接执行 taos.exe，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下：
|
||||
|
||||
```mysql
|
||||
C:\TDengine>taos
|
||||
Welcome to the TDengine shell from Linux, Client Version:2.0.5.0
|
||||
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
|
||||
taos> show databases;
|
||||
name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
|
||||
===================================================================================================================================================================================================================================================================
|
||||
test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
|
||||
log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
|
||||
Query OK, 2 row(s) in set (0.045000s)
|
||||
taos>
|
||||
```
|
||||
|
||||
## C/C++ Connector
|
||||
|
||||
**C/C++连接器支持的系统有**：
|
||||
|
||||
| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 |
|
||||
| ------------ | ------------ | -------- | -------- | -------- | ---------- |
|
||||
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
|
||||
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** |
|
||||
|
||||
C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）：
|
||||
|
||||
```C
|
||||
#include <taos.h>
|
||||
```
|
||||
|
||||
在编译时需要链接TDengine动态库 _libtaos.so_ （安装后，位于 _/usr/local/taos/driver_，gcc编译时，请加上 -ltaos）。
|
||||
注意：
|
||||
|
||||
如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。
|
||||
* 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ，安装后，位于 _/usr/local/taos/driver_。Windows为 taos.dll，安装后位于 *C:\TDengine*。
|
||||
* 如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。
|
||||
|
||||
使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c。
|
||||
|
||||
### 基础API
|
||||
|
||||
|
|
@ -45,22 +166,18 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
|
||||
初始化运行环境。如果应用没有主动调用该API，那么应用在调用`taos_connect`时将自动调用，故应用程序一般无需手动调用该API。
|
||||
|
||||
|
||||
- `void taos_cleanup()`
|
||||
|
||||
清理运行环境，应用退出前应调用此API。
|
||||
|
||||
|
||||
- `int taos_options(TSDB_OPTION option, const void * arg, ...)`
|
||||
|
||||
设置客户端选项，目前只支持时区设置（_TSDB_OPTION_TIMEZONE_）和编码设置（_TSDB_OPTION_LOCALE_）。时区和编码默认为操作系统当前设置。
|
||||
|
||||
|
||||
- `char *taos_get_client_info()`
|
||||
|
||||
获取客户端版本信息。
|
||||
|
||||
|
||||
- `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)`
|
||||
|
||||
创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含：
|
||||
|
|
@ -73,23 +190,18 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
|
||||
返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。
|
||||
|
||||
|
||||
- `char *taos_get_server_info(TAOS *taos)`
|
||||
|
||||
获取服务端版本信息。
|
||||
|
||||
|
||||
- `int taos_select_db(TAOS *taos, const char *db)`
|
||||
|
||||
将当前的缺省数据库设置为`db`。
|
||||
|
||||
|
||||
- `void taos_close(TAOS *taos)`
|
||||
|
||||
关闭连接, 其中`taos`是`taos_connect`函数返回的指针。
|
||||
|
||||
|
||||
|
||||
### 同步查询API
|
||||
|
||||
传统的数据库操作API，都属于同步操作。应用调用API后，一直处于阻塞状态，直到服务器返回结果。TDengine支持如下API：
|
||||
|
|
@ -98,37 +210,30 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
|
||||
该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。
|
||||
|
||||
|
||||
- `int taos_result_precision(TAOS_RES *res)`
|
||||
|
||||
返回结果集时间戳字段的精度，`0` 代表毫秒，`1` 代表微秒，`2` 代表纳秒。
|
||||
|
||||
|
||||
- `TAOS_ROW taos_fetch_row(TAOS_RES *res)`
|
||||
|
||||
按行获取查询结果集中的数据。
|
||||
|
||||
|
||||
- `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)`
|
||||
|
||||
批量获取查询结果集中的数据，返回值为获取到的数据的行数。
|
||||
|
||||
|
||||
- `int taos_num_fields(TAOS_RES *res)` 和 `int taos_field_count(TAOS_RES *res)`
|
||||
|
||||
这两个API等价，用于获取查询结果集中的列数。
|
||||
|
||||
|
||||
- `int* taos_fetch_lengths(TAOS_RES *res)`
|
||||
|
||||
获取结果集中每个字段的长度。 返回值是一个数组，其长度为结果集的列数。
|
||||
|
||||
|
||||
- `int taos_affected_rows(TAOS_RES *res)`
|
||||
|
||||
获取被所执行的 SQL 语句影响的行数。
|
||||
|
||||
|
||||
- `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)`
|
||||
|
||||
获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下：
|
||||
|
|
@ -141,30 +246,24 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
} TAOS_FIELD;
|
||||
```
|
||||
|
||||
|
||||
- `void taos_stop_query(TAOS_RES *res)`
|
||||
|
||||
停止一个查询的执行。
|
||||
|
||||
|
||||
- `void taos_free_result(TAOS_RES *res)`
|
||||
|
||||
释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。
|
||||
|
||||
|
||||
- `char *taos_errstr(TAOS_RES *res)`
|
||||
|
||||
获取最近一次API调用失败的原因,返回值为字符串。
|
||||
|
||||
|
||||
- `char *taos_errno(TAOS_RES *res)`
|
||||
|
||||
获取最近一次API调用失败的原因，返回值为错误代码。
|
||||
|
||||
|
||||
**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
|
||||
|
||||
|
||||
### 异步查询API
|
||||
|
||||
同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。
|
||||
|
|
@ -189,7 +288,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
* res：`taos_query_a`回调时返回的结果集
|
||||
* fp：回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体；`numOfRows`是获取到的数据的行数（不是整个查询结果集的函数）。 在回调函数中，应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后，应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理，直到返回的记录数（numOfRows）为零（结果返回完成）或记录数为负值（查询出错）。
|
||||
|
||||
|
||||
- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);`
|
||||
|
||||
异步获取一条记录。其中：
|
||||
|
|
@ -199,7 +297,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
|
|||
|
||||
TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。
|
||||
|
||||
|
||||
### 参数绑定API
|
||||
|
||||
除了直接调用 `taos_query` 进行查询，TDengine也提供了支持参数绑定的Prepare API，与 MySQL 一样，这些API目前也仅支持用问号`?`来代表待绑定的参数，具体如下：
|
||||
|
|
@ -244,7 +341,6 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线
|
|||
|
||||
执行完毕，释放所有资源。
|
||||
|
||||
|
||||
### 连续查询接口
|
||||
|
||||
TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时间段，对一张或多张数据库的表(数据流)进行各种实时聚合计算操作。操作简单，仅有打开、关闭流的API。具体如下：
|
||||
|
|
@ -261,11 +357,9 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
|
|||
|
||||
返回值为NULL，表示创建成功，返回值不为空，表示成功。
|
||||
|
||||
|
||||
- `void taos_close_stream (TAOS_STREAM *tstr)`
|
||||
|
||||
关闭数据流，其中提供的参数是taos_open_stream的返回值。用户停止流式计算的时候，务必关闭该数据流。
|
||||
|
||||
|
||||
### 数据订阅接口
|
||||
|
||||
|
|
@ -290,7 +384,6 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
|
|||
* param：调用 `taos_subscribe`时客户程序提供的附加参数
|
||||
* code：错误码
|
||||
|
||||
|
||||
* `TAOS_RES *taos_consume(TAOS_SUB *tsub)`
|
||||
|
||||
同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。
|
||||
|
|
@ -299,11 +392,10 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
|
|||
|
||||
取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。
|
||||
|
||||
|
||||
## Python Connector
|
||||
|
||||
### 安装准备
|
||||
* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）][4]
|
||||
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
|
||||
* 已安装python 2.7 or >= 3.4
|
||||
* 已安装pip 或 pip3
|
||||
|
||||
|
|
@ -323,12 +415,12 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
|
|||
在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos <em>cmd</em> 命令行界面
|
||||
```cmd
|
||||
cd C:\TDengine\connector\python\windows
|
||||
pip install python2\
|
||||
python -m pip install python2\
|
||||
```
|
||||
或
|
||||
```cmd
|
||||
cd C:\TDengine\connector\python\windows
|
||||
pip install python3\
|
||||
python -m pip install python3\
|
||||
```
|
||||
|
||||
*如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。
|
||||
|
|
@ -407,7 +499,6 @@ for d in data:
|
|||
sub.close()
|
||||
```
|
||||
|
||||
|
||||
* 关闭连接
|
||||
```python
|
||||
c1.close()
|
||||
|
|
@ -431,7 +522,6 @@ conn.close()
|
|||
|
||||
用于生成taos.TDengineConnection的实例。
|
||||
|
||||
|
||||
### Python客户端使用示例代码
|
||||
在tests/examples/python中，我们提供了一个示例Python程序read_example.py，可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后，通过import taos引入taos类。主要步骤如下
|
||||
|
||||
|
|
@ -638,31 +728,39 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间
|
|||
|
||||
## CSharp Connector
|
||||
|
||||
在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
|
||||
C#连接器支持的系统有：Linux 64/Windows x64/Windows x86
|
||||
|
||||
#### 安装TDengine客户端
|
||||
### 安装准备
|
||||
|
||||
C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。
|
||||
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
|
||||
* .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。
|
||||
* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
|
||||
|
||||
安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件
|
||||
### 安装验证
|
||||
|
||||
- TDengineDriver.cs 调用taos.dll文件的Native C方法
|
||||
- TDengineTest.cs 参考程序示例
|
||||
运行install_directory/examples/C#/C#Checker/C#Checker.exe
|
||||
|
||||
在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件
|
||||
```cmd
|
||||
cd {install_directory}/examples/C#/C#Checker
|
||||
csc /optimize *.cs
|
||||
C#Checker.exe -h <fqdn>
|
||||
```
|
||||
|
||||
#### 使用方法
|
||||
### C#连接器的使用
|
||||
|
||||
- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
|
||||
- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法
|
||||
- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中
|
||||
在Windows系统上，.NET应用程序可以使用TDengine的.NET接口来执行所有数据库的操作。使用.NET接口的步骤如下所示：
|
||||
|
||||
#### 注意事项
|
||||
1. 将.NET接口文件TDengineDrivercs.cs加入到应用程序所在.NET项目中。
|
||||
2. 用户可以参考TDengineTest.cs来定义数据库连接参数，以及如何执行数据插入、查询等操作；
|
||||
|
||||
- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。
|
||||
- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。
|
||||
此.NET接口需要用到taos.dll文件，所以在执行应用程序前，拷贝Windows客户端install_directory/driver目录中的taos.dll文件到.NET项目最后生成.exe可执行文件所在文件夹。之后运行exe文件，即可访问TDengine数据库并做插入、查询等操作。
|
||||
|
||||
#### 第三方驱动
|
||||
**注意：**
|
||||
|
||||
1. TDengine V2.0.3.0之后同时支持32位和64位Windows系统，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请选择对应的“X86” 或“x64”。
|
||||
2. 此.NET接口目前已经在Visual Studio 2015/2017中验证过，其它VS版本尚待验证。
|
||||
|
||||
### 第三方驱动
|
||||
|
||||
Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考
|
||||
|
||||
|
|
@ -673,10 +771,15 @@ https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos
|
|||
https://www.taosdata.com/blog/2020/11/02/1901.html
|
||||
```
|
||||
|
||||
|
||||
## Go Connector
|
||||
|
||||
TDengine提供了GO驱动程序`taosSql`. `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`
|
||||
### 安装准备
|
||||
|
||||
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
|
||||
|
||||
TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。
|
||||
|
||||
使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go。
|
||||
|
||||
```Go
|
||||
import (
|
||||
|
|
@ -684,9 +787,9 @@ import (
|
|||
_ "github.com/taosdata/driver-go/taosSql"
|
||||
)
|
||||
```
|
||||
**建议Go版本是1.13或以上，并开启模块支持：**
|
||||
**建议使用Go版本1.13或以上，并开启模块支持：**
|
||||
|
||||
```
|
||||
```bash
|
||||
go env -w GO111MODULE=on
|
||||
go env -w GOPROXY=https://goproxy.io,direct
|
||||
```
|
||||
|
|
@ -725,48 +828,86 @@ go env -w GOPROXY=https://goproxy.io,direct
|
|||
|
||||
## Node.js Connector
|
||||
|
||||
TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)：
|
||||
Node.js连接器支持的系统有：
|
||||
|
||||
首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器.
|
||||
| **CPU类型** | x64（64bit） | | | aarch64 | aarch32 |
|
||||
| ------------ | ------------ | -------- | -------- | -------- | -------- |
|
||||
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
|
||||
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** |
|
||||
|
||||
### 安装准备
|
||||
|
||||
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
|
||||
|
||||
### 安装Node.js连接器
|
||||
|
||||
用户可以通过<a href="https://www.npmjs.com/">npm</a>来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下：
|
||||
|
||||
首先，通过<a href="https://www.npmjs.com/">npm</a>安装node.js 连接器.
|
||||
|
||||
```bash
|
||||
npm install td2.0-connector
|
||||
```
|
||||
我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下
|
||||
|
||||
我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
|
||||
我们使用<a href="https://github.com/nodejs/node-gyp">node-gyp</a>和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
|
||||
|
||||
### Linux
|
||||
|
||||
- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
|
||||
- `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。
|
||||
- `make`
|
||||
- c语言编译器比如[GCC](https://gcc.gnu.org)
|
||||
- c语言编译器比如<a href="https://gcc.gnu.org">GCC</a>
|
||||
|
||||
### Windows
|
||||
|
||||
#### 安装方法1
|
||||
|
||||
使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
|
||||
使用微软的<a href="https://github.com/felixrieseberg/windows-build-tools">windows-build-tools</a>在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
|
||||
|
||||
#### 安装方法2
|
||||
|
||||
手动安装以下工具:
|
||||
|
||||
- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community)
|
||||
- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7`
|
||||
- 安装Visual Studio相关：<a href="https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools>Visual Studio Build 工具</a> 或者 <a href="https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community">Visual Studio 2017 Community</a>
|
||||
- 安装 <a href="https://www.python.org/downloads/">Python</a> 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7`
|
||||
- 进入`cmd`命令行界面, `npm config set msvs_version 2017`
|
||||
|
||||
如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules)
|
||||
如果以上步骤不能成功执行, 可以参考微软的node.js用户手册<a href="https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules">Microsoft's Node.js Guidelines for Windows</a>
|
||||
|
||||
如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64".
|
||||
|
||||
### 使用方法
|
||||
### 示例程序
|
||||
|
||||
示例程序源码位于install_directory/examples/nodejs，有：
|
||||
|
||||
Node-example.js node.js示例源程序
|
||||
Node-example-raw.js
|
||||
|
||||
### 安装验证
|
||||
|
||||
在安装好TDengine客户端后，使用nodejsChecker.js程序能够验证当前环境是否支持nodejs方式访问Tdengine。
|
||||
|
||||
验证方法：
|
||||
|
||||
1. 新建安装验证目录，例如：~/tdengine-test，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。
|
||||
|
||||
2. 在命令中执行以下命令：
|
||||
|
||||
```bash
|
||||
npm init -y
|
||||
npm install td2.0-connector
|
||||
node nodejsChecker.js host=localhost
|
||||
```
|
||||
|
||||
3. 执行以上步骤后，在命令行会输出nodejs连接Tdengine实例，并执行简答插入和查询的结果。
|
||||
|
||||
### Node.js连接器的使用
|
||||
|
||||
(http://docs.taosdata.com/node)
|
||||
以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node)
|
||||
以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考<a href="http://docs.taosdata.com/node">该文档</a>
|
||||
|
||||
#### 连接
|
||||
#### 建立连接
|
||||
|
||||
使用node.js连接器时，必须先<em>require</em> ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信
|
||||
|
||||
|
|
@ -782,6 +923,26 @@ var cursor = conn.cursor(); // Initializing a new cursor
|
|||
conn.close();
|
||||
```
|
||||
|
||||
#### 执行SQL和插入数据
|
||||
|
||||
对于DDL语句（例如create database、create table、use等），可以使用cursor的execute方法。代码如下：
|
||||
|
||||
```js
|
||||
cursor.execute('create database if not exists test;')
|
||||
```
|
||||
|
||||
以上代码创建了一个名称为test的数据库。对于DDL语句，一般没有返回值，cursor的execute返回值为0。
|
||||
|
||||
对于Insert语句，代码如下：
|
||||
|
||||
```js
|
||||
var affectRows = cursor.execute('insert into test.weather values(now, 22.3, 34);')
|
||||
```
|
||||
|
||||
execute方法的返回值为该语句影响的行数，上面的sql向test库的weather表中，插入了一条数据，则返回值affectRows为1。
|
||||
|
||||
TDengine目前还不支持update和delete语句。
|
||||
|
||||
#### 查询
|
||||
|
||||
可通过 ```cursor.query``` 函数来查询数据库。
|
||||
|
|
@ -808,7 +969,6 @@ query.execute().then(function(result) {
|
|||
```
|
||||
如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下：
|
||||
|
||||
|
||||
```javascript
|
||||
var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true)
|
||||
promise.then(function(result) {
|
||||
|
|
@ -828,12 +988,7 @@ promise2.then(function(result) {
|
|||
})
|
||||
```
|
||||
|
||||
|
||||
### 示例
|
||||
[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
|
||||
|
||||
[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.
|
||||
|
||||
|
||||
[4]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client
|
||||
<a href="https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js">这里</a>提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
|
||||
|
||||
<a href="https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js">这里</a>同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.
|
||||
|
|
@ -1,17 +1,21 @@
|
|||
# Java Connector
|
||||
|
||||
Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。
|
||||
Java连接器支持的系统有：
|
||||
| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 |
|
||||
| ------------ | ------------ | -------- | -------- | -------- | -------- |
|
||||
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
|
||||
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** |
|
||||
|
||||
TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
|
||||
|
||||
由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
|
||||
|
||||
* libtaos.so
|
||||
* libtaos.so
|
||||
在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
|
||||
|
||||
|
||||
* taos.dll
|
||||
在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
|
||||
|
||||
|
||||
> 注意：在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。
|
||||
|
||||
TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点：
|
||||
|
|
@ -21,7 +25,6 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一
|
|||
* 目前不支持表间的 union 操作。
|
||||
* 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。
|
||||
|
||||
|
||||
## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
|
||||
|
||||
| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
|
||||
|
|
@ -70,7 +73,6 @@ maven 项目中使用如下 pom.xml 配置即可：
|
|||
|
||||
下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。
|
||||
|
||||
|
||||
## 使用说明
|
||||
|
||||
### 获取连接
|
||||
|
|
@ -212,7 +214,6 @@ while(resultSet.next()){
|
|||
```
|
||||
> 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
|
||||
|
||||
|
||||
### 订阅
|
||||
|
||||
#### 创建
|
||||
|
|
@ -227,7 +228,7 @@ TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from met
|
|||
* sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据
|
||||
* restart：如果订阅已经存在，是重新开始，还是继续之前的订阅
|
||||
|
||||
如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic' 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
|
||||
如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
|
||||
|
||||
#### 消费数据
|
||||
|
||||
|
|
@ -255,7 +256,6 @@ sub.close(true);
|
|||
|
||||
`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。
|
||||
|
||||
|
||||
### 关闭资源
|
||||
|
||||
```java
|
||||
|
|
@ -263,7 +263,7 @@ resultSet.close();
|
|||
stmt.close();
|
||||
conn.close();
|
||||
```
|
||||
> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。
|
||||
> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。
|
||||
|
||||
## 与连接池使用
|
||||
|
||||
|
|
@ -289,18 +289,18 @@ conn.close();
|
|||
config.setMinimumIdle(3); //minimum number of idle connection
|
||||
config.setMaximumPoolSize(10); //maximum number of connection in the pool
|
||||
config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool
|
||||
config.setIdleTimeout(60000); // max idle time for recycle idle connection
|
||||
config.setIdleTimeout(60000); // max idle time for recycle idle connection
|
||||
config.setConnectionTestQuery("describe log.dn"); //validation query
|
||||
config.setValidationTimeout(3000); //validation query timeout
|
||||
|
||||
HikariDataSource ds = new HikariDataSource(config); //create datasource
|
||||
|
||||
|
||||
Connection connection = ds.getConnection(); // get connection
|
||||
Statement statement = connection.createStatement(); // get statement
|
||||
|
||||
//query or insert
|
||||
|
||||
//query or insert
|
||||
// ...
|
||||
|
||||
|
||||
connection.close(); // put back to conneciton pool
|
||||
}
|
||||
```
|
||||
|
|
@ -342,7 +342,7 @@ public static void main(String[] args) throws Exception {
|
|||
properties.put("testWhileIdle","true"); // test connection while idle
|
||||
properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true
|
||||
properties.put("testOnReturn","false"); // don't need while testWhileIdle is true
|
||||
|
||||
|
||||
//create druid datasource
|
||||
DataSource ds = DruidDataSourceFactory.createDataSource(properties);
|
||||
Connection connection = ds.getConnection(); // get connection
|
||||
|
|
@ -376,15 +376,15 @@ Query OK, 1 row(s) in set (0.000141s)
|
|||
## 常见问题
|
||||
|
||||
* java.lang.UnsatisfiedLinkError: no taos in java.library.path
|
||||
|
||||
|
||||
**原因**：程序没有找到依赖的本地函数库 taos。
|
||||
|
||||
|
||||
**解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。
|
||||
|
||||
|
||||
* java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform
|
||||
|
||||
|
||||
**原因**：目前 TDengine 只支持 64 位 JDK。
|
||||
|
||||
|
||||
**解决方法**：重新安装 64 位 JDK。
|
||||
|
||||
* 其它问题请参考 [Issues][7]
|
||||
|
|
@ -399,7 +399,7 @@ Query OK, 1 row(s) in set (0.000141s)
|
|||
[8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver
|
||||
[9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver
|
||||
[10]: https://maven.aliyun.com/mvn/search
|
||||
[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
|
||||
[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
|
||||
[12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo
|
||||
[13]: https://www.taosdata.com/cn/documentation20/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE
|
||||
[14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client
|
||||
|
|
|
|||
|
|
@ -27,43 +27,40 @@
|
|||
* 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6042的访问权限
|
||||
* 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname
|
||||
* 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端
|
||||
|
||||
|
||||
2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用
|
||||
|
||||
3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd*
|
||||
|
||||
4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。
|
||||
4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">一篇文章说清楚TDengine的FQDN</a>。
|
||||
|
||||
5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
|
||||
|
||||
6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的
|
||||
|
||||
7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里
|
||||
7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里
|
||||
|
||||
8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
|
||||
|
||||
9. 如果仍不能排除连接故障
|
||||
|
||||
|
||||
* Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
|
||||
检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
|
||||
检查服务器侧TCP端口连接是否工作：`nc -l {port}`
|
||||
检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}`
|
||||
|
||||
|
||||
* Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
|
||||
|
||||
10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。
|
||||
|
||||
|
||||
10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：<a href="https://www.taosdata.com/blog/2020/09/08/1816.html">TDengine 内嵌网络检测工具使用指南</a>。
|
||||
|
||||
## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？
|
||||
产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查：
|
||||
|
||||
1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。
|
||||
1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">一篇文章说清楚TDengine的FQDN</a>。
|
||||
2. 如果网络配置有DNS server, 请检查是否正常工作
|
||||
3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。
|
||||
4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法连接服务器的
|
||||
|
||||
|
||||
## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误
|
||||
|
||||
如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。
|
||||
|
|
@ -86,7 +83,7 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统
|
|||
|
||||
## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？
|
||||
|
||||
windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下：
|
||||
Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下：
|
||||
```JAVA
|
||||
Class.forName("com.taosdata.jdbc.TSDBDriver");
|
||||
Properties properties = new Properties();
|
||||
|
|
@ -94,7 +91,7 @@ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8");
|
|||
Connection = DriverManager.getConnection(url, properties);
|
||||
```
|
||||
## 12.TDengine GO windows驱动的如何编译？
|
||||
请看为此问题撰写的<a href='blog/2020/01/06/tdengine-go-windows驱动的编译/'>技术博客
|
||||
请看为此问题撰写的<a href='blog/2020/01/06/tdengine-go-windows驱动的编译/'>技术博客</a>
|
||||
|
||||
## 13.JDBC报错： the excuted SQL is not a DML or a DDL？
|
||||
请更新至最新的JDBC驱动
|
||||
|
|
@ -105,18 +102,14 @@ Connection = DriverManager.getConnection(url, properties);
|
|||
<version>2.0.4</version>
|
||||
</dependency>
|
||||
```
|
||||
## 14. taos connect failed, reason: invalid timestamp
|
||||
## 14. taos connect failed, reason: invalid timestamp
|
||||
|
||||
常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。
|
||||
|
||||
|
||||
|
||||
## 15. 表名显示不全
|
||||
|
||||
由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
|
||||
|
||||
|
||||
|
||||
## 16. 如何进行数据迁移？
|
||||
|
||||
TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A移动机器B时，注意如下两件事：
|
||||
|
|
@ -125,13 +118,11 @@ TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A
|
|||
- 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。
|
||||
- 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。
|
||||
|
||||
|
||||
|
||||
## 17. 怎么报告问题？
|
||||
|
||||
如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包:
|
||||
如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包：
|
||||
1. /var/log/taos
|
||||
2. /etc/taos
|
||||
2. /etc/taos
|
||||
|
||||
附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在<a href='https://github.com/taosdata/TDengine'> GitHub</a>提交Issue。
|
||||
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6,
|
|||
- 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。
|
||||
|
||||
## Prometheus直接写入
|
||||
[Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
|
||||
<a href="https://www.prometheus.io/">Prometheus</a>作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具<a href="https://github.com/taosdata/Bailongma">Bailongma</a>，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文<a href="https://www.taosdata.com/blog/2020/02/03/1189.html">用Docker容器快速搭建一个Devops监控Demo</a>即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
|
||||
|
||||
### 从源代码编译blm_prometheus
|
||||
用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
|
||||
用户需要从github下载<a href="https://github.com/taosdata/Bailongma">Bailongma</a>的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
|
||||
- Linux操作系统的服务器
|
||||
- 安装好Golang, 1.10版本以上
|
||||
- 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器）
|
||||
|
|
@ -45,10 +45,10 @@ go build
|
|||
一切正常的情况下，就会在对应的目录下生成一个blm_prometheus的可执行程序。
|
||||
|
||||
### 安装Prometheus
|
||||
通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/)
|
||||
通过Prometheus的官网下载安装。<a href="https://prometheus.io/download/">下载地址</a>
|
||||
|
||||
### 配置Prometheus
|
||||
参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的<remote_write>部分，增加以下配置
|
||||
参考Prometheus的<a href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/">配置文档</a>，在Prometheus的配置文件中的<remote_write>部分，增加以下配置
|
||||
|
||||
- url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节
|
||||
|
||||
|
|
@ -60,7 +60,7 @@ blm_prometheus程序有以下选项，在启动blm_prometheus程序时可以通
|
|||
--tdengine-name
|
||||
如果TDengine安装在一台具备域名的服务器上，也可以通过配置TDengine的域名来访问TDengine。在K8S环境下，可以配置成TDengine所运行的service name
|
||||
|
||||
--batch-size
|
||||
--batch-size
|
||||
blm_prometheus会将收到的prometheus的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。
|
||||
|
||||
--dbname
|
||||
|
|
@ -113,10 +113,10 @@ select * from apiserver_request_latencies_bucket;
|
|||
```
|
||||
|
||||
## Telegraf直接写入
|
||||
[Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具，TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
|
||||
<a href="https://www.influxdata.com/time-series-platform/telegraf/"Telegraf</a>是一流行的IT运维数据采集开源工具，TDengine提供一个小工具<a href="https://github.com/taosdata/Bailongma">Bailongma</a>，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文<a href="https://www.taosdata.com/blog/2020/02/03/1189.html">用Docker容器快速搭建一个Devops监控Demo</a>即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
|
||||
|
||||
### 从源代码编译blm_telegraf
|
||||
用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
|
||||
用户需要从github下载<a href="https://github.com/taosdata/Bailongma">Bailongma</a>的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
|
||||
|
||||
- Linux操作系统的服务器
|
||||
- 安装好Golang, 1.10版本以上
|
||||
|
|
@ -137,7 +137,7 @@ go build
|
|||
### 配置Telegraf
|
||||
修改Telegraf配置文件/etc/telegraf/telegraf.conf中与TDengine有关的配置项。
|
||||
|
||||
在output plugins部分，增加[[outputs.http]]配置项：
|
||||
在output plugins部分，增加[[outputs.http]]配置项：
|
||||
|
||||
- url： bailongma API服务提供的URL, 参考下面的启动示例章节
|
||||
- data_format: "json"
|
||||
|
|
@ -148,16 +148,16 @@ go build
|
|||
- hostname: 区分不同采集设备的机器名称，需确保其唯一性
|
||||
- metric_batch_size: 100，允许Telegraf每批次写入记录最大数量，增大其数量可以降低Telegraf的请求发送频率。
|
||||
|
||||
关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的[文档](https://docs.influxdata.com/telegraf/v1.11/)。
|
||||
关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的<a href="https://docs.influxdata.com/telegraf/v1.11/">文档</a>。
|
||||
|
||||
### 启动blm_telegraf程序
|
||||
blm_telegraf程序有以下选项，在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。
|
||||
|
||||
```sh
|
||||
--host
|
||||
--host
|
||||
TDengine服务端的IP地址，缺省值为空
|
||||
|
||||
--batch-size
|
||||
--batch-size
|
||||
blm_telegraf会将收到的telegraf的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。
|
||||
|
||||
--dbname
|
||||
|
|
@ -218,15 +218,12 @@ use telegraf;
|
|||
select * from cpu;
|
||||
```
|
||||
|
||||
|
||||
|
||||
MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
|
||||
|
||||
## EMQ Broker 直接写入
|
||||
|
||||
[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
|
||||
<a href="https://github.com/emqx/emqx">EMQ</a>是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考<a href="https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine">EMQ 官方文档</a>。
|
||||
|
||||
## HiveMQ Broker 直接写入
|
||||
|
||||
[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。
|
||||
|
||||
<a href="https://www.hivemq.com/">HiveMQ</a> 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 <a href="https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md">HiveMQ extension - TDengine 说明文档</a>。
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储
|
|||
|
||||
- master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master.
|
||||
- slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。
|
||||
- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务
|
||||
- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务。
|
||||
- offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。
|
||||
|
||||
**Quorum:**
|
||||
|
|
@ -83,10 +83,10 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性
|
|||
如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的连接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的连接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下：
|
||||
|
||||
1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced.
|
||||
2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程
|
||||
2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程。
|
||||
3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。
|
||||
4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。
|
||||
5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version).
|
||||
5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version)。
|
||||
|
||||
具体的流程图如下：
|
||||
|
||||
|
|
@ -124,7 +124,7 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性
|
|||
|
||||
如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。
|
||||
|
||||
1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mnode里的文件，数量是固定的，对应于acct, user, db, table等文件。
|
||||
1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mode里的文件，数量是固定的，对应于acct, user, db, table等文件。
|
||||
2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。
|
||||
3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。
|
||||
4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则
|
||||
|
|
@ -212,10 +212,10 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系
|
|||
|
||||
相同之处：
|
||||
|
||||
- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程
|
||||
- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的
|
||||
- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程。
|
||||
- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的。
|
||||
- 数据转发流程完全一样，Master(leader)需要等待回复确认。
|
||||
- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步
|
||||
- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步。
|
||||
|
||||
不同之处：
|
||||
|
||||
|
|
@ -226,7 +226,7 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系
|
|||
|
||||
## Meta Data的数据复制
|
||||
|
||||
TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析
|
||||
TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析。
|
||||
|
||||
TDengine里Meta Data包括以下：
|
||||
|
||||
|
|
|
|||
|
|
@ -39,10 +39,10 @@
|
|||
# number of management nodes in the system
|
||||
# numOfMnodes 3
|
||||
|
||||
# enable/disable backuping vnode directory when removing dnode
|
||||
# enable/disable backuping vnode directory when removing vnode
|
||||
# vnodeBak 1
|
||||
|
||||
# if report installation / use information
|
||||
# enable/disable installation / usage report
|
||||
# telemetryReporting 1
|
||||
|
||||
# enable/disable load balancing
|
||||
|
|
@ -81,7 +81,7 @@
|
|||
# minimum time window, milli-second
|
||||
# minIntervalTime 10
|
||||
|
||||
# maximum delay before launching a stream compution, milli-second
|
||||
# maximum delay before launching a stream computation, milli-second
|
||||
# maxStreamCompDelay 20000
|
||||
|
||||
# maximum delay before launching a stream computation for the first time, milli-second
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
# max number of connections allowed in dnode
|
||||
# maxShellConns 5000
|
||||
|
||||
# max numerber of connections allowed in client
|
||||
# max number of connections allowed in client
|
||||
# maxConnections 5000
|
||||
|
||||
# stop writing logs when the disk size of the log folder is less than this value
|
||||
|
|
@ -187,7 +187,7 @@
|
|||
# restfulRowLimit 10240
|
||||
|
||||
# The following parameter is used to limit the maximum number of lines in log files.
|
||||
# max number of rows per log filters
|
||||
# max number of lines per log filters
|
||||
# numOfLogLines 10000000
|
||||
|
||||
# enable/disable async log
|
||||
|
|
@ -199,7 +199,9 @@
|
|||
|
||||
# The following parameters are used for debug purpose only.
|
||||
# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
|
||||
# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log.
|
||||
# 131: output warning and error
|
||||
# 135: output debug, warning and error
|
||||
# 143: output trace, debug, warning and error to log
|
||||
# 199: output debug, warning and error to both screen and file
|
||||
# 207: output trace, debug, warning and error to both screen and file
|
||||
|
||||
|
|
@ -231,10 +233,10 @@
|
|||
# cDebugFlag 131
|
||||
|
||||
# debug flag for JNI
|
||||
# jniDebugflag 131
|
||||
# jniDebugFlag 131
|
||||
|
||||
# debug flag for storage
|
||||
# uDebugflag 131
|
||||
# uDebugFlag 131
|
||||
|
||||
# debug flag for http server
|
||||
# httpDebugFlag 131
|
||||
|
|
@ -243,12 +245,12 @@
|
|||
# monDebugFlag 131
|
||||
|
||||
# debug flag for query
|
||||
# qDebugflag 131
|
||||
# qDebugFlag 131
|
||||
|
||||
# debug flag for vnode
|
||||
# vDebugflag 131
|
||||
# vDebugFlag 131
|
||||
|
||||
# debug flag for http server
|
||||
# debug flag for TSDB
|
||||
# tsdbDebugFlag 131
|
||||
|
||||
# debug flag for continue query
|
||||
|
|
|
|||
|
|
@ -265,8 +265,14 @@ function install_config() {
|
|||
[ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir}
|
||||
${csudo} chmod 644 ${cfg_install_dir}/*
|
||||
fi
|
||||
|
||||
# Save standard input to 6 and open / dev / TTY on standard input
|
||||
exec 6<&0 0</dev/tty
|
||||
|
||||
local_fqdn_check
|
||||
|
||||
# restore the backup standard input, and turn off 6
|
||||
exec 0<&6 6<&-
|
||||
|
||||
${csudo} mv ${cfg_dir}/taos.cfg ${cfg_dir}/taos.cfg.org
|
||||
${csudo} ln -s ${cfg_install_dir}/taos.cfg ${cfg_dir}
|
||||
|
|
@ -422,7 +428,7 @@ function install_service() {
|
|||
}
|
||||
|
||||
function install_TDengine() {
|
||||
echo -e "${GREEN}Start to install TDEngine...${NC}"
|
||||
echo -e "${GREEN}Start to install TDengine...${NC}"
|
||||
|
||||
#install log and data dir , then ln to /usr/local/taos
|
||||
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
|
||||
|
|
|
|||
|
|
@ -119,4 +119,4 @@ if ((${service_mod}==2)); then
|
|||
kill_taosd
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}TDEngine is removed successfully!${NC}"
|
||||
echo -e "${GREEN}TDengine is removed successfully!${NC}"
|
||||
|
|
|
|||
|
|
@ -2,19 +2,39 @@
|
|||
#
|
||||
# This file is used to set config for core when taosd crash
|
||||
|
||||
set -e
|
||||
# Color setting
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[1;32m'
|
||||
GREEN_DARK='\033[0;32m'
|
||||
GREEN_UNDERLINE='\033[4;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
# set -e
|
||||
# set -x
|
||||
corePath=$1
|
||||
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo"
|
||||
fi
|
||||
|
||||
#ulimit -c unlimited
|
||||
if [[ ! -n ${corePath} ]]; then
|
||||
echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:"
|
||||
read corePath
|
||||
while true; do
|
||||
if [[ ! -z "$corePath" ]]; then
|
||||
break
|
||||
else
|
||||
read -p "Please enter a file directory to save the coredump file:" corePath
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
ulimit -c unlimited
|
||||
${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||:
|
||||
${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||:
|
||||
source /etc/profile
|
||||
|
||||
${csudo} mkdir -p /coredump ||:
|
||||
${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||:
|
||||
${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||:
|
||||
${csudo} mkdir -p ${corePath} ||:
|
||||
${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||:
|
||||
${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
name: tdengine
|
||||
base: core18
|
||||
version: '2.0.11.0'
|
||||
version: '2.0.12.0'
|
||||
icon: snap/gui/t-dengine.svg
|
||||
summary: an open-source big data platform designed and optimized for IoT.
|
||||
description: |
|
||||
|
|
@ -72,7 +72,7 @@ parts:
|
|||
- usr/bin/taosd
|
||||
- usr/bin/taos
|
||||
- usr/bin/taosdemo
|
||||
- usr/lib/libtaos.so.2.0.11.0
|
||||
- usr/lib/libtaos.so.2.0.12.0
|
||||
- usr/lib/libtaos.so.1
|
||||
- usr/lib/libtaos.so
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ static void bnUnLock() {
|
|||
|
||||
static bool bnCheckFree(SDnodeObj *pDnode) {
|
||||
if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
|
||||
mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
|
||||
mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -92,13 +92,12 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
|
|||
}
|
||||
|
||||
static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
|
||||
// SVnodeGid tmp = *pVnodeGid1;
|
||||
// *pVnodeGid1 = *pVnodeGid2;
|
||||
// *pVnodeGid2 = tmp;
|
||||
SVnodeGid tmp = *pVnodeGid1;
|
||||
*pVnodeGid1 = *pVnodeGid2;
|
||||
*pVnodeGid2 = tmp;
|
||||
}
|
||||
|
||||
int32_t bnAllocVnodes(SVgObj *pVgroup) {
|
||||
static int32_t randIndex = 0;
|
||||
int32_t dnode = 0;
|
||||
int32_t vnodes = 0;
|
||||
|
||||
|
|
@ -120,8 +119,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
|
|||
break;
|
||||
} else {
|
||||
mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
|
||||
mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
|
||||
pDnode->alternativeRole);
|
||||
dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -137,7 +135,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
|
|||
while (1) {
|
||||
pIter = mnodeGetNextDnode(pIter, &pDnode);
|
||||
if (pDnode == NULL) break;
|
||||
mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
|
||||
mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status],
|
||||
pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
|
||||
mnodeDecDnodeRef(pDnode);
|
||||
}
|
||||
|
|
@ -149,36 +147,6 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* make the choice more random.
|
||||
* replica 1: no choice
|
||||
* replica 2: there are 2 combinations
|
||||
* replica 3 or larger: there are 6 combinations
|
||||
*/
|
||||
if (pVgroup->numOfVnodes == 1) {
|
||||
} else if (pVgroup->numOfVnodes == 2) {
|
||||
if (randIndex++ % 2 == 0) {
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
|
||||
}
|
||||
} else {
|
||||
int32_t randVal = randIndex++ % 6;
|
||||
if (randVal == 1) { // 1, 0, 2
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
|
||||
} else if (randVal == 2) { // 1, 2, 0
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
|
||||
} else if (randVal == 3) { // 2, 1, 0
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
|
||||
} else if (randVal == 4) { // 2, 0, 1
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
|
||||
}
|
||||
if (randVal == 5) { // 0, 2, 1
|
||||
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
|
||||
} else {
|
||||
} // 0, 1, 2
|
||||
}
|
||||
|
||||
bnReleaseDnodes();
|
||||
bnUnLock();
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -214,44 +182,8 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
|
|||
static int32_t bnRemoveVnode(SVgObj *pVgroup) {
|
||||
if (pVgroup->numOfVnodes <= 1) return -1;
|
||||
|
||||
SVnodeGid *pRmVnode = NULL;
|
||||
SVnodeGid *pSelVnode = NULL;
|
||||
int32_t maxScore = 0;
|
||||
|
||||
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
|
||||
SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]);
|
||||
SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId);
|
||||
|
||||
if (pDnode == NULL) {
|
||||
mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId);
|
||||
pRmVnode = pVnode;
|
||||
break;
|
||||
}
|
||||
|
||||
if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
|
||||
mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId);
|
||||
pRmVnode = pVnode;
|
||||
} else if (pVnode->dnodeId == pVgroup->lbDnodeId) {
|
||||
mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId);
|
||||
pRmVnode = pVnode;
|
||||
} else {
|
||||
if (pSelVnode == NULL) {
|
||||
pSelVnode = pVnode;
|
||||
maxScore = pDnode->score;
|
||||
} else {
|
||||
if (maxScore < pDnode->score) {
|
||||
pSelVnode = pVnode;
|
||||
maxScore = pDnode->score;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mnodeDecDnodeRef(pDnode);
|
||||
}
|
||||
|
||||
if (pRmVnode != NULL) {
|
||||
pSelVnode = pRmVnode;
|
||||
}
|
||||
SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1];
|
||||
mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId);
|
||||
|
||||
if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
|
||||
mDebug("vgId:%d, is not ready", pVgroup->vgId);
|
||||
|
|
@ -275,36 +207,42 @@ static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* desc: add vnode to vgroup, find a new one if dest dnode is null
|
||||
**/
|
||||
static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) {
|
||||
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
|
||||
SDnodeObj *pDnode = tsBnDnodes.list[i];
|
||||
if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
|
||||
if (!bnCheckFree(pDnode)) continue;
|
||||
|
||||
mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
|
||||
return pDnode;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
|
||||
if (pDestDnode == NULL) {
|
||||
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
|
||||
SDnodeObj *pDnode = tsBnDnodes.list[i];
|
||||
if (pDnode == pSrcDnode) continue;
|
||||
if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
|
||||
if (!bnCheckFree(pDnode)) continue;
|
||||
|
||||
pDestDnode = pDnode;
|
||||
mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
|
||||
if (pDestDnode == NULL || pSrcDnode == pDestDnode) {
|
||||
return TSDB_CODE_MND_DNODE_NOT_EXIST;
|
||||
}
|
||||
|
||||
SVnodeGid vnodeGids[TSDB_MAX_REPLICA];
|
||||
memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
|
||||
|
||||
int32_t numOfVnodes = pVgroup->numOfVnodes;
|
||||
vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId;
|
||||
vnodeGids[numOfVnodes].pDnode = pDestDnode;
|
||||
numOfVnodes++;
|
||||
|
||||
for (int32_t v = 0; v < numOfVnodes; ++v) {
|
||||
if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) {
|
||||
bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]);
|
||||
pVgroup->lbDnodeId = pSrcDnode->dnodeId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (pDestDnode == NULL) {
|
||||
return TSDB_CODE_MND_DNODE_NOT_EXIST;
|
||||
}
|
||||
|
||||
SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes;
|
||||
pVnodeGid->dnodeId = pDestDnode->dnodeId;
|
||||
pVnodeGid->pDnode = pDestDnode;
|
||||
pVgroup->numOfVnodes++;
|
||||
|
||||
if (pSrcDnode != NULL) {
|
||||
pVgroup->lbDnodeId = pSrcDnode->dnodeId;
|
||||
}
|
||||
|
||||
memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
|
||||
pVgroup->numOfVnodes = numOfVnodes;
|
||||
atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
|
||||
|
||||
mnodeUpdateVgroup(pVgroup);
|
||||
|
|
@ -315,16 +253,16 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes
|
|||
static bool bnMonitorBalance() {
|
||||
if (tsBnDnodes.size < 2) return false;
|
||||
|
||||
mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size);
|
||||
for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
|
||||
SDnodeObj *pDnode = tsBnDnodes.list[src];
|
||||
mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
|
||||
pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
|
||||
pDnode->openVnodes);
|
||||
mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId,
|
||||
dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes);
|
||||
}
|
||||
|
||||
float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
|
||||
if (scoresDiff < 0.01) {
|
||||
mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
|
||||
mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -392,7 +330,7 @@ void bnReset() {
|
|||
tsAccessSquence = 0;
|
||||
}
|
||||
|
||||
static int32_t bnMonitorVgroups() {
|
||||
static bool bnMonitorVgroups() {
|
||||
void * pIter = NULL;
|
||||
SVgObj *pVgroup = NULL;
|
||||
bool hasUpdatingVgroup = false;
|
||||
|
|
@ -412,7 +350,13 @@ static int32_t bnMonitorVgroups() {
|
|||
} else if (vgReplica < dbReplica) {
|
||||
mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
|
||||
hasUpdatingVgroup = true;
|
||||
code = bnAddVnode(pVgroup, NULL, NULL);
|
||||
|
||||
SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup);
|
||||
if (pAvailDnode == NULL) {
|
||||
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
|
||||
} else {
|
||||
code = bnAddVnode(pVgroup, NULL, pAvailDnode);
|
||||
}
|
||||
}
|
||||
|
||||
mnodeDecVgroupRef(pVgroup);
|
||||
|
|
@ -545,6 +489,7 @@ void bnCheckStatus() {
|
|||
mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
|
||||
pDnode->lastAccess, pDnode->status);
|
||||
bnSetVgroupOffline(pDnode);
|
||||
bnStartTimer(3000);
|
||||
}
|
||||
}
|
||||
mnodeDecDnodeRef(pDnode);
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void
|
|||
cols++;
|
||||
|
||||
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
|
||||
STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
|
||||
STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]);
|
||||
cols++;
|
||||
|
||||
numOfRows++;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) {
|
|||
}
|
||||
|
||||
pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
|
||||
mDebug("balance thread wakes up to work");
|
||||
bool updateSoon = bnStart();
|
||||
mDebug("balance thread finished this poll, updateSoon:%d", updateSoon);
|
||||
|
||||
bnStartTimer(updateSoon ? 1000 : -1);
|
||||
pthread_mutex_unlock(&(tsBnThread.mutex));
|
||||
}
|
||||
|
|
@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) {
|
|||
tsBnThread.timer = NULL;
|
||||
tsAccessSquence++;
|
||||
|
||||
bnCheckStatus();
|
||||
bnStartTimer(-1);
|
||||
bnCheckStatus();
|
||||
|
||||
if (handle == NULL) {
|
||||
if (tsAccessSquence % tsBalanceInterval == 0) {
|
||||
|
|
@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) {
|
|||
|
||||
bool updateSoon = (mseconds != -1);
|
||||
if (updateSoon) {
|
||||
mTrace("balance function will be called after %" PRId64 " ms", mseconds);
|
||||
taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
|
||||
} else {
|
||||
taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
|
||||
|
|
|
|||
|
|
@ -307,7 +307,6 @@ typedef struct STscObj {
|
|||
SRpcCorEpSet *tscCorMgmtEpSet;
|
||||
void* pDnodeConn;
|
||||
pthread_mutex_t mutex;
|
||||
T_REF_DECLARE()
|
||||
} STscObj;
|
||||
|
||||
typedef struct SSubqueryState {
|
||||
|
|
@ -483,7 +482,6 @@ extern int tscObjRef;
|
|||
extern void * tscTmr;
|
||||
extern void * tscQhandle;
|
||||
extern int tscKeepConn[];
|
||||
extern int tsInsertHeadSize;
|
||||
extern int tscNumOfThreads;
|
||||
extern int tscRefId;
|
||||
|
||||
|
|
|
|||
|
|
@ -388,10 +388,10 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
|
|||
return;
|
||||
}
|
||||
|
||||
assert(pSql->res.code != TSDB_CODE_SUCCESS);
|
||||
tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
|
||||
|
||||
SSqlRes *pRes = &pSql->res;
|
||||
|
||||
if (pSql->fp == NULL || pSql->fetchFp == NULL){
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2597,14 +2597,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
|
|||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
static void buildHistogramInfo(SAPercentileInfo* pInfo) {
|
||||
pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo));
|
||||
pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo));
|
||||
}
|
||||
|
||||
static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
|
||||
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
|
||||
|
||||
SAPercentileInfo* pInfo = NULL;
|
||||
|
||||
if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
|
||||
return (SAPercentileInfo*) pCtx->aOutputBuf;
|
||||
pInfo = (SAPercentileInfo*) pCtx->aOutputBuf;
|
||||
} else {
|
||||
return GET_ROWCELL_INTERBUF(pResInfo);
|
||||
pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
}
|
||||
|
||||
buildHistogramInfo(pInfo);
|
||||
return pInfo;
|
||||
}
|
||||
|
||||
static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
|
||||
|
|
@ -2624,6 +2633,8 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
|
|||
|
||||
SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
|
||||
SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
|
||||
|
||||
assert(pInfo->pHisto->elems != NULL);
|
||||
|
||||
for (int32_t i = 0; i < pCtx->size; ++i) {
|
||||
char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
|
||||
|
|
|
|||
|
|
@ -911,6 +911,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
|
|||
}
|
||||
}
|
||||
|
||||
if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) {
|
||||
pRes->numOfRows = 0;
|
||||
pBeforeFillData->num = 0;
|
||||
pLocalReducer->discard = true;
|
||||
return;
|
||||
}
|
||||
|
||||
pRes->numOfRowsGroup += pRes->numOfRows;
|
||||
|
||||
// impose the limitation of output rows on the final result
|
||||
|
|
|
|||
|
|
@ -996,33 +996,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
|
|||
return false;
|
||||
}
|
||||
|
||||
int32_t nLen = 0;
|
||||
for (int32_t i = 0; i < numOfTags; ++i) {
|
||||
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
|
||||
if (p->bytes == 0) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
|
||||
return false;
|
||||
}
|
||||
|
||||
nLen += p->bytes;
|
||||
}
|
||||
|
||||
// max tag row length must be less than TSDB_MAX_TAGS_LEN
|
||||
if (nLen > TSDB_MAX_TAGS_LEN) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
|
||||
return false;
|
||||
}
|
||||
|
||||
// field name must be unique
|
||||
for (int32_t i = 0; i < numOfTags; ++i) {
|
||||
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
|
||||
|
||||
if (has(pFieldList, 0, p->name) == true) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/* timestamp in tag is not allowed */
|
||||
for (int32_t i = 0; i < numOfTags; ++i) {
|
||||
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
|
||||
|
|
@ -1054,6 +1027,33 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
|
|||
}
|
||||
}
|
||||
|
||||
int32_t nLen = 0;
|
||||
for (int32_t i = 0; i < numOfTags; ++i) {
|
||||
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
|
||||
if (p->bytes == 0) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
|
||||
return false;
|
||||
}
|
||||
|
||||
nLen += p->bytes;
|
||||
}
|
||||
|
||||
// max tag row length must be less than TSDB_MAX_TAGS_LEN
|
||||
if (nLen > TSDB_MAX_TAGS_LEN) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
|
||||
return false;
|
||||
}
|
||||
|
||||
// field name must be unique
|
||||
for (int32_t i = 0; i < numOfTags; ++i) {
|
||||
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
|
||||
|
||||
if (has(pFieldList, 0, p->name) == true) {
|
||||
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -115,9 +115,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
|
|||
|
||||
pObj->signature = pObj;
|
||||
pObj->pDnodeConn = pDnodeConn;
|
||||
T_REF_INIT_VAL(pObj, 1);
|
||||
|
||||
|
||||
tstrncpy(pObj->user, user, sizeof(pObj->user));
|
||||
secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
|
||||
memcpy(pObj->pass, secretEncrypt, secretEncryptLen);
|
||||
|
|
@ -172,11 +170,9 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
|
|||
if (taos != NULL) {
|
||||
*taos = pObj;
|
||||
}
|
||||
|
||||
registerSqlObj(pSql);
|
||||
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
|
||||
|
||||
pObj->rid = taosAddRef(tscRefId, pObj);
|
||||
registerSqlObj(pSql);
|
||||
|
||||
return pSql;
|
||||
}
|
||||
|
||||
|
|
@ -288,34 +284,21 @@ void taos_close(TAOS *taos) {
|
|||
return;
|
||||
}
|
||||
|
||||
// make sure that the close connection can only be executed once.
|
||||
pObj->signature = NULL;
|
||||
taosTmrStopA(&(pObj->pTimer));
|
||||
|
||||
if (pObj->hbrid > 0) {
|
||||
if (RID_VALID(pObj->hbrid)) {
|
||||
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
|
||||
if (pHb != NULL) {
|
||||
if (pHb->rpcRid > 0) { // wait for rsp from dnode
|
||||
if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode
|
||||
rpcCancelRequest(pHb->rpcRid);
|
||||
pHb->rpcRid = -1;
|
||||
}
|
||||
|
||||
tscDebug("%p HB is freed", pHb);
|
||||
taos_free_result(pHb);
|
||||
taosReleaseRef(tscObjRef, pHb->self);
|
||||
taos_free_result(pHb);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t ref = T_REF_DEC(pObj);
|
||||
assert(ref >= 0);
|
||||
|
||||
if (ref > 0) {
|
||||
tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
|
||||
return;
|
||||
}
|
||||
|
||||
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
|
||||
|
||||
taosRemoveRef(tscRefId, pObj->rid);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ int tscObjRef = -1;
|
|||
void * tscTmr;
|
||||
void * tscQhandle;
|
||||
void * tscCheckDiskUsageTmr;
|
||||
int tsInsertHeadSize;
|
||||
int tscRefId = -1;
|
||||
|
||||
int tscNumOfThreads;
|
||||
|
|
|
|||
|
|
@ -460,15 +460,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
|
|||
|
||||
assert(p->self != 0);
|
||||
tscFreeSqlObj(p);
|
||||
|
||||
int32_t ref = T_REF_DEC(pTscObj);
|
||||
assert(ref >= 0);
|
||||
|
||||
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref);
|
||||
if (ref == 0) {
|
||||
tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj);
|
||||
taosRemoveRef(tscRefId, pTscObj->rid);
|
||||
}
|
||||
taosReleaseRef(tscRefId, pTscObj->rid);
|
||||
}
|
||||
|
||||
void tscFreeTableMetaHelper(void *pTableMeta) {
|
||||
|
|
@ -810,6 +802,7 @@ static void extractTableMeta(SSqlCmd* pCmd) {
|
|||
}
|
||||
|
||||
int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
|
||||
const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
|
||||
SSqlCmd* pCmd = &pSql->cmd;
|
||||
|
||||
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
|
||||
|
|
@ -824,7 +817,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
|
|||
STableDataBlocks* dataBuf = NULL;
|
||||
|
||||
int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
|
||||
tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
|
||||
INSERT_HEAD_SIZE, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret);
|
||||
taosHashCleanup(pVnodeDataBlockHashList);
|
||||
|
|
@ -1917,9 +1910,7 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
|
|||
}
|
||||
|
||||
void registerSqlObj(SSqlObj* pSql) {
|
||||
int32_t ref = T_REF_INC(pSql->pTscObj);
|
||||
tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref);
|
||||
|
||||
taosAcquireRef(tscRefId, pSql->pTscObj->rid);
|
||||
pSql->self = taosAddRef(tscObjRef, pSql);
|
||||
}
|
||||
|
||||
|
|
@ -2629,4 +2620,4 @@ int32_t copyTagData(STagData* dst, const STagData* src) {
|
|||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,7 +101,8 @@ extern int32_t tsAlternativeRole;
|
|||
extern int32_t tsBalanceInterval;
|
||||
extern int32_t tsOfflineThreshold;
|
||||
extern int32_t tsMnodeEqualVnodeNum;
|
||||
extern int32_t tsFlowCtrl;
|
||||
extern int32_t tsEnableFlowCtrl;
|
||||
extern int32_t tsEnableSlaveQuery;
|
||||
|
||||
// restful
|
||||
extern int32_t tsEnableHttpModule;
|
||||
|
|
|
|||
|
|
@ -138,7 +138,8 @@ int32_t tsAlternativeRole = 0;
|
|||
int32_t tsBalanceInterval = 300; // seconds
|
||||
int32_t tsOfflineThreshold = 86400*100; // seconds 10days
|
||||
int32_t tsMnodeEqualVnodeNum = 4;
|
||||
int32_t tsFlowCtrl = 1;
|
||||
int32_t tsEnableFlowCtrl = 1;
|
||||
int32_t tsEnableSlaveQuery = 1;
|
||||
|
||||
// restful
|
||||
int32_t tsEnableHttpModule = 1;
|
||||
|
|
@ -198,8 +199,8 @@ float tsTotalTmpDirGB = 0;
|
|||
float tsTotalDataDirGB = 0;
|
||||
float tsAvailTmpDirectorySpace = 0;
|
||||
float tsAvailDataDirGB = 0;
|
||||
float tsReservedTmpDirectorySpace = 0.1f;
|
||||
float tsMinimalDataDirGB = 0.5f;
|
||||
float tsReservedTmpDirectorySpace = 1.0f;
|
||||
float tsMinimalDataDirGB = 1.0f;
|
||||
int32_t tsTotalMemoryMB = 0;
|
||||
int32_t tsVersion = 0;
|
||||
|
||||
|
|
@ -221,7 +222,7 @@ int32_t uDebugFlag = 131;
|
|||
int32_t debugFlag = 0;
|
||||
int32_t sDebugFlag = 135;
|
||||
int32_t wDebugFlag = 135;
|
||||
int32_t tsdbDebugFlag = 131;
|
||||
uint32_t tsdbDebugFlag = 131;
|
||||
int32_t cqDebugFlag = 131;
|
||||
|
||||
int32_t (*monStartSystemFp)() = NULL;
|
||||
|
|
@ -542,7 +543,7 @@ static void doInitGlobalConfig(void) {
|
|||
cfg.ptr = &tsOfflineThreshold;
|
||||
cfg.valType = TAOS_CFG_VTYPE_INT32;
|
||||
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
|
||||
cfg.minValue = 5;
|
||||
cfg.minValue = 3;
|
||||
cfg.maxValue = 7200000;
|
||||
cfg.ptrLength = 0;
|
||||
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
|
||||
|
|
@ -1004,7 +1005,17 @@ static void doInitGlobalConfig(void) {
|
|||
|
||||
// module configs
|
||||
cfg.option = "flowctrl";
|
||||
cfg.ptr = &tsFlowCtrl;
|
||||
cfg.ptr = &tsEnableFlowCtrl;
|
||||
cfg.valType = TAOS_CFG_VTYPE_INT32;
|
||||
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
|
||||
cfg.minValue = 0;
|
||||
cfg.maxValue = 1;
|
||||
cfg.ptrLength = 0;
|
||||
cfg.unitType = TAOS_CFG_UTYPE_NONE;
|
||||
taosInitConfigOption(cfg);
|
||||
|
||||
cfg.option = "slaveQuery";
|
||||
cfg.ptr = &tsEnableSlaveQuery;
|
||||
cfg.valType = TAOS_CFG_VTYPE_INT32;
|
||||
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
|
||||
cfg.minValue = 0;
|
||||
|
|
@ -1443,6 +1454,12 @@ int32_t taosCheckGlobalCfg() {
|
|||
tsNumOfCores = 1;
|
||||
}
|
||||
|
||||
if (tsMaxTablePerVnode < tsMinTablePerVnode) {
|
||||
uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)",
|
||||
tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode);
|
||||
tsMaxTablePerVnode = tsMinTablePerVnode;
|
||||
}
|
||||
|
||||
// todo refactor
|
||||
tsVersion = 0;
|
||||
for (int i = 0; i < 10; i++) {
|
||||
|
|
|
|||
|
|
@ -349,11 +349,12 @@ CTaosInterface.prototype.useResult = function useResult(result) {
|
|||
return fields;
|
||||
}
|
||||
CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
|
||||
let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
|
||||
let num_of_rows = this.libtaos.taos_fetch_block(result, pblock)
|
||||
if (num_of_rows == 0) {
|
||||
//let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
|
||||
let pblock = this.libtaos.taos_fetch_row(result);
|
||||
if (pblock == null) {
|
||||
return {block:null, num_of_rows:0};
|
||||
}
|
||||
|
||||
var fieldL = this.libtaos.taos_fetch_lengths(result);
|
||||
|
||||
let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
|
||||
|
|
@ -361,7 +362,6 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
|
|||
var fieldlens = [];
|
||||
|
||||
if (ref.isNull(fieldL) == false) {
|
||||
|
||||
for (let i = 0; i < fields.length; i ++) {
|
||||
let plen = ref.reinterpret(fieldL, 4, i*4);
|
||||
let len = plen.readInt32LE(0);
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ static void dnodeCleanupTmr() {
|
|||
int32_t dnodeInitSystem() {
|
||||
dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
|
||||
tscEmbedded = 1;
|
||||
taosIgnSIGPIPE();
|
||||
taosBlockSIGPIPE();
|
||||
taosResolveCRC();
|
||||
taosInitGlobalCfg();
|
||||
|
|
@ -120,7 +121,6 @@ int32_t dnodeInitSystem() {
|
|||
taosSetCoreDump();
|
||||
taosInitNotes();
|
||||
dnodeInitTmr();
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
|
||||
if (dnodeCreateDir(tsLogDir) < 0) {
|
||||
printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ static void addRuntimeInfo(SBufferWriter* bw) {
|
|||
|
||||
static void sendTelemetryReport() {
|
||||
char buf[128];
|
||||
uint32_t ip = taosGetIpFromFqdn(TELEMETRY_SERVER);
|
||||
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
|
||||
if (ip == 0xffffffff) {
|
||||
dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno));
|
||||
return;
|
||||
|
|
@ -308,4 +308,4 @@ void dnodeCleanupTelemetry() {
|
|||
pthread_join(tsTelemetryThread, NULL);
|
||||
tsem_destroy(&tsExitSem);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,8 @@ static void *dnodeProcessMgmtQueue(void *wparam) {
|
|||
static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
|
||||
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
|
||||
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
|
||||
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
|
||||
pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion);
|
||||
pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion);
|
||||
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
|
||||
pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize);
|
||||
pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks);
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ void dnodeCleanupVRead() {
|
|||
void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
|
||||
int32_t queuedMsgNum = 0;
|
||||
int32_t leftLen = pMsg->contLen;
|
||||
int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID;
|
||||
char * pCont = pMsg->pCont;
|
||||
|
||||
while (leftLen > 0) {
|
||||
|
|
@ -64,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
|
|||
assert(pHead->contLen > 0);
|
||||
void *pVnode = vnodeAcquire(pHead->vgId);
|
||||
if (pVnode != NULL) {
|
||||
int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
|
||||
code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
|
||||
if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
|
||||
vnodeRelease(pVnode);
|
||||
}
|
||||
|
|
@ -74,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
if (queuedMsgNum == 0) {
|
||||
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
|
||||
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code};
|
||||
rpcSendResponse(&rpcRsp);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
|
|||
int32_t numOfMsgs;
|
||||
int32_t qtype;
|
||||
|
||||
taosBlockSIGPIPE();
|
||||
dDebug("dnode vwrite worker:%d is running", pWorker->workerId);
|
||||
|
||||
while (1) {
|
||||
|
|
|
|||
|
|
@ -518,14 +518,15 @@ typedef struct SRetrieveTableRsp {
|
|||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int32_t cfgVersion;
|
||||
int32_t dbCfgVersion;
|
||||
int64_t totalStorage;
|
||||
int64_t compStorage;
|
||||
int64_t pointsWritten;
|
||||
uint8_t status;
|
||||
uint8_t role;
|
||||
uint8_t replica;
|
||||
uint8_t reserved[5];
|
||||
uint8_t reserved;
|
||||
int32_t vgCfgVersion;
|
||||
} SVnodeLoad;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -641,7 +642,7 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
uint32_t vgId;
|
||||
int32_t cfgVersion;
|
||||
int32_t dbCfgVersion;
|
||||
int32_t maxTables;
|
||||
int32_t cacheBlockSize;
|
||||
int32_t totalBlocks;
|
||||
|
|
@ -660,7 +661,8 @@ typedef struct {
|
|||
int8_t wals;
|
||||
int8_t quorum;
|
||||
int8_t update;
|
||||
int8_t reserved[15];
|
||||
int8_t reserved[11];
|
||||
int32_t vgCfgVersion;
|
||||
} SVnodeCfg;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ extern "C" {
|
|||
default: \
|
||||
(_v) = (_finalType)GET_INT32_VAL(_data); \
|
||||
break; \
|
||||
};
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ typedef struct DemoArguments {
|
|||
{0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3},
|
||||
#endif
|
||||
{0, 'd', "database", 0, "Destination database. Default is 'test'.", 3},
|
||||
{0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3},
|
||||
{0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3},
|
||||
{0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3},
|
||||
{0, 's', "sql file", 0, "The select sql file.", 3},
|
||||
{0, 'M', 0, 0, "Use metric flag.", 13},
|
||||
|
|
@ -205,10 +205,10 @@ typedef struct DemoArguments {
|
|||
arguments->tb_prefix = arg;
|
||||
break;
|
||||
case 'M':
|
||||
arguments->use_metric = false;
|
||||
arguments->use_metric = true;
|
||||
break;
|
||||
case 'x':
|
||||
arguments->insert_only = false;
|
||||
arguments->insert_only = true;
|
||||
break;
|
||||
case 'c':
|
||||
if (wordexp(arg, &full_path, 0) != 0) {
|
||||
|
|
@ -406,9 +406,9 @@ typedef struct DemoArguments {
|
|||
} else if (strcmp(argv[i], "-m") == 0) {
|
||||
arguments->tb_prefix = argv[++i];
|
||||
} else if (strcmp(argv[i], "-M") == 0) {
|
||||
arguments->use_metric = false;
|
||||
arguments->use_metric = true;
|
||||
} else if (strcmp(argv[i], "-x") == 0) {
|
||||
arguments->insert_only = false;
|
||||
arguments->insert_only = true;
|
||||
} else if (strcmp(argv[i], "-c") == 0) {
|
||||
strcpy(configDir, argv[++i]);
|
||||
} else if (strcmp(argv[i], "-O") == 0) {
|
||||
|
|
@ -476,6 +476,14 @@ typedef struct {
|
|||
int notFinished;
|
||||
tsem_t lock_sem;
|
||||
int counter;
|
||||
|
||||
// insert delay statitics
|
||||
int64_t cntDelay;
|
||||
int64_t totalDelay;
|
||||
int64_t avgDelay;
|
||||
int64_t maxDelay;
|
||||
int64_t minDelay;
|
||||
|
||||
} info;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -575,7 +583,7 @@ int main(int argc, char *argv[]) {
|
|||
arguments.num_of_DPT = 100000;
|
||||
arguments.num_of_RPR = 1000;
|
||||
arguments.use_metric = true;
|
||||
arguments.insert_only = true;
|
||||
arguments.insert_only = false;
|
||||
// end change
|
||||
|
||||
parse_args(argc, argv, &arguments);
|
||||
|
|
@ -739,6 +747,9 @@ int main(int argc, char *argv[]) {
|
|||
printf("Inserting data......\n");
|
||||
pthread_t *pids = malloc(threads * sizeof(pthread_t));
|
||||
info *infos = malloc(threads * sizeof(info));
|
||||
|
||||
memset(pids, 0, threads * sizeof(pthread_t));
|
||||
memset(infos, 0, threads * sizeof(info));
|
||||
|
||||
int a = ntables / threads;
|
||||
if (a < 1) {
|
||||
|
|
@ -768,6 +779,7 @@ int main(int argc, char *argv[]) {
|
|||
t_info->end_table_id = i < b ? last + a : last + a - 1;
|
||||
last = t_info->end_table_id + 1;
|
||||
t_info->counter = 0;
|
||||
t_info->minDelay = INT16_MAX;
|
||||
|
||||
tsem_init(&(t_info->mutex_sem), 0, 1);
|
||||
t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1;
|
||||
|
|
@ -799,12 +811,29 @@ int main(int argc, char *argv[]) {
|
|||
t, (int64_t)ntables * nrecords_per_table, nrecords_per_request,
|
||||
(int64_t)ntables * nrecords_per_table / t);
|
||||
|
||||
int64_t totalDelay = 0;
|
||||
int64_t maxDelay = 0;
|
||||
int64_t minDelay = INT16_MAX;
|
||||
int64_t cntDelay = 0;
|
||||
double avgDelay = 0;
|
||||
for (int i = 0; i < threads; i++) {
|
||||
info *t_info = infos + i;
|
||||
taos_close(t_info->taos);
|
||||
tsem_destroy(&(t_info->mutex_sem));
|
||||
tsem_destroy(&(t_info->lock_sem));
|
||||
|
||||
totalDelay += t_info->totalDelay;
|
||||
cntDelay += t_info->cntDelay;
|
||||
if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
|
||||
if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
|
||||
}
|
||||
avgDelay = (double)totalDelay / cntDelay;
|
||||
|
||||
fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
|
||||
avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
|
||||
|
||||
printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
|
||||
avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
|
||||
|
||||
free(pids);
|
||||
free(infos);
|
||||
|
|
@ -859,7 +888,7 @@ int main(int argc, char *argv[]) {
|
|||
}
|
||||
|
||||
|
||||
if (!insert_only) {
|
||||
if (false == insert_only) {
|
||||
// query data
|
||||
pthread_t read_id;
|
||||
info *rInfo = malloc(sizeof(info));
|
||||
|
|
@ -998,7 +1027,7 @@ void * createTable(void *sarg)
|
|||
/* Create all the tables; */
|
||||
printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
|
||||
for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) {
|
||||
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols);
|
||||
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols);
|
||||
queryDB(winfo->taos, command);
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static int queryDbExec(TAOS *taos, char *command, int type) {
|
||||
int i;
|
||||
TAOS_RES *res = NULL;
|
||||
int32_t code = -1;
|
||||
|
||||
for (i = 0; i < 5; i++) {
|
||||
if (NULL != res) {
|
||||
taos_free_result(res);
|
||||
res = NULL;
|
||||
}
|
||||
|
||||
res = taos_query(taos, command);
|
||||
code = taos_errno(res);
|
||||
if (0 == code) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res));
|
||||
taos_free_result(res);
|
||||
//taos_close(taos);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (1 == type) {
|
||||
int affectedRows = taos_affected_rows(res);
|
||||
taos_free_result(res);
|
||||
return affectedRows;
|
||||
}
|
||||
|
||||
taos_free_result(res);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void queryDB(TAOS *taos, char *command) {
|
||||
int i;
|
||||
TAOS_RES *pSql = NULL;
|
||||
|
|
@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) {
|
|||
}
|
||||
|
||||
/* puts(buffer); */
|
||||
queryDB(winfo->taos, buffer);
|
||||
int64_t startTs;
|
||||
int64_t endTs;
|
||||
startTs = taosGetTimestampUs();
|
||||
//queryDB(winfo->taos, buffer);
|
||||
int affectedRows = queryDbExec(winfo->taos, buffer, 1);
|
||||
|
||||
if (0 <= affectedRows){
|
||||
endTs = taosGetTimestampUs();
|
||||
int64_t delay = endTs - startTs;
|
||||
if (delay > winfo->maxDelay) winfo->maxDelay = delay;
|
||||
if (delay < winfo->minDelay) winfo->minDelay = delay;
|
||||
winfo->cntDelay++;
|
||||
winfo->totalDelay += delay;
|
||||
//winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay;
|
||||
}
|
||||
|
||||
if (tID == winfo->end_table_id) {
|
||||
i = inserted;
|
||||
|
|
|
|||
|
|
@ -332,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
|
|||
break;
|
||||
case 'N':
|
||||
arguments->data_batch = atoi(arg);
|
||||
if (arguments->data_batch >= INT16_MAX) {
|
||||
arguments->data_batch = INT16_MAX - 1;
|
||||
}
|
||||
break;
|
||||
case 'L':
|
||||
{
|
||||
|
|
|
|||
|
|
@ -144,7 +144,8 @@ typedef struct SVgObj {
|
|||
int8_t status;
|
||||
int8_t reserved0[4];
|
||||
SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
|
||||
int8_t reserved1[12];
|
||||
int32_t vgCfgVersion;
|
||||
int8_t reserved1[8];
|
||||
int8_t updateEnd[4];
|
||||
int32_t refCount;
|
||||
int32_t numOfTables;
|
||||
|
|
@ -181,7 +182,7 @@ typedef struct SDbObj {
|
|||
int8_t reserved0[4];
|
||||
char acct[TSDB_USER_LEN];
|
||||
int64_t createdTime;
|
||||
int32_t cfgVersion;
|
||||
int32_t dbCfgVersion;
|
||||
SDbCfg cfg;
|
||||
int8_t status;
|
||||
int8_t reserved1[11];
|
||||
|
|
|
|||
|
|
@ -55,12 +55,12 @@ typedef enum EDnodeOfflineReason {
|
|||
TAOS_DN_OFF_OTHERS
|
||||
} EDnodeOfflineReason;
|
||||
|
||||
extern char* dnodeStatus[];
|
||||
extern char* dnodeRoles[];
|
||||
|
||||
int32_t mnodeInitDnodes();
|
||||
void mnodeCleanupDnodes();
|
||||
|
||||
char* mnodeGetDnodeStatusStr(int32_t dnodeStatus);
|
||||
void mgmtMonitorDnodeModule();
|
||||
|
||||
int32_t mnodeGetDnodesNum();
|
||||
int32_t mnodeGetOnlinDnodesCpuCoreNum();
|
||||
int32_t mnodeGetOnlineDnodesNum();
|
||||
|
|
|
|||
|
|
@ -1015,7 +1015,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
|
|||
|
||||
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
|
||||
pDb->cfg = newCfg;
|
||||
pDb->cfgVersion++;
|
||||
pDb->dbCfgVersion++;
|
||||
SSdbRow row = {
|
||||
.type = SDB_OPER_GLOBAL,
|
||||
.pTable = tsDbSdb,
|
||||
|
|
|
|||
|
|
@ -63,7 +63,6 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
|
|||
static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
|
||||
static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
|
||||
static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
|
||||
static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole);
|
||||
static void mnodeUpdateDnodeEps();
|
||||
|
||||
static char* offlineReason[] = {
|
||||
|
|
@ -557,7 +556,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
|
|||
for (int32_t j = 0; j < openVnodes; ++j) {
|
||||
SVnodeLoad *pVload = &pStatus->load[j];
|
||||
pVload->vgId = htonl(pVload->vgId);
|
||||
pVload->cfgVersion = htonl(pVload->cfgVersion);
|
||||
pVload->dbCfgVersion = htonl(pVload->dbCfgVersion);
|
||||
pVload->vgCfgVersion = htonl(pVload->vgCfgVersion);
|
||||
|
||||
SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId);
|
||||
if (pVgroup == NULL) {
|
||||
|
|
@ -833,12 +833,12 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
|
|||
cols++;
|
||||
|
||||
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
|
||||
char* status = mnodeGetDnodeStatusStr(pDnode->status);
|
||||
char* status = dnodeStatus[pDnode->status];
|
||||
STR_TO_VARSTR(pWrite, status);
|
||||
cols++;
|
||||
|
||||
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
|
||||
char* role = mnodeGetDnodeAlternativeRoleStr(pDnode->alternativeRole);
|
||||
char* role = dnodeRoles[pDnode->alternativeRole];
|
||||
STR_TO_VARSTR(pWrite, role);
|
||||
cols++;
|
||||
|
||||
|
|
@ -1154,21 +1154,17 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
|
|||
return numOfRows;
|
||||
}
|
||||
|
||||
char* mnodeGetDnodeStatusStr(int32_t dnodeStatus) {
|
||||
switch (dnodeStatus) {
|
||||
case TAOS_DN_STATUS_OFFLINE: return "offline";
|
||||
case TAOS_DN_STATUS_DROPPING: return "dropping";
|
||||
case TAOS_DN_STATUS_BALANCING: return "balancing";
|
||||
case TAOS_DN_STATUS_READY: return "ready";
|
||||
default: return "undefined";
|
||||
}
|
||||
}
|
||||
char* dnodeStatus[] = {
|
||||
"offline",
|
||||
"dropping",
|
||||
"balancing",
|
||||
"ready",
|
||||
"undefined"
|
||||
};
|
||||
|
||||
static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
|
||||
switch (alternativeRole) {
|
||||
case TAOS_DN_ALTERNATIVE_ROLE_ANY: return "any";
|
||||
case TAOS_DN_ALTERNATIVE_ROLE_MNODE: return "mnode";
|
||||
case TAOS_DN_ALTERNATIVE_ROLE_VNODE: return "vnode";
|
||||
default:return "any";
|
||||
}
|
||||
}
|
||||
char* dnodeRoles[] = {
|
||||
"any",
|
||||
"mnode",
|
||||
"vnode",
|
||||
"any"
|
||||
};
|
||||
|
|
|
|||
|
|
@ -377,6 +377,24 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static bool mnodeAllOnline() {
|
||||
void *pIter = NULL;
|
||||
bool allOnline = true;
|
||||
|
||||
while (1) {
|
||||
SMnodeObj *pMnode = NULL;
|
||||
pIter = mnodeGetNextMnode(pIter, &pMnode);
|
||||
if (pMnode == NULL) break;
|
||||
if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) {
|
||||
allOnline = false;
|
||||
mnodeDecMnodeRef(pMnode);
|
||||
}
|
||||
}
|
||||
mnodeCancelGetNextMnode(pIter);
|
||||
|
||||
return allOnline;
|
||||
}
|
||||
|
||||
void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
|
||||
SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
|
||||
pMnode->mnodeId = dnodeId;
|
||||
|
|
@ -389,6 +407,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
|
|||
.fpRsp = mnodeCreateMnodeCb
|
||||
};
|
||||
|
||||
if (needConfirm && !mnodeAllOnline()) {
|
||||
mDebug("wait all mnode online then create new mnode");
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (needConfirm) {
|
||||
code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp);
|
||||
|
|
|
|||
|
|
@ -1081,6 +1081,8 @@ static void *sdbWorkerFp(void *pWorker) {
|
|||
int32_t qtype;
|
||||
void * unUsed;
|
||||
|
||||
taosBlockSIGPIPE();
|
||||
|
||||
while (1) {
|
||||
int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed);
|
||||
if (numOfMsgs == 0) {
|
||||
|
|
|
|||
|
|
@ -256,6 +256,8 @@ SVgObj *mnodeGetVgroup(int32_t vgId) {
|
|||
}
|
||||
|
||||
void mnodeUpdateVgroup(SVgObj *pVgroup) {
|
||||
pVgroup->vgCfgVersion++;
|
||||
|
||||
SSdbRow row = {
|
||||
.type = SDB_OPER_GLOBAL,
|
||||
.pTable = tsVgroupSdb,
|
||||
|
|
@ -339,10 +341,11 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
|
|||
pVgroup->pointsWritten = htobe64(pVload->pointsWritten);
|
||||
}
|
||||
|
||||
if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) {
|
||||
mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mnode cfgVersion:%d replica:%d",
|
||||
pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
|
||||
pVgroup->numOfVnodes);
|
||||
if (pVload->dbCfgVersion != pVgroup->pDb->dbCfgVersion || pVload->replica != pVgroup->numOfVnodes ||
|
||||
pVload->vgCfgVersion != pVgroup->vgCfgVersion) {
|
||||
mError("dnode:%d, vgId:%d, vnode cfgVersion:%d:%d repica:%d not match with mnode cfgVersion:%d:%d replica:%d",
|
||||
pDnode->dnodeId, pVload->vgId, pVload->dbCfgVersion, pVload->vgCfgVersion, pVload->replica,
|
||||
pVgroup->pDb->dbCfgVersion, pVgroup->vgCfgVersion, pVgroup->numOfVnodes);
|
||||
mnodeSendAlterVgroupMsg(pVgroup);
|
||||
}
|
||||
}
|
||||
|
|
@ -656,7 +659,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
|
|||
|
||||
pShow->bytes[cols] = 4;
|
||||
pSchema[cols].type = TSDB_DATA_TYPE_INT;
|
||||
strcpy(pSchema[cols].name, "onlineVnodes");
|
||||
strcpy(pSchema[cols].name, "onlines");
|
||||
pSchema[cols].bytes = htons(pShow->bytes[cols]);
|
||||
cols++;
|
||||
|
||||
|
|
@ -671,13 +674,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
|
|||
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
|
||||
pShow->bytes[cols] = 2;
|
||||
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
|
||||
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
|
||||
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1);
|
||||
pSchema[cols].bytes = htons(pShow->bytes[cols]);
|
||||
cols++;
|
||||
|
||||
pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
|
||||
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
|
||||
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
|
||||
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1);
|
||||
pSchema[cols].bytes = htons(pShow->bytes[cols]);
|
||||
cols++;
|
||||
}
|
||||
|
|
@ -840,7 +843,8 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
|
|||
|
||||
SVnodeCfg *pCfg = &pVnode->cfg;
|
||||
pCfg->vgId = htonl(pVgroup->vgId);
|
||||
pCfg->cfgVersion = htonl(pDb->cfgVersion);
|
||||
pCfg->dbCfgVersion = htonl(pDb->dbCfgVersion);
|
||||
pCfg->vgCfgVersion = htonl(pVgroup->vgCfgVersion);
|
||||
pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
|
||||
pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks);
|
||||
pCfg->maxTables = htonl(maxTables + 1);
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ extern "C" {
|
|||
|
||||
// TAOS_OS_FUNC_SOCKET
|
||||
int32_t taosSetNonblocking(SOCKET sock, int32_t on);
|
||||
void taosIgnSIGPIPE();
|
||||
void taosBlockSIGPIPE();
|
||||
|
||||
// TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
|
||||
|
|
|
|||
|
|
@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void taosIgnSIGPIPE() {
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
}
|
||||
|
||||
void taosBlockSIGPIPE() {
|
||||
sigset_t signal_mask;
|
||||
sigemptyset(&signal_mask);
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void taosIgnSIGPIPE() {}
|
||||
void taosBlockSIGPIPE() {}
|
||||
|
||||
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) {
|
|||
else
|
||||
httpCode = 400;
|
||||
|
||||
if (pContext->parser->httpCode != 0) {
|
||||
if (pContext->parser && pContext->parser->httpCode != 0) {
|
||||
httpCode = pContext->parser->httpCode;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,13 +33,6 @@ struct SColumnFilterElem;
|
|||
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
|
||||
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
|
||||
|
||||
typedef struct SGroupResInfo {
|
||||
int32_t groupId;
|
||||
int32_t numOfDataPages;
|
||||
int32_t pageId;
|
||||
int32_t rowId;
|
||||
} SGroupResInfo;
|
||||
|
||||
typedef struct SResultRowPool {
|
||||
int32_t elemSize;
|
||||
int32_t blockSize;
|
||||
|
|
@ -72,6 +65,12 @@ typedef struct SResultRow {
|
|||
union {STimeWindow win; char* key;}; // start key of current time window
|
||||
} SResultRow;
|
||||
|
||||
typedef struct SGroupResInfo {
|
||||
int32_t rowId;
|
||||
int32_t index;
|
||||
SArray* pRows; // SArray<SResultRow*>
|
||||
} SGroupResInfo;
|
||||
|
||||
/**
|
||||
* If the number of generated results is greater than this value,
|
||||
* query query will be halt and return results to client immediate.
|
||||
|
|
@ -89,7 +88,6 @@ typedef struct SResultRowInfo {
|
|||
int32_t size:24; // number of result set
|
||||
int32_t capacity; // max capacity
|
||||
int32_t curIndex; // current start active index
|
||||
int64_t startTime; // start time of the first time window for sliding query
|
||||
int64_t prevSKey; // previous (not completed) sliding window start key
|
||||
} SResultRowInfo;
|
||||
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto);
|
|||
|
||||
void tHistogramPrint(SHistogramInfo* pHisto);
|
||||
|
||||
int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val);
|
||||
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
|
||||
|
||||
SHeapEntry* tHeapCreate(int32_t numOfEntries);
|
||||
void tHeapSort(SHeapEntry* pEntry, int32_t len);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,6 @@ void* destroyResultRowPool(SResultRowPool* p);
|
|||
int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
|
||||
int32_t getNumOfUsedResultRows(SResultRowPool* p);
|
||||
|
||||
uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv);
|
||||
bool isPointInterpoQuery(SQuery *pQuery);
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -158,8 +158,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
}
|
||||
|
||||
#if defined(USE_ARRAYLIST)
|
||||
int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
|
||||
assert(idx >= 0 && idx <= (*pHisto)->maxEntries);
|
||||
int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
|
||||
assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
|
||||
|
||||
if ((*pHisto)->elems[idx].val == val && idx >= 0) {
|
||||
(*pHisto)->elems[idx].num += 1;
|
||||
|
|
@ -356,7 +356,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) {
|
||||
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
|
||||
int32_t end = len - 1;
|
||||
int32_t start = 0;
|
||||
|
||||
|
|
@ -466,7 +466,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) {
|
|||
*/
|
||||
int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
|
||||
#if defined(USE_ARRAYLIST)
|
||||
int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v);
|
||||
int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
|
||||
if (pHisto->elems[slotIdx].val != v) {
|
||||
slotIdx -= 1;
|
||||
|
||||
|
|
|
|||
|
|
@ -384,7 +384,12 @@ void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType)
|
|||
pField->name[pName->n] = 0;
|
||||
|
||||
pField->type = pType->type;
|
||||
pField->bytes = pType->bytes;
|
||||
if(pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR){
|
||||
pField->bytes = 0;
|
||||
} else {
|
||||
pField->bytes = pType->bytes;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
|
||||
|
|
|
|||
|
|
@ -313,7 +313,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32
|
|||
|
||||
// allocate buf
|
||||
if (availablePage == NULL) {
|
||||
pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
|
||||
pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES + 2); // add extract bytes in case of zipped buffer increased.
|
||||
} else {
|
||||
pi->pData = availablePage;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,8 +96,6 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo
|
|||
|
||||
pResultRowInfo->curIndex = -1;
|
||||
pResultRowInfo->size = 0;
|
||||
|
||||
pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
|
||||
pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +108,7 @@ void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
|
|||
assert(num >= 0 && num <= numOfClosed);
|
||||
|
||||
int16_t type = pResultRowInfo->type;
|
||||
int64_t uid = getResultInfoUId(pRuntimeEnv);
|
||||
int64_t uid = 0;
|
||||
|
||||
char *key = NULL;
|
||||
int16_t bytes = -1;
|
||||
|
|
@ -181,11 +179,12 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
|
|||
assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
|
||||
|
||||
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
|
||||
if (pResultRowInfo->pResult[i]->closed) {
|
||||
SResultRow* pRow = pResultRowInfo->pResult[i];
|
||||
if (pRow->closed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pResultRowInfo->pResult[i]->closed = true;
|
||||
pRow->closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -383,18 +382,4 @@ void* destroyResultRowPool(SResultRowPool* p) {
|
|||
|
||||
tfree(p);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
|
||||
if (!pRuntimeEnv->stableQuery) {
|
||||
return 0; // for simple table query, the uid is always set to be 0;
|
||||
}
|
||||
|
||||
SQuery* pQuery = pRuntimeEnv->pQuery;
|
||||
if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable);
|
||||
return id->uid;
|
||||
}
|
||||
|
|
@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) {
|
|||
pHisto->elems[i].val = i;
|
||||
}
|
||||
|
||||
int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
|
||||
int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
|
||||
assert(idx == 1);
|
||||
|
||||
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
|
||||
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
|
||||
assert(idx == 9);
|
||||
|
||||
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
|
||||
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
|
||||
assert(idx == 10);
|
||||
|
||||
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
|
||||
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
|
||||
assert(idx == 0);
|
||||
|
||||
idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
|
||||
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
|
||||
assert(idx == 4);
|
||||
|
||||
free(pHisto);
|
||||
|
|
|
|||
|
|
@ -576,7 +576,7 @@ static void rpcFreeMsg(void *msg) {
|
|||
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) {
|
||||
SRpcConn *pConn;
|
||||
|
||||
uint32_t peerIp = taosGetIpFromFqdn(peerFqdn);
|
||||
uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn);
|
||||
if (peerIp == 0xFFFFFFFF) {
|
||||
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
|
||||
terrno = TSDB_CODE_RPC_FQDN_ERROR;
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ extern "C" {
|
|||
#define SYNC_MAX_FWDS 512
|
||||
#define SYNC_FWD_TIMER 300
|
||||
#define SYNC_ROLE_TIMER 15000 // ms
|
||||
#define SYNC_CHECK_INTERVAL 1 // ms
|
||||
#define SYNC_CHECK_INTERVAL 1000 // ms
|
||||
#define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms
|
||||
|
||||
#define nodeRole pNode->peerInfo[pNode->selfIndex]->role
|
||||
|
|
@ -86,9 +86,10 @@ typedef struct SsyncPeer {
|
|||
int32_t peerFd; // forward FD
|
||||
int32_t numOfRetrieves; // number of retrieves tried
|
||||
int32_t fileChanged; // a flag to indicate file is changed during retrieving process
|
||||
int32_t refCount;
|
||||
int64_t rid;
|
||||
void * timer;
|
||||
void * pConn;
|
||||
int32_t refCount; // reference count
|
||||
struct SSyncNode *pSyncNode;
|
||||
} SSyncPeer;
|
||||
|
||||
|
|
@ -98,6 +99,7 @@ typedef struct SSyncNode {
|
|||
int8_t quorum;
|
||||
int8_t selfIndex;
|
||||
uint32_t vgId;
|
||||
int32_t refCount;
|
||||
int64_t rid;
|
||||
SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator
|
||||
SSyncPeer * pMaster;
|
||||
|
|
@ -121,13 +123,13 @@ extern int32_t tsSyncNum;
|
|||
extern char tsNodeFqdn[TSDB_FQDN_LEN];
|
||||
extern char * syncStatus[];
|
||||
|
||||
void *syncRetrieveData(void *param);
|
||||
void *syncRestoreData(void *param);
|
||||
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
|
||||
void syncRestartConnection(SSyncPeer *pPeer);
|
||||
void syncBroadcastStatus(SSyncNode *pNode);
|
||||
void syncAddPeerRef(SSyncPeer *pPeer);
|
||||
int32_t syncDecPeerRef(SSyncPeer *pPeer);
|
||||
void * syncRetrieveData(void *param);
|
||||
void * syncRestoreData(void *param);
|
||||
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
|
||||
void syncRestartConnection(SSyncPeer *pPeer);
|
||||
void syncBroadcastStatus(SSyncNode *pNode);
|
||||
SSyncPeer *syncAcquirePeer(int64_t rid);
|
||||
void syncReleasePeer(SSyncPeer *pPeer);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,14 +25,14 @@ typedef struct {
|
|||
uint32_t serverIp;
|
||||
int16_t port;
|
||||
int32_t bufferSize;
|
||||
void (*processBrokenLink)(void *ahandle);
|
||||
int32_t (*processIncomingMsg)(void *ahandle, void *buffer);
|
||||
void (*processBrokenLink)(int64_t handleId);
|
||||
int32_t (*processIncomingMsg)(int64_t handleId, void *buffer);
|
||||
void (*processIncomingConn)(int32_t fd, uint32_t ip);
|
||||
} SPoolInfo;
|
||||
|
||||
void *syncOpenTcpThreadPool(SPoolInfo *pInfo);
|
||||
void syncCloseTcpThreadPool(void *);
|
||||
void *syncAllocateTcpConn(void *, void *ahandle, int32_t connFd);
|
||||
void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd);
|
||||
void syncFreeTcpConn(void *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@
|
|||
|
||||
static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
|
||||
static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
|
||||
static void arbProcessBrokenLink(void *param);
|
||||
static int32_t arbProcessPeerMsg(void *param, void *buffer);
|
||||
static void arbProcessBrokenLink(int64_t rid);
|
||||
static int32_t arbProcessPeerMsg(int64_t rid, void *buffer);
|
||||
static tsem_t tsArbSem;
|
||||
static void * tsArbTcpPool;
|
||||
|
||||
|
|
@ -138,20 +138,20 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
|
|||
|
||||
sDebug("%s, arbitrator request is accepted", pNode->id);
|
||||
pNode->nodeFd = connFd;
|
||||
pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, pNode, connFd);
|
||||
pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static void arbProcessBrokenLink(void *param) {
|
||||
SNodeConn *pNode = param;
|
||||
static void arbProcessBrokenLink(int64_t rid) {
|
||||
SNodeConn *pNode = (SNodeConn *)rid;
|
||||
|
||||
sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno));
|
||||
tfree(pNode);
|
||||
}
|
||||
|
||||
static int32_t arbProcessPeerMsg(void *param, void *buffer) {
|
||||
SNodeConn *pNode = param;
|
||||
static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) {
|
||||
SNodeConn *pNode = (SNodeConn *)rid;
|
||||
SSyncHead head;
|
||||
int32_t bytes = 0;
|
||||
char * cont = (char *)buffer;
|
||||
|
|
|
|||
|
|
@ -35,19 +35,21 @@ char tsNodeFqdn[TSDB_FQDN_LEN] = {0};
|
|||
static void * tsTcpPool = NULL;
|
||||
static void * tsSyncTmrCtrl = NULL;
|
||||
static void * tsVgIdHash = NULL;
|
||||
static int32_t tsSyncRefId = -1;
|
||||
static int32_t tsNodeRefId = -1;
|
||||
static int32_t tsPeerRefId = -1;
|
||||
|
||||
// local functions
|
||||
static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
|
||||
static void syncRecoverFromMaster(SSyncPeer *pPeer);
|
||||
static void syncCheckPeerConnection(void *param, void *tmrId);
|
||||
static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId);
|
||||
static void syncProcessBrokenLink(void *param);
|
||||
static int32_t syncProcessPeerMsg(void *param, void *buffer);
|
||||
static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId);
|
||||
static void syncProcessBrokenLink(int64_t rid);
|
||||
static int32_t syncProcessPeerMsg(int64_t rid, void *buffer);
|
||||
static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
|
||||
static void syncRemovePeer(SSyncPeer *pPeer);
|
||||
static void syncAddArbitrator(SSyncNode *pNode);
|
||||
static void syncFreeNode(void *);
|
||||
static void syncFreePeer(void *);
|
||||
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
|
||||
static void syncMonitorFwdInfos(void *param, void *tmrId);
|
||||
static void syncMonitorNodeRole(void *param, void *tmrId);
|
||||
|
|
@ -55,7 +57,12 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t c
|
|||
static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
|
||||
static void syncRestartPeer(SSyncPeer *pPeer);
|
||||
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp);
|
||||
|
||||
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
|
||||
static void syncStartCheckPeerConn(SSyncPeer *pPeer);
|
||||
static void syncStopCheckPeerConn(SSyncPeer *pPeer);
|
||||
static SSyncNode *syncAcquireNode(int64_t rid);
|
||||
static void syncReleaseNode(SSyncNode *pNode);
|
||||
|
||||
char* syncRole[] = {
|
||||
"offline",
|
||||
|
|
@ -87,29 +94,34 @@ int32_t syncInit() {
|
|||
tsTcpPool = syncOpenTcpThreadPool(&info);
|
||||
if (tsTcpPool == NULL) {
|
||||
sError("failed to init tcpPool");
|
||||
syncCleanUp();
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
|
||||
if (tsSyncTmrCtrl == NULL) {
|
||||
sError("failed to init tmrCtrl");
|
||||
syncCloseTcpThreadPool(tsTcpPool);
|
||||
tsTcpPool = NULL;
|
||||
syncCleanUp();
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
|
||||
if (tsVgIdHash == NULL) {
|
||||
sError("failed to init vgIdHash");
|
||||
taosTmrCleanUp(tsSyncTmrCtrl);
|
||||
syncCloseTcpThreadPool(tsTcpPool);
|
||||
tsTcpPool = NULL;
|
||||
tsSyncTmrCtrl = NULL;
|
||||
syncCleanUp();
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsSyncRefId = taosOpenRef(200, syncFreeNode);
|
||||
if (tsSyncRefId < 0) {
|
||||
tsNodeRefId = taosOpenRef(200, syncFreeNode);
|
||||
if (tsNodeRefId < 0) {
|
||||
sError("failed to init node ref");
|
||||
syncCleanUp();
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsPeerRefId = taosOpenRef(1000, syncFreePeer);
|
||||
if (tsPeerRefId < 0) {
|
||||
sError("failed to init peer ref");
|
||||
syncCleanUp();
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -121,12 +133,12 @@ int32_t syncInit() {
|
|||
}
|
||||
|
||||
void syncCleanUp() {
|
||||
if (tsTcpPool) {
|
||||
if (tsTcpPool != NULL) {
|
||||
syncCloseTcpThreadPool(tsTcpPool);
|
||||
tsTcpPool = NULL;
|
||||
}
|
||||
|
||||
if (tsSyncTmrCtrl) {
|
||||
if (tsSyncTmrCtrl != NULL) {
|
||||
taosTmrCleanUp(tsSyncTmrCtrl);
|
||||
tsSyncTmrCtrl = NULL;
|
||||
}
|
||||
|
|
@ -136,8 +148,15 @@ void syncCleanUp() {
|
|||
tsVgIdHash = NULL;
|
||||
}
|
||||
|
||||
taosCloseRef(tsSyncRefId);
|
||||
tsSyncRefId = -1;
|
||||
if (tsNodeRefId != -1) {
|
||||
taosCloseRef(tsNodeRefId);
|
||||
tsNodeRefId = -1;
|
||||
}
|
||||
|
||||
if (tsPeerRefId != -1) {
|
||||
taosCloseRef(tsPeerRefId);
|
||||
tsPeerRefId = -1;
|
||||
}
|
||||
|
||||
sInfo("sync module is cleaned up");
|
||||
}
|
||||
|
|
@ -170,7 +189,8 @@ int64_t syncStart(const SSyncInfo *pInfo) {
|
|||
pNode->quorum = pCfg->quorum;
|
||||
if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
|
||||
|
||||
pNode->rid = taosAddRef(tsSyncRefId, pNode);
|
||||
pNode->refCount = 1;
|
||||
pNode->rid = taosAddRef(tsNodeRefId, pNode);
|
||||
if (pNode->rid < 0) {
|
||||
syncFreeNode(pNode);
|
||||
return -1;
|
||||
|
|
@ -232,13 +252,18 @@ int64_t syncStart(const SSyncInfo *pInfo) {
|
|||
(*pNode->notifyRole)(pNode->vgId, nodeRole);
|
||||
}
|
||||
|
||||
syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb
|
||||
for (int32_t index = 0; index < pNode->replica; ++index) {
|
||||
syncStartCheckPeerConn(pNode->peerInfo[index]);
|
||||
}
|
||||
|
||||
return pNode->rid;
|
||||
}
|
||||
|
||||
void syncStop(int64_t rid) {
|
||||
SSyncPeer *pPeer;
|
||||
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return;
|
||||
|
||||
sInfo("vgId:%d, cleanup sync", pNode->vgId);
|
||||
|
|
@ -259,14 +284,14 @@ void syncStop(int64_t rid) {
|
|||
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
taosRemoveRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
taosRemoveRef(tsNodeRefId, rid);
|
||||
}
|
||||
|
||||
int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
|
||||
int32_t i, j;
|
||||
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
|
||||
|
||||
sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
|
||||
|
|
@ -274,6 +299,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
|
|||
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
|
||||
syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb
|
||||
for (int32_t index = 0; index < pNode->replica; ++index) {
|
||||
syncStopCheckPeerConn(pNode->peerInfo[index]);
|
||||
}
|
||||
|
||||
for (i = 0; i < pNode->replica; ++i) {
|
||||
for (j = 0; j < pNewCfg->replica; ++j) {
|
||||
if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) &&
|
||||
|
|
@ -330,28 +360,32 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
|
|||
(*pNode->notifyRole)(pNode->vgId, nodeRole);
|
||||
}
|
||||
|
||||
syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb
|
||||
for (int32_t index = 0; index < pNode->replica; ++index) {
|
||||
syncStartCheckPeerConn(pNode->peerInfo[index]);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum);
|
||||
syncBroadcastStatus(pNode);
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) {
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
if (pNode == NULL) return 0;
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return 0;
|
||||
|
||||
int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
|
||||
syncReleaseNode(pNode);
|
||||
return code;
|
||||
}
|
||||
|
||||
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return;
|
||||
|
||||
SSyncPeer *pPeer = pNode->pMaster;
|
||||
|
|
@ -367,14 +401,14 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
|
|||
}
|
||||
}
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
}
|
||||
|
||||
#if 0
|
||||
void syncRecover(int64_t rid) {
|
||||
SSyncPeer *pPeer;
|
||||
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return;
|
||||
|
||||
// to do: add a few lines to check if recover is OK
|
||||
|
|
@ -395,12 +429,12 @@ void syncRecover(int64_t rid) {
|
|||
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return -1;
|
||||
|
||||
pNodesRole->selfIndex = pNode->selfIndex;
|
||||
|
|
@ -409,8 +443,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
|
|||
pNodesRole->role[i] = pNode->peerInfo[i]->role;
|
||||
}
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
|
||||
syncReleaseNode(pNode);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -446,24 +479,61 @@ static void syncAddArbitrator(SSyncNode *pNode) {
|
|||
static void syncFreeNode(void *param) {
|
||||
SSyncNode *pNode = param;
|
||||
|
||||
int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
|
||||
sDebug("vgId:%d, syncnode is freed, refCount:%d", pNode->vgId, refCount);
|
||||
|
||||
pthread_mutex_destroy(&pNode->mutex);
|
||||
tfree(pNode->pRecv);
|
||||
tfree(pNode->pSyncFwds);
|
||||
tfree(pNode);
|
||||
}
|
||||
|
||||
void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); }
|
||||
|
||||
int32_t syncDecPeerRef(SSyncPeer *pPeer) {
|
||||
if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) {
|
||||
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
|
||||
|
||||
sDebug("%s, resource is freed", pPeer->id);
|
||||
tfree(pPeer);
|
||||
return 0;
|
||||
SSyncNode *syncAcquireNode(int64_t rid) {
|
||||
SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid);
|
||||
if (pNode == NULL) {
|
||||
sDebug("failed to acquire syncnode from refId:%" PRId64, rid);
|
||||
} else {
|
||||
int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1);
|
||||
sTrace("vgId:%d, acquire syncnode refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount);
|
||||
}
|
||||
|
||||
return 1;
|
||||
return pNode;
|
||||
}
|
||||
|
||||
void syncReleaseNode(SSyncNode *pNode) {
|
||||
int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
|
||||
sTrace("vgId:%d, dec syncnode refId:%" PRId64 " refCount:%d", pNode->vgId, pNode->rid, refCount);
|
||||
|
||||
taosReleaseRef(tsNodeRefId, pNode->rid);
|
||||
}
|
||||
|
||||
static void syncFreePeer(void *param) {
|
||||
SSyncPeer *pPeer = param;
|
||||
|
||||
int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1);
|
||||
sDebug("%s, peer is freed, refCount:%d", pPeer->id, refCount);
|
||||
|
||||
syncReleaseNode(pPeer->pSyncNode);
|
||||
tfree(pPeer);
|
||||
}
|
||||
|
||||
SSyncPeer *syncAcquirePeer(int64_t rid) {
|
||||
SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid);
|
||||
if (pPeer == NULL) {
|
||||
sDebug("failed to acquire peer from refId:%" PRId64, rid);
|
||||
} else {
|
||||
int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1);
|
||||
sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount);
|
||||
}
|
||||
|
||||
return pPeer;
|
||||
}
|
||||
|
||||
void syncReleasePeer(SSyncPeer *pPeer) {
|
||||
int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1);
|
||||
sTrace("%s, dec peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount);
|
||||
|
||||
taosReleaseRef(tsPeerRefId, pPeer->rid);
|
||||
}
|
||||
|
||||
static void syncClosePeerConn(SSyncPeer *pPeer) {
|
||||
|
|
@ -473,7 +543,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) {
|
|||
taosClose(pPeer->syncFd);
|
||||
if (pPeer->peerFd >= 0) {
|
||||
pPeer->peerFd = -1;
|
||||
syncFreeTcpConn(pPeer->pConn);
|
||||
void *pConn = pPeer->pConn;
|
||||
if (pConn != NULL) syncFreeTcpConn(pPeer->pConn);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -482,11 +553,32 @@ static void syncRemovePeer(SSyncPeer *pPeer) {
|
|||
|
||||
pPeer->ip = 0;
|
||||
syncClosePeerConn(pPeer);
|
||||
syncDecPeerRef(pPeer);
|
||||
//taosRemoveRef(tsPeerRefId, pPeer->rid);
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncStartCheckPeerConn(SSyncPeer *pPeer) {
|
||||
if (pPeer == NULL) return;
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
|
||||
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
|
||||
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
|
||||
if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
|
||||
sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs);
|
||||
taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
}
|
||||
}
|
||||
|
||||
static void syncStopCheckPeerConn(SSyncPeer *pPeer) {
|
||||
if (pPeer == NULL) return;
|
||||
|
||||
taosTmrStopA(&pPeer->timer);
|
||||
sDebug("%s, stop check peer connection", pPeer->id);
|
||||
}
|
||||
|
||||
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
|
||||
uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn);
|
||||
uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn);
|
||||
if (ip == 0xFFFFFFFF) {
|
||||
sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno));
|
||||
terrno = TSDB_CODE_RPC_FQDN_ERROR;
|
||||
|
|
@ -508,17 +600,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
|
|||
pPeer->role = TAOS_SYNC_ROLE_OFFLINE;
|
||||
pPeer->pSyncNode = pNode;
|
||||
pPeer->refCount = 1;
|
||||
pPeer->rid = taosAddRef(tsPeerRefId, pPeer);
|
||||
|
||||
sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port);
|
||||
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
|
||||
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
|
||||
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
|
||||
if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
|
||||
sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs);
|
||||
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
}
|
||||
sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid);
|
||||
|
||||
taosAcquireRef(tsSyncRefId, pNode->rid);
|
||||
(void)syncAcquireNode(pNode->rid);
|
||||
return pPeer;
|
||||
}
|
||||
|
||||
|
|
@ -560,6 +646,9 @@ static void syncChooseMaster(SSyncNode *pNode) {
|
|||
index = i;
|
||||
}
|
||||
}
|
||||
sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index);
|
||||
} else {
|
||||
sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica);
|
||||
}
|
||||
|
||||
// add arbitrator connection
|
||||
|
|
@ -580,6 +669,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (index >= 0) {
|
||||
sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId,
|
||||
pNode->peerInfo[index]->id, index, onlineNum, replica);
|
||||
}
|
||||
}
|
||||
|
||||
if (index >= 0) {
|
||||
|
|
@ -621,9 +715,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
|
|||
|
||||
if (onlineNum <= replica * 0.5) {
|
||||
if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
|
||||
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
|
||||
if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) {
|
||||
sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica);
|
||||
} else {
|
||||
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
|
||||
sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
|
||||
}
|
||||
(*pNode->notifyRole)(pNode->vgId, nodeRole);
|
||||
sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
|
||||
}
|
||||
} else {
|
||||
for (int32_t index = 0; index < pNode->replica; ++index) {
|
||||
|
|
@ -678,7 +776,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
|
|||
if (pMaster) {
|
||||
// master is there
|
||||
pNode->pMaster = pMaster;
|
||||
sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version);
|
||||
sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version);
|
||||
|
||||
if (syncValidateMaster(pPeer) < 0) return;
|
||||
|
||||
|
|
@ -711,10 +809,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
|
|||
}
|
||||
|
||||
if (consistent) {
|
||||
sDebug("vgId:%d, choose master", pNode->vgId);
|
||||
sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica);
|
||||
syncChooseMaster(pNode);
|
||||
} else {
|
||||
sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId);
|
||||
sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -743,7 +841,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
|
|||
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
|
||||
if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
|
||||
sDebug("%s, check peer connection in 1000 ms", pPeer->id);
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -772,25 +870,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
|
|||
}
|
||||
|
||||
// start a new thread to retrieve the data
|
||||
syncAddPeerRef(pPeer);
|
||||
(void)syncAcquirePeer(pPeer->rid);
|
||||
|
||||
pthread_attr_t thattr;
|
||||
pthread_t thread;
|
||||
pthread_attr_init(&thattr);
|
||||
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
|
||||
int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
|
||||
int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid);
|
||||
pthread_attr_destroy(&thattr);
|
||||
|
||||
if (ret != 0) {
|
||||
sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno));
|
||||
syncDecPeerRef(pPeer);
|
||||
} else {
|
||||
pPeer->sstatus = TAOS_SYNC_STATUS_START;
|
||||
sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
|
||||
}
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncNotStarted(void *param, void *tmrId) {
|
||||
SSyncPeer *pPeer = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
|
|
@ -799,15 +902,22 @@ static void syncNotStarted(void *param, void *tmrId) {
|
|||
sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
|
||||
syncRestartConnection(pPeer);
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncTryRecoverFromMaster(void *param, void *tmrId) {
|
||||
SSyncPeer *pPeer = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
syncRecoverFromMaster(pPeer);
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncRecoverFromMaster(SSyncPeer *pPeer) {
|
||||
|
|
@ -823,7 +933,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
|
|||
// Ensure the sync of mnode not interrupted
|
||||
if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) {
|
||||
sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
|
||||
taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -832,7 +942,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
|
|||
SSyncMsg msg;
|
||||
syncBuildSyncReqMsg(&msg, pNode->vgId);
|
||||
|
||||
taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
|
||||
if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) {
|
||||
sError("%s, failed to send sync-req to peer", pPeer->id);
|
||||
|
|
@ -920,8 +1030,10 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t syncProcessPeerMsg(void *param, void *buffer) {
|
||||
SSyncPeer *pPeer = param;
|
||||
static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) {
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return -1;
|
||||
|
||||
SSyncHead *pHead = buffer;
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
|
|
@ -942,12 +1054,17 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) {
|
|||
}
|
||||
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
syncReleasePeer(pPeer);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) {
|
||||
if (pPeer->peerFd < 0 || pPeer->ip == 0) return;
|
||||
static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) {
|
||||
if (pPeer->peerFd < 0 || pPeer->ip == 0) {
|
||||
sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd);
|
||||
syncRestartConnection(pPeer);
|
||||
return -1;
|
||||
}
|
||||
|
||||
SSyncNode * pNode = pPeer->pSyncNode;
|
||||
SPeersStatus msg;
|
||||
|
|
@ -970,9 +1087,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type
|
|||
sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d",
|
||||
pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role],
|
||||
syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd);
|
||||
return 0;
|
||||
} else {
|
||||
sDebug("%s, failed to send status msg, restart", pPeer->id);
|
||||
syncRestartConnection(pPeer);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -989,7 +1108,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
|
|||
int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
|
||||
if (connFd < 0) {
|
||||
sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno));
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1000,17 +1119,19 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
|
|||
sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId);
|
||||
pPeer->peerFd = connFd;
|
||||
pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
|
||||
pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd);
|
||||
syncAddPeerRef(pPeer);
|
||||
pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd);
|
||||
} else {
|
||||
sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno));
|
||||
taosClose(connFd);
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
|
||||
taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
|
||||
}
|
||||
}
|
||||
|
||||
static void syncCheckPeerConnection(void *param, void *tmrId) {
|
||||
SSyncPeer *pPeer = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
|
|
@ -1019,6 +1140,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) {
|
|||
syncSetupPeerConnection(pPeer);
|
||||
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
|
||||
|
|
@ -1029,8 +1152,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
|
|||
pthread_attr_init(&thattr);
|
||||
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
|
||||
|
||||
syncAddPeerRef(pPeer);
|
||||
int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
|
||||
(void)syncAcquirePeer(pPeer->rid);
|
||||
|
||||
int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, (void *)pPeer->rid);
|
||||
pthread_attr_destroy(&thattr);
|
||||
|
||||
if (ret < 0) {
|
||||
|
|
@ -1038,10 +1162,11 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
|
|||
nodeSStatus = TAOS_SYNC_STATUS_INIT;
|
||||
sError("%s, failed to create sync thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
|
||||
taosClose(pPeer->syncFd);
|
||||
syncDecPeerRef(pPeer);
|
||||
} else {
|
||||
sInfo("%s, sync connection is up", pPeer->id);
|
||||
}
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
|
||||
|
|
@ -1073,7 +1198,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
|
|||
return;
|
||||
}
|
||||
|
||||
sDebug("vgId:%d, sync msg is received, tranId:%u", vgId, msg.tranId);
|
||||
sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId);
|
||||
|
||||
SSyncNode *pNode = *ppNode;
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
|
|
@ -1101,8 +1226,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
|
|||
sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
|
||||
syncClosePeerConn(pPeer);
|
||||
pPeer->peerFd = connFd;
|
||||
pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd);
|
||||
syncAddPeerRef(pPeer);
|
||||
pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd);
|
||||
sDebug("%s, ready to exchange data", pPeer->id);
|
||||
syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId());
|
||||
}
|
||||
|
|
@ -1111,23 +1235,21 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
|
|||
pthread_mutex_unlock(&pNode->mutex);
|
||||
}
|
||||
|
||||
static void syncProcessBrokenLink(void *param) {
|
||||
if (param == NULL) return; // the connection for arbitrator
|
||||
SSyncPeer *pPeer = param;
|
||||
static void syncProcessBrokenLink(int64_t rid) {
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
|
||||
pthread_mutex_lock(&pNode->mutex);
|
||||
|
||||
sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd);
|
||||
pPeer->peerFd = -1;
|
||||
|
||||
if (syncDecPeerRef(pPeer) != 0) {
|
||||
syncRestartConnection(pPeer);
|
||||
}
|
||||
|
||||
syncRestartConnection(pPeer);
|
||||
pthread_mutex_unlock(&pNode->mutex);
|
||||
taosReleaseRef(tsSyncRefId, pNode->rid);
|
||||
|
||||
syncReleasePeer(pPeer);
|
||||
}
|
||||
|
||||
static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
|
||||
|
|
@ -1198,7 +1320,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
|
|||
|
||||
static void syncMonitorNodeRole(void *param, void *tmrId) {
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return;
|
||||
|
||||
for (int32_t index = 0; index < pNode->replica; index++) {
|
||||
|
|
@ -1215,12 +1337,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) {
|
|||
}
|
||||
|
||||
pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
}
|
||||
|
||||
static void syncMonitorFwdInfos(void *param, void *tmrId) {
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
|
||||
SSyncNode *pNode = syncAcquireNode(rid);
|
||||
if (pNode == NULL) return;
|
||||
|
||||
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
|
||||
|
|
@ -1246,7 +1368,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
|
|||
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
|
||||
}
|
||||
|
||||
taosReleaseRef(tsSyncRefId, rid);
|
||||
syncReleaseNode(pNode);
|
||||
}
|
||||
|
||||
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) {
|
||||
|
|
@ -1266,7 +1388,9 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
|
|||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SYN_INVALID_VERSION;
|
||||
if (pNode->replica != 1) {
|
||||
return TSDB_CODE_SYN_INVALID_VERSION;
|
||||
}
|
||||
}
|
||||
|
||||
// always update version
|
||||
|
|
|
|||
|
|
@ -90,15 +90,18 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
|
|||
break;
|
||||
}
|
||||
|
||||
sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id,
|
||||
minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic);
|
||||
|
||||
// remove extra files on slave between the current and last index
|
||||
syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
|
||||
pindex = minfo.index;
|
||||
|
||||
// check the file info
|
||||
sinfo = minfo;
|
||||
sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size);
|
||||
sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size,
|
||||
&sinfo.fversion);
|
||||
sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
|
||||
sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, sinfo.name,
|
||||
sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic);
|
||||
|
||||
// if file not there or magic is not the same, file shall be synced
|
||||
memset(&fileAck, 0, sizeof(SFileAck));
|
||||
|
|
@ -116,6 +119,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
|
|||
if (fileAck.sync == 0) {
|
||||
sDebug("%s, %s is the same", pPeer->id, minfo.name);
|
||||
continue;
|
||||
} else {
|
||||
sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
|
||||
}
|
||||
|
||||
// if sync is required, open file, receive from master, and write to file
|
||||
|
|
@ -155,7 +160,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t syncRestoreWal(SSyncPeer *pPeer) {
|
||||
static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
int32_t ret, code = -1;
|
||||
uint64_t lastVer = 0;
|
||||
|
|
@ -198,6 +203,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
|
|||
}
|
||||
|
||||
free(pHead);
|
||||
*wver = lastVer;
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -321,12 +327,19 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
|
|||
|
||||
nodeVersion = fversion;
|
||||
|
||||
sInfo("%s, start to restore wal", pPeer->id);
|
||||
if (syncRestoreWal(pPeer) < 0) {
|
||||
sError("%s, failed to restore wal", pPeer->id);
|
||||
sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
|
||||
uint64_t wver = 0;
|
||||
code = syncRestoreWal(pPeer, &wver); // lastwar
|
||||
if (code < 0) {
|
||||
sError("%s, failed to restore wal, code:%d", pPeer->id, code);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (wver != 0) {
|
||||
nodeVersion = wver;
|
||||
sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion);
|
||||
}
|
||||
|
||||
nodeSStatus = TAOS_SYNC_STATUS_CACHE;
|
||||
sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
|
||||
if (syncProcessBufferedFwd(pPeer) < 0) {
|
||||
|
|
@ -338,7 +351,10 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
|
|||
}
|
||||
|
||||
void *syncRestoreData(void *param) {
|
||||
SSyncPeer *pPeer = param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return NULL;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
|
||||
taosBlockSIGPIPE();
|
||||
|
|
@ -369,7 +385,7 @@ void *syncRestoreData(void *param) {
|
|||
taosClose(pPeer->syncFd);
|
||||
syncCloseRecvBuffer(pNode);
|
||||
__sync_fetch_and_sub(&tsSyncNum, 1);
|
||||
syncDecPeerRef(pPeer);
|
||||
syncReleasePeer(pPeer);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
|
|||
fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
|
||||
&fileInfo.size, &fileInfo.fversion);
|
||||
syncBuildFileInfo(&fileInfo, pNode->vgId);
|
||||
sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
|
||||
sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, fileInfo.name,
|
||||
fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic);
|
||||
|
||||
// send the file info
|
||||
int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo));
|
||||
|
|
@ -144,6 +145,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
|
|||
fileInfo.index++;
|
||||
sDebug("%s, %s is the same", pPeer->id, fileInfo.name);
|
||||
continue;
|
||||
} else {
|
||||
sDebug("%s, %s will be sent", pPeer->id, fileInfo.name);
|
||||
}
|
||||
|
||||
// get the full path to file
|
||||
|
|
@ -461,7 +464,10 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
|
|||
}
|
||||
|
||||
void *syncRetrieveData(void *param) {
|
||||
SSyncPeer *pPeer = (SSyncPeer *)param;
|
||||
int64_t rid = (int64_t)param;
|
||||
SSyncPeer *pPeer = syncAcquirePeer(rid);
|
||||
if (pPeer == NULL) return NULL;
|
||||
|
||||
SSyncNode *pNode = pPeer->pSyncNode;
|
||||
taosBlockSIGPIPE();
|
||||
|
||||
|
|
@ -490,7 +496,7 @@ void *syncRetrieveData(void *param) {
|
|||
|
||||
pPeer->fileChanged = 0;
|
||||
taosClose(pPeer->syncFd);
|
||||
syncDecPeerRef(pPeer);
|
||||
syncReleasePeer(pPeer);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ typedef struct SPoolObj {
|
|||
|
||||
typedef struct {
|
||||
SThreadObj *pThread;
|
||||
void * ahandle;
|
||||
int64_t handleId;
|
||||
int32_t fd;
|
||||
int32_t closedByApp;
|
||||
} SConnObj;
|
||||
|
|
@ -112,7 +112,7 @@ void syncCloseTcpThreadPool(void *param) {
|
|||
tfree(pPool);
|
||||
}
|
||||
|
||||
void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
|
||||
void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) {
|
||||
struct epoll_event event;
|
||||
SPoolObj *pPool = param;
|
||||
|
||||
|
|
@ -130,7 +130,7 @@ void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
|
|||
|
||||
pConn->fd = connFd;
|
||||
pConn->pThread = pThread;
|
||||
pConn->ahandle = pPeer;
|
||||
pConn->handleId = rid;
|
||||
pConn->closedByApp = 0;
|
||||
|
||||
event.events = EPOLLIN | EPOLLRDHUP;
|
||||
|
|
@ -164,7 +164,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
|
|||
SPoolInfo * pInfo = &pPool->info;
|
||||
|
||||
if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
|
||||
(*pInfo->processBrokenLink)(pConn->ahandle);
|
||||
(*pInfo->processBrokenLink)(pConn->handleId);
|
||||
|
||||
pThread->numOfFds--;
|
||||
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
|
||||
|
|
@ -221,7 +221,7 @@ static void *syncProcessTcpData(void *param) {
|
|||
}
|
||||
|
||||
if (pConn->closedByApp == 0) {
|
||||
if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) {
|
||||
if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) {
|
||||
syncFreeTcpConn(pConn);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,19 +31,19 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int tsdbDebugFlag;
|
||||
extern uint32_t tsdbDebugFlag;
|
||||
|
||||
#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}
|
||||
#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}
|
||||
#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}
|
||||
#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }}
|
||||
#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
|
||||
#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
|
||||
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
|
||||
#define TSDB_MAX_TABLE_SCHEMAS 16
|
||||
#define TSDB_FILE_HEAD_SIZE 512
|
||||
#define TSDB_FILE_DELIMITER 0xF00AFA0F
|
||||
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
|
||||
#define TSDB_FILE_HEAD_SIZE 512
|
||||
#define TSDB_FILE_DELIMITER 0xF00AFA0F
|
||||
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
|
||||
|
||||
#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,11 @@ _err:
|
|||
|
||||
static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
|
||||
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
|
||||
SMemTable *pIMem = pRepo->imem;
|
||||
tsdbLockRepo(pRepo);
|
||||
pRepo->imem = NULL;
|
||||
tsdbUnlockRepo(pRepo);
|
||||
tsdbUnRefMemTable(pRepo, pIMem);
|
||||
sem_post(&(pRepo->readyToCommit));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "tsdbMain.h"
|
||||
|
||||
#define TSDB_DATA_SKIPLIST_LEVEL 5
|
||||
#define TSDB_MAX_INSERT_BATCH 512
|
||||
|
||||
static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
|
||||
static void tsdbFreeMemTable(SMemTable *pMemTable);
|
||||
|
|
@ -205,7 +206,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
|
|||
int tsdbAsyncCommit(STsdbRepo *pRepo) {
|
||||
if (pRepo->mem == NULL) return 0;
|
||||
|
||||
SMemTable *pIMem = pRepo->imem;
|
||||
ASSERT(pRepo->imem == NULL);
|
||||
|
||||
sem_wait(&(pRepo->readyToCommit));
|
||||
|
||||
|
|
@ -220,8 +221,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
|
|||
tsdbScheduleCommit(pRepo);
|
||||
if (tsdbUnlockRepo(pRepo) < 0) return -1;
|
||||
|
||||
if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -606,19 +605,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
|
|||
STable * pTable = NULL;
|
||||
SSubmitBlkIter blkIter = {0};
|
||||
SDataRow row = NULL;
|
||||
void ** rows = NULL;
|
||||
void * rows[TSDB_MAX_INSERT_BATCH] = {0};
|
||||
int rowCounter = 0;
|
||||
|
||||
ASSERT(pBlock->tid < pMeta->maxTables);
|
||||
pTable = pMeta->tables[pBlock->tid];
|
||||
ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
|
||||
|
||||
rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
|
||||
if (rows == NULL) {
|
||||
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsdbInitSubmitBlkIter(pBlock, &blkIter);
|
||||
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
|
||||
if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
|
||||
|
|
@ -632,9 +625,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
|
|||
if (rows[rowCounter] != NULL) {
|
||||
rowCounter++;
|
||||
}
|
||||
|
||||
if (rowCounter == TSDB_MAX_INSERT_BATCH) {
|
||||
if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
rowCounter = 0;
|
||||
memset(rows, 0, sizeof(rows));
|
||||
}
|
||||
}
|
||||
|
||||
if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
|
||||
if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
|
|
@ -642,11 +644,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
|
|||
pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
|
||||
pRepo->stat.totalStorage += points * schemaVLen(pSchema);
|
||||
|
||||
free(rows);
|
||||
return 0;
|
||||
|
||||
_err:
|
||||
free(rows);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -956,9 +956,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
|
|||
return code;
|
||||
}
|
||||
|
||||
SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
|
||||
SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0];
|
||||
if (pCheckInfo->lastKey < pBlock->keyLast) {
|
||||
cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
|
||||
cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
|
||||
} else {
|
||||
cur->pos = pBlock->numOfRows - 1;
|
||||
}
|
||||
|
|
@ -1704,7 +1704,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) {
|
||||
static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists);
|
||||
|
||||
static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) {
|
||||
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
|
||||
SQueryFilePos* cur = &pQueryHandle->cur;
|
||||
|
||||
while(1) {
|
||||
int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
|
||||
if (code != TSDB_CODE_SUCCESS || *exists) {
|
||||
return code;
|
||||
}
|
||||
|
||||
if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
|
||||
(cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
|
||||
// all data blocks in current file has been checked already, try next file if exists
|
||||
return getFirstFileDataBlock(pQueryHandle, exists);
|
||||
} else { // next block of the same file
|
||||
cur->slot += step;
|
||||
cur->mixBlock = false;
|
||||
cur->blockCompleted = false;
|
||||
pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) {
|
||||
pQueryHandle->numOfBlocks = 0;
|
||||
SQueryFilePos* cur = &pQueryHandle->cur;
|
||||
|
||||
|
|
@ -1789,7 +1814,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
|
|||
cur->fid = pQueryHandle->pFileGroup->fileId;
|
||||
|
||||
STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
|
||||
return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists);
|
||||
return getDataBlockRv(pQueryHandle, pBlockInfo, exists);
|
||||
}
|
||||
|
||||
static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) {
|
||||
assert(cur != NULL && numOfBlocks > 0);
|
||||
return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav);
|
||||
}
|
||||
|
||||
static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
|
||||
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
|
||||
|
||||
SQueryFilePos* cur = &pQueryHandle->cur;
|
||||
assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0);
|
||||
|
||||
cur->slot += step;
|
||||
cur->mixBlock = false;
|
||||
cur->blockCompleted = false;
|
||||
}
|
||||
|
||||
static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
|
||||
|
|
@ -1800,14 +1841,14 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
|
|||
if (!pQueryHandle->locateStart) {
|
||||
pQueryHandle->locateStart = true;
|
||||
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
|
||||
int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
|
||||
int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
|
||||
|
||||
pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
|
||||
tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
|
||||
tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
|
||||
pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
|
||||
|
||||
return getDataBlocksInFilesImpl(pQueryHandle, exists);
|
||||
return getFirstFileDataBlock(pQueryHandle, exists);
|
||||
} else {
|
||||
// check if current file block is all consumed
|
||||
STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
|
||||
|
|
@ -1815,27 +1856,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
|
|||
|
||||
// current block is done, try next
|
||||
if ((!cur->mixBlock) || cur->blockCompleted) {
|
||||
if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
|
||||
(cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
|
||||
// all data blocks in current file has been checked already, try next file if exists
|
||||
return getDataBlocksInFilesImpl(pQueryHandle, exists);
|
||||
} else {
|
||||
// next block of the same file
|
||||
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
|
||||
cur->slot += step;
|
||||
|
||||
cur->mixBlock = false;
|
||||
cur->blockCompleted = false;
|
||||
|
||||
STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
|
||||
return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
|
||||
}
|
||||
// all data blocks in current file has been checked already, try next file if exists
|
||||
} else {
|
||||
tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo);
|
||||
tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos,
|
||||
pQueryHandle->qinfo);
|
||||
int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo);
|
||||
*exists = pQueryHandle->realNumOfRows > 0;
|
||||
*exists = (pQueryHandle->realNumOfRows > 0);
|
||||
|
||||
return code;
|
||||
if (code != TSDB_CODE_SUCCESS || *exists) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
// current block is empty, try next block in file
|
||||
// all data blocks in current file has been checked already, try next file if exists
|
||||
if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) {
|
||||
return getFirstFileDataBlock(pQueryHandle, exists);
|
||||
} else {
|
||||
moveToNextDataBlockInCurrentFile(pQueryHandle);
|
||||
STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
|
||||
return getDataBlockRv(pQueryHandle, pNext, exists);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid);
|
|||
// return the number of references in system
|
||||
int taosListRef();
|
||||
|
||||
#define RID_VALID(x) ((x) > 0)
|
||||
|
||||
/* sample code to iterate the refs
|
||||
|
||||
void demoIterateRefs(int rsetId) {
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
|
|||
int32_t taosKeepTcpAlive(SOCKET sockFd);
|
||||
|
||||
int32_t taosGetFqdn(char *);
|
||||
uint32_t taosGetIpFromFqdn(const char *);
|
||||
uint32_t taosGetIpv4FromFqdn(const char *);
|
||||
void tinet_ntoa(char *ipstr, uint32_t ip);
|
||||
uint32_t ip2uint(const char *const ip_addr);
|
||||
|
||||
|
|
|
|||
|
|
@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
|
|||
return;
|
||||
}
|
||||
|
||||
__cache_wr_lock(pCacheObj);
|
||||
STrashElem *pElem = calloc(1, sizeof(STrashElem));
|
||||
pElem->pData = pNode;
|
||||
pElem->prev = NULL;
|
||||
pElem->prev = NULL;
|
||||
pElem->next = NULL;
|
||||
pNode->inTrashcan = true;
|
||||
pNode->pTNodeHeader = pElem;
|
||||
|
||||
__cache_wr_lock(pCacheObj);
|
||||
pElem->next = pCacheObj->pTrash;
|
||||
if (pCacheObj->pTrash) {
|
||||
pCacheObj->pTrash->prev = pElem;
|
||||
|
|
@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
|
|||
pCacheObj->numOfElemsInTrash++;
|
||||
__cache_unlock(pCacheObj);
|
||||
|
||||
uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
|
||||
pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
|
||||
uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key,
|
||||
pNode->data, pElem, pCacheObj->numOfElemsInTrash);
|
||||
}
|
||||
|
||||
void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ int32_t tsLogKeepDays = 0;
|
|||
int32_t tsAsyncLog = 1;
|
||||
float tsTotalLogDirGB = 0;
|
||||
float tsAvailLogDirGB = 0;
|
||||
float tsMinimalLogDirGB = 0.1f;
|
||||
float tsMinimalLogDirGB = 1.0f;
|
||||
#ifdef _TD_POWER_
|
||||
char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power";
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -449,7 +449,7 @@ static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) {
|
|||
int32_t endPort = startPort + 11;
|
||||
uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen);
|
||||
|
||||
uint32_t serverIp = taosGetIpFromFqdn(host);
|
||||
uint32_t serverIp = taosGetIpv4FromFqdn(host);
|
||||
if (serverIp == 0xFFFFFFFF) {
|
||||
uError("failed to resolve fqdn:%s", host);
|
||||
exit(-1);
|
||||
|
|
|
|||
|
|
@ -280,7 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
|
|||
tSkipListRLock(pSkipList);
|
||||
|
||||
if (iter->order == TSDB_ORDER_ASC) {
|
||||
if (iter->cur == pSkipList->pTail) return false;
|
||||
// no data in the skip list
|
||||
if (iter->cur == pSkipList->pTail || iter->next == NULL) {
|
||||
iter->cur = pSkipList->pTail;
|
||||
tSkipListUnlock(pSkipList);
|
||||
return false;
|
||||
}
|
||||
|
||||
iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
|
||||
|
||||
// a new node is inserted into between iter->cur and iter->next, ignore it
|
||||
|
|
@ -292,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
|
|||
iter->step++;
|
||||
} else {
|
||||
if (iter->cur == pSkipList->pHead) {
|
||||
iter->cur = pSkipList->pHead;
|
||||
tSkipListUnlock(pSkipList);
|
||||
return false;
|
||||
}
|
||||
|
||||
iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
|
||||
|
||||
// a new node is inserted into between iter->cur and iter->next, ignore it
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@
|
|||
#include "tsocket.h"
|
||||
#include "taoserror.h"
|
||||
|
||||
#ifndef SIGPIPE
|
||||
#define SIGPIPE EPIPE
|
||||
#endif
|
||||
|
||||
int32_t taosGetFqdn(char *fqdn) {
|
||||
char hostname[1024];
|
||||
hostname[1023] = '\0';
|
||||
|
|
@ -40,9 +44,9 @@ int32_t taosGetFqdn(char *fqdn) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
uint32_t taosGetIpFromFqdn(const char *fqdn) {
|
||||
uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
|
||||
struct addrinfo hints = {0};
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_family = AF_INET;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
struct addrinfo *result = NULL;
|
||||
|
|
@ -115,6 +119,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
|
|||
nleft -= nwritten;
|
||||
ptr += nwritten;
|
||||
}
|
||||
|
||||
if (errno == SIGPIPE || errno == EPIPE) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return (nbytes - nleft);
|
||||
|
|
@ -142,6 +150,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
|
|||
nleft -= nread;
|
||||
ptr += nread;
|
||||
}
|
||||
|
||||
if (errno == SIGPIPE || errno == EPIPE) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return (nbytes - nleft);
|
||||
|
|
|
|||
|
|
@ -56,7 +56,8 @@ typedef struct {
|
|||
int64_t sync;
|
||||
void * events;
|
||||
void * cq; // continuous query
|
||||
int32_t cfgVersion;
|
||||
int32_t dbCfgVersion;
|
||||
int32_t vgCfgVersion;
|
||||
STsdbCfg tsdbCfg;
|
||||
SSyncCfg syncCfg;
|
||||
SWalCfg walCfg;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
|
||||
static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
|
||||
tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
|
||||
pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion;
|
||||
pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion;
|
||||
pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion;
|
||||
pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize;
|
||||
pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks;
|
||||
pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile;
|
||||
|
|
@ -95,12 +96,20 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
|
|||
}
|
||||
tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db));
|
||||
|
||||
cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
|
||||
if (!cfgVersion || cfgVersion->type != cJSON_Number) {
|
||||
cJSON *dbCfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
|
||||
if (!dbCfgVersion || dbCfgVersion->type != cJSON_Number) {
|
||||
vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file);
|
||||
goto PARSE_VCFG_ERROR;
|
||||
}
|
||||
vnodeMsg.cfg.cfgVersion = cfgVersion->valueint;
|
||||
vnodeMsg.cfg.dbCfgVersion = dbCfgVersion->valueint;
|
||||
|
||||
cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion");
|
||||
if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) {
|
||||
vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file);
|
||||
vnodeMsg.cfg.vgCfgVersion = 0;
|
||||
} else {
|
||||
vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint;
|
||||
}
|
||||
|
||||
cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
|
||||
if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
|
||||
|
|
@ -278,7 +287,8 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
|
|||
|
||||
len += snprintf(content + len, maxLen - len, "{\n");
|
||||
len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db);
|
||||
len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion);
|
||||
len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.dbCfgVersion);
|
||||
len += snprintf(content + len, maxLen - len, " \"vgCfgVersion\": %d,\n", pMsg->cfg.vgCfgVersion);
|
||||
len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize);
|
||||
len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks);
|
||||
len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile);
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
|
|||
SVnodeObj *pVnode = vparam;
|
||||
|
||||
// vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
|
||||
// cfgVersion can be corrected by status msg
|
||||
// dbCfgVersion can be corrected by status msg
|
||||
if (!vnodeSetUpdatingStatus(pVnode)) {
|
||||
vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
|||
|
|
@ -89,7 +89,10 @@ static void vnodeIncRef(void *ptNode) {
|
|||
}
|
||||
|
||||
void *vnodeAcquire(int32_t vgId) {
|
||||
SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
|
||||
SVnodeObj **ppVnode = NULL;
|
||||
if (tsVnodesHash != NULL) {
|
||||
ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
|
||||
}
|
||||
|
||||
if (ppVnode == NULL || *ppVnode == NULL) {
|
||||
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
|
||||
|
|
@ -134,7 +137,8 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
|
|||
|
||||
SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
|
||||
pLoad->vgId = htonl(pVnode->vgId);
|
||||
pLoad->cfgVersion = htonl(pVnode->cfgVersion);
|
||||
pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion);
|
||||
pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion);
|
||||
pLoad->totalStorage = htobe64(totalStorage);
|
||||
pLoad->compStorage = htobe64(compStorage);
|
||||
pLoad->pointsWritten = htobe64(pointsWritten);
|
||||
|
|
|
|||
|
|
@ -65,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) {
|
|||
return TSDB_CODE_APP_NOT_READY;
|
||||
}
|
||||
|
||||
if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
|
||||
vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica,
|
||||
syncRole[pVnode->role], pVnode->refCount, pVnode);
|
||||
return TSDB_CODE_APP_NOT_READY;
|
||||
if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica,
|
||||
syncRole[pVnode->role], pVnode->refCount, pVnode);
|
||||
return TSDB_CODE_APP_NOT_READY;
|
||||
}
|
||||
|
||||
void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) {
|
||||
|
|
@ -303,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
|
|||
// NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
|
||||
code = TSDB_CODE_QRY_HAS_RSP;
|
||||
} else {
|
||||
void *h1 = qGetResultRetrieveMsg(*qhandle);
|
||||
assert(h1 == NULL);
|
||||
//void *h1 = qGetResultRetrieveMsg(*qhandle);
|
||||
|
||||
/* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */
|
||||
//assert(h1 == NULL);
|
||||
|
||||
freehandle = qQueryCompleted(*qhandle);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) {
|
|||
if (pWrite->qtype != TAOS_QTYPE_RPC) return 0;
|
||||
if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0;
|
||||
|
||||
if (tsFlowCtrl == 0) {
|
||||
if (tsEnableFlowCtrl == 0) {
|
||||
int32_t ms = pow(2, pVnode->flowctrlLevel + 2);
|
||||
if (ms > 100) ms = 100;
|
||||
vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms);
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ def pre_test(){
|
|||
sh '''
|
||||
cd ${WKC}
|
||||
git reset --hard
|
||||
git checkout ${BRANCH}
|
||||
git checkout $BRANCH_NAME
|
||||
git pull
|
||||
git submodule update
|
||||
cd ${WK}
|
||||
git reset --hard
|
||||
git checkout ${BRANCH}
|
||||
git checkout $BRANCH_NAME
|
||||
git pull
|
||||
export TZ=Asia/Harbin
|
||||
date
|
||||
|
|
@ -28,7 +28,7 @@ def pre_test(){
|
|||
pipeline {
|
||||
agent none
|
||||
environment{
|
||||
BRANCH = $branch_name
|
||||
|
||||
WK = '/var/lib/jenkins/workspace/TDinternal'
|
||||
WKC= '/var/lib/jenkins/workspace/TDinternal/community'
|
||||
}
|
||||
|
|
@ -50,12 +50,7 @@ pipeline {
|
|||
agent{label 'master'}
|
||||
steps {
|
||||
pre_test()
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WKC}/tests/pytest
|
||||
python3 concurrent_inquiry.py -c 1
|
||||
'''
|
||||
}
|
||||
|
||||
sh '''
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh b1
|
||||
|
|
@ -82,7 +77,37 @@ pipeline {
|
|||
./handle_crash_gen_val_log.sh
|
||||
'''
|
||||
}
|
||||
sh'''
|
||||
systemctl start taosd
|
||||
sleep 10
|
||||
'''
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WKC}/tests/gotest
|
||||
bash batchtest.sh
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WKC}/tests/examples/python/PYTHONConnectorChecker
|
||||
python3 PythonChecker.py
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WKC}/tests/examples/JDBC/JDBCDemo/
|
||||
mvn clean package assembly:single -DskipTests >/dev/null
|
||||
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
|
||||
dotnet run
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
systemctl stop taosd
|
||||
cd ${WKC}/tests
|
||||
./test-all.sh b2
|
||||
date
|
||||
|
|
@ -95,6 +120,15 @@ pipeline {
|
|||
|
||||
steps {
|
||||
pre_test()
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WKC}/tests/pytest
|
||||
nohup taosd >/dev/null &
|
||||
sleep 10
|
||||
python3 concurrent_inquiry.py -c 1
|
||||
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
cd ${WKC}/tests/pytest
|
||||
./valgrind-test.sh 2>&1 > mem-error-out.log
|
||||
|
|
@ -106,41 +140,7 @@ pipeline {
|
|||
date'''
|
||||
}
|
||||
}
|
||||
stage('connector'){
|
||||
agent{label "release"}
|
||||
steps{
|
||||
sh'''
|
||||
cd ${WORKSPACE}
|
||||
git checkout develop
|
||||
'''
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WORKSPACE}/tests/gotest
|
||||
bash batchtest.sh
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
|
||||
python3 PythonChecker.py
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
|
||||
mvn clean package assembly:single >/dev/null
|
||||
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
|
||||
'''
|
||||
}
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
|
||||
sh '''
|
||||
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
|
||||
dotnet run
|
||||
'''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
stage('arm64_build'){
|
||||
agent{label 'arm64'}
|
||||
steps{
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ class ConcurrentInquiry:
|
|||
# stableNum = 2,subtableNum = 1000,insertRows = 100):
|
||||
def __init__(self,ts,host,user,password,dbname,
|
||||
stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop,
|
||||
stableNum ,subtableNum ,insertRows ):
|
||||
stableNum ,subtableNum ,insertRows ,mix_table):
|
||||
self.n_numOfTherads = n_Therads
|
||||
self.r_numOfTherads = r_Therads
|
||||
self.ts=ts
|
||||
|
|
@ -60,6 +60,7 @@ class ConcurrentInquiry:
|
|||
self.stableNum = stableNum
|
||||
self.subtableNum = subtableNum
|
||||
self.insertRows = insertRows
|
||||
self.mix_table = mix_table
|
||||
def SetThreadsNum(self,num):
|
||||
self.numOfTherads=num
|
||||
|
||||
|
|
@ -195,7 +196,13 @@ class ConcurrentInquiry:
|
|||
pick_func+=alias
|
||||
sel_col_list.append(pick_func)
|
||||
|
||||
sql=sql+','.join(sel_col_list)+' from '+random.choice(self.stb_list+self.subtb_list)+' ' #select col & func
|
||||
sql=sql+','.join(sel_col_list) #select col & func
|
||||
if self.mix_table == 0:
|
||||
sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' '
|
||||
elif self.mix_table == 1:
|
||||
sql = sql + ' from '+random.choice(self.subtb_list)+' '
|
||||
else:
|
||||
sql = sql + ' from '+random.choice(self.stb_list)+' '
|
||||
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill]
|
||||
sel_con=random.sample(con_func,random.randint(0,len(con_func)))
|
||||
sel_con_list=[]
|
||||
|
|
@ -212,8 +219,23 @@ class ConcurrentInquiry:
|
|||
col_intersection = []
|
||||
tag_intersection = []
|
||||
subtable = None
|
||||
|
||||
if bool(random.getrandbits(1)):
|
||||
if self.mix_table == 0:
|
||||
if bool(random.getrandbits(1)):
|
||||
subtable = True
|
||||
tbname = random.sample(self.subtb_list,2)
|
||||
for i in tbname:
|
||||
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
else:
|
||||
tbname = random.sample(self.stb_list,2)
|
||||
for i in tbname:
|
||||
col_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
elif self.mix_table == 1:
|
||||
subtable = True
|
||||
tbname = random.sample(self.subtb_list,2)
|
||||
for i in tbname:
|
||||
|
|
@ -228,12 +250,9 @@ class ConcurrentInquiry:
|
|||
tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
|
||||
col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
|
||||
tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
|
||||
|
||||
|
||||
con_rand=random.randint(0,len(condition_list))
|
||||
col_rand=random.randint(0,len(col_list))
|
||||
tag_rand=random.randint(0,len(tag_list))
|
||||
|
||||
sql='select ' #select
|
||||
|
||||
sel_col_tag=[]
|
||||
|
|
@ -247,12 +266,16 @@ class ConcurrentInquiry:
|
|||
|
||||
sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func
|
||||
join_section = None
|
||||
temp = None
|
||||
if subtable:
|
||||
join_section = ''.join(random.choices(col_intersection))
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
|
||||
temp = random.choices(col_intersection)
|
||||
join_section = temp.pop()
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
|
||||
else:
|
||||
join_section = ''.join(random.choices(col_intersection+tag_intersection))
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
|
||||
temp = random.choices(col_intersection+tag_intersection)
|
||||
join_section = temp.pop()
|
||||
print(random.choices(col_intersection))
|
||||
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
|
||||
return sql
|
||||
|
||||
def random_pick(self):
|
||||
|
|
@ -365,7 +388,9 @@ class ConcurrentInquiry:
|
|||
print(
|
||||
"Failure thread%d, sql: %s \nexception: %s" %
|
||||
(threadID, str(sql),str(e)))
|
||||
#exit(-1)
|
||||
err_uec='Unable to establish connection'
|
||||
if err_uec in str(e) and loop >0:
|
||||
exit(-1)
|
||||
loop -= 1
|
||||
if loop == 0: break
|
||||
|
||||
|
|
@ -392,7 +417,9 @@ class ConcurrentInquiry:
|
|||
print(
|
||||
"Failure thread%d, sql: %s \nexception: %s" %
|
||||
(threadID, str(sql),str(e)))
|
||||
#exit(-1)
|
||||
err_uec='Unable to establish connection'
|
||||
if err_uec in str(e) and loop >0:
|
||||
exit(-1)
|
||||
loop -= 1
|
||||
if loop == 0: break
|
||||
|
||||
|
|
@ -516,12 +543,20 @@ parser.add_argument(
|
|||
default=2,
|
||||
type=int,
|
||||
help='Number of stables (default: 2)')
|
||||
parser.add_argument(
|
||||
'-m',
|
||||
'--mix-stable-subtable',
|
||||
action='store',
|
||||
default=0,
|
||||
type=int,
|
||||
help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
|
||||
|
||||
args = parser.parse_args()
|
||||
q = ConcurrentInquiry(
|
||||
args.ts,args.host_name,args.user,args.password,args.db_name,
|
||||
args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
|
||||
args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records )
|
||||
args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
|
||||
args.mix_stable_subtable )
|
||||
|
||||
if args.create_table:
|
||||
q.gen_data()
|
||||
|
|
|
|||
|
|
@ -2224,22 +2224,25 @@ class ClientManager:
|
|||
if svcMgr: # gConfig.auto_start_service:
|
||||
svcMgr.stopTaosServices()
|
||||
svcMgr = None
|
||||
|
||||
|
||||
# Release global variables
|
||||
gConfig = None
|
||||
gSvcMgr = None
|
||||
logger = None
|
||||
|
||||
thPool = None
|
||||
dbManager.cleanUp() # destructor wouldn't run in time
|
||||
dbManager = None
|
||||
|
||||
# Print exec status, etc., AFTER showing messages from the server
|
||||
self.conclude()
|
||||
# print("TC failed (2) = {}".format(self.tc.isFailed()))
|
||||
# Linux return code: ref https://shapeshed.com/unix-exit-codes/
|
||||
ret = 1 if self.tc.isFailed() else 0
|
||||
self.tc.cleanup()
|
||||
|
||||
# Release global variables
|
||||
gConfig = None
|
||||
gSvcMgr = None
|
||||
logger = None
|
||||
|
||||
# Release variables here
|
||||
self.tc = None
|
||||
thPool = None
|
||||
dbManager = None
|
||||
|
||||
gc.collect() # force garbage collection
|
||||
# h = hpy()
|
||||
|
|
|
|||
|
|
@ -394,6 +394,7 @@ class DbManager():
|
|||
cType == 'native') else DbConn.createRest(dbTarget)
|
||||
try:
|
||||
self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected
|
||||
Logging.debug("DbManager opened DB connection...")
|
||||
except taos.error.ProgrammingError as err:
|
||||
# print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err))
|
||||
if (err.msg == 'client disconnected'): # cannot open DB connection
|
||||
|
|
@ -412,6 +413,10 @@ class DbManager():
|
|||
# Moved to Database()
|
||||
# self._stateMachine = StateMechine(self._dbConn)
|
||||
|
||||
def __del__(self):
|
||||
''' Release the underlying DB connection upon deletion of DbManager '''
|
||||
self.cleanUp()
|
||||
|
||||
def getDbConn(self):
|
||||
return self._dbConn
|
||||
|
||||
|
|
@ -438,5 +443,8 @@ class DbManager():
|
|||
return "table_{}".format(tblNum)
|
||||
|
||||
def cleanUp(self):
|
||||
self._dbConn.close()
|
||||
if self._dbConn:
|
||||
self._dbConn.close()
|
||||
self._dbConn = None
|
||||
Logging.debug("DbManager closed DB connection...")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import threading
|
|||
import random
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import taos
|
||||
|
||||
|
|
@ -53,7 +54,7 @@ class Logging:
|
|||
# global misc.logger
|
||||
_logger = logging.getLogger('CrashGen') # real logger
|
||||
_logger.addFilter(LoggingFilter())
|
||||
ch = logging.StreamHandler()
|
||||
ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file
|
||||
_logger.addHandler(ch)
|
||||
|
||||
# Logging adapter, to be used as a logger
|
||||
|
|
|
|||
|
|
@ -19,5 +19,5 @@ if __name__ == "__main__":
|
|||
mExec.init()
|
||||
exitCode = mExec.run()
|
||||
|
||||
print("Exiting with code: {}".format(exitCode))
|
||||
print("\nCrash_Gen is now exiting with status code: {}".format(exitCode))
|
||||
sys.exit(exitCode)
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1
|
|||
python3 ./test.py -f functions/function_stddev.py -r 1
|
||||
python3 ./test.py -f functions/function_sum.py -r 1
|
||||
python3 ./test.py -f functions/function_top.py -r 1
|
||||
#python3 ./test.py -f functions/function_twa.py -r 1
|
||||
python3 ./test.py -f functions/function_twa.py -r 1
|
||||
python3 ./test.py -f functions/function_twa_test2.py
|
||||
python3 queryCount.py
|
||||
python3 ./test.py -f query/queryGroupbyWithInterval.py
|
||||
|
|
|
|||
|
|
@ -46,23 +46,17 @@ class TDTestCase:
|
|||
tdSql.error("select twa(ts) from test")
|
||||
tdSql.error("select twa(ts) from test1")
|
||||
|
||||
tdSql.error("select twa(col1) from test")
|
||||
tdSql.error("select twa(col1) from test1")
|
||||
tdSql.error("select twa(col1) from test")
|
||||
|
||||
tdSql.error("select twa(col2) from test")
|
||||
tdSql.error("select twa(col2) from test1")
|
||||
|
||||
tdSql.error("select twa(col3) from test")
|
||||
tdSql.error("select twa(col3) from test1")
|
||||
|
||||
tdSql.error("select twa(col4) from test")
|
||||
tdSql.error("select twa(col4) from test1")
|
||||
tdSql.error("select twa(col4) from test")
|
||||
|
||||
tdSql.error("select twa(col5) from test")
|
||||
tdSql.error("select twa(col5) from test1")
|
||||
|
||||
tdSql.error("select twa(col6) from test")
|
||||
tdSql.error("select twa(col6) from test1")
|
||||
tdSql.error("select twa(col6) from test")
|
||||
|
||||
tdSql.error("select twa(col7) from test")
|
||||
tdSql.error("select twa(col7) from test1")
|
||||
|
|
@ -72,59 +66,23 @@ class TDTestCase:
|
|||
|
||||
tdSql.error("select twa(col9) from test")
|
||||
tdSql.error("select twa(col9) from test1")
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
def stop(self):
|
||||
|
|
|
|||
|
|
@ -28,12 +28,17 @@ class TDTestCase:
|
|||
self.ts = 1537146000000
|
||||
|
||||
def run(self):
|
||||
tdSql.execute("use db")
|
||||
tdSql.prepare()
|
||||
|
||||
intData = []
|
||||
floatData = []
|
||||
|
||||
tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
|
||||
col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
|
||||
tdSql.execute("create table test1 using test tags('beijing')")
|
||||
for i in range(self.rowNum):
|
||||
tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')"
|
||||
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
|
||||
intData.append(i + 1)
|
||||
floatData.append(i + 0.1)
|
||||
|
||||
|
|
@ -41,23 +46,17 @@ class TDTestCase:
|
|||
tdSql.error("select twa(ts) from test")
|
||||
tdSql.error("select twa(ts) from test1")
|
||||
|
||||
tdSql.error("select twa(col1) from test")
|
||||
tdSql.error("select twa(col1) from test1")
|
||||
tdSql.error("select twa(col1) from test")
|
||||
|
||||
tdSql.error("select twa(col2) from test")
|
||||
tdSql.error("select twa(col2) from test1")
|
||||
|
||||
tdSql.error("select twa(col3) from test")
|
||||
tdSql.error("select twa(col3) from test1")
|
||||
|
||||
tdSql.error("select twa(col4) from test")
|
||||
tdSql.error("select twa(col4) from test1")
|
||||
tdSql.error("select twa(col4) from test")
|
||||
|
||||
tdSql.error("select twa(col5) from test")
|
||||
tdSql.error("select twa(col5) from test1")
|
||||
|
||||
tdSql.error("select twa(col6) from test")
|
||||
tdSql.error("select twa(col6) from test1")
|
||||
tdSql.error("select twa(col6) from test")
|
||||
|
||||
tdSql.error("select twa(col7) from test")
|
||||
tdSql.error("select twa(col7) from test1")
|
||||
|
|
@ -67,59 +66,23 @@ class TDTestCase:
|
|||
|
||||
tdSql.error("select twa(col9) from test")
|
||||
tdSql.error("select twa(col9) from test1")
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
|
||||
tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum))
|
||||
|
||||
def stop(self):
|
||||
|
|
@ -127,4 +90,4 @@ class TDTestCase:
|
|||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
|
|
@ -39,6 +39,10 @@ class TDTestCase:
|
|||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 5.5)
|
||||
|
||||
tdSql.query("select twa(c) from t1")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 5.5)
|
||||
|
||||
tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s)")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0, 1, 1.49995)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,10 @@ GREEN='\033[1;32m'
|
|||
GREEN_DARK='\033[0;32m'
|
||||
GREEN_UNDERLINE='\033[4;32m'
|
||||
NC='\033[0m'
|
||||
nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
|
||||
#nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
|
||||
nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null &
|
||||
./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4
|
||||
pidof taosd|xargs kill
|
||||
pidof taosd|xargs kill -9
|
||||
grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
|
||||
|
||||
for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`
|
||||
|
|
@ -31,4 +32,4 @@ if [ -n "$defiMemError" ]; then
|
|||
exit 8
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class TDTestCase:
|
|||
|
||||
tdLog.info("================= step4")
|
||||
tdDnodes.stop(1)
|
||||
tdLog.sleep(5)
|
||||
#tdLog.sleep(5)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdLog.info("================= step5")
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class TDTestCase:
|
|||
|
||||
tdLog.info("================= step4")
|
||||
tdDnodes.stop(1)
|
||||
tdLog.sleep(5)
|
||||
#tdLog.sleep(5)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdLog.info("================= step5")
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class TDTestCase:
|
|||
|
||||
tdLog.info("================= step4")
|
||||
tdDnodes.stop(1)
|
||||
tdLog.sleep(5)
|
||||
#tdLog.sleep(5)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdLog.info("================= step5")
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue