xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #4561 from taosdata/release/s108 

Release/s108
This commit is contained in:
huili 2020-12-14 16:42:08 +08:00 committed by GitHub
parent 3206528eab 5137e8818e
commit 11ff582011
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
257 changed files with 9864 additions and 3708 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -13,7 +13,6 @@ ENDIF ()
SET(TD_ACCOUNT FALSE) SET(TD_ACCOUNT FALSE)
SET(TD_ADMIN FALSE) SET(TD_ADMIN FALSE)
SET(TD_GRANT FALSE) SET(TD_GRANT FALSE)
SET(TD_SYNC TRUE)
SET(TD_MQTT TRUE) SET(TD_MQTT TRUE)
SET(TD_TSDB_PLUGINS FALSE) SET(TD_TSDB_PLUGINS FALSE)

301
Jenkinsfile vendored
View File

@ -1,3 +1,47 @@
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine'
}
// execute this before anything else, including requesting any time on an agent
if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) {
print "INFO: Build skipped due to trigger being Branch Indexing"
currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful
return
}
def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
sudo rmtaos
'''
}
sh '''
cd ${WKC}
rm -rf *
cd ${WK}
git reset --hard
git checkout develop
git pull
cd ${WKC}
rm -rf *
mv ${WORKSPACE}/* .
cd ${WK}
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests
'''
return 1
}
pipeline { pipeline {
agent none agent none
environment{ environment{
@ -8,85 +52,31 @@ pipeline {
stages { stages {
stage('Parallel test stage') { stage('Parallel test stage') {
parallel { parallel {
stage('pytest') { stage('python p1') {
agent{label '184'} agent{label 'p1'}
steps { steps {
pre_test()
sh ''' sh '''
date
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke ./test-all.sh p1
./test-all.sh pytest
date''' date'''
} }
} }
stage('test_b1') { stage('test_b1') {
agent{label 'master'} agent{label 'b1'}
steps { steps {
pre_test()
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh b1 ./test-all.sh b1
date''' date'''
} }
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "185"} agent{label "b2"}
steps { steps {
sh ''' pre_test()
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
@ -109,193 +99,42 @@ pipeline {
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "186"} agent{label "b3"}
steps { steps {
pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
'''
}
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
date date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b3 ./test-all.sh b3
date''' date'''
} }
} }
stage('connector'){ stage('python p2'){
agent{label "release"} agent{label "p2"}
steps{ steps{
sh''' pre_test()
cd ${WORKSPACE} sh '''
git checkout develop date
cd ${WKC}/tests
./test-all.sh p2
date
''' '''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
} }
} }
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
} }
} }
} }
post {
success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
} }

4
cmake/define.inc
View File

@ -13,10 +13,6 @@ IF (TD_GRANT)
ADD_DEFINITIONS(-D_GRANT) ADD_DEFINITIONS(-D_GRANT)
ENDIF () ENDIF ()
IF (TD_SYNC)
ADD_DEFINITIONS(-D_SYNC)
ENDIF ()
IF (TD_MQTT) IF (TD_MQTT)
ADD_DEFINITIONS(-D_MQTT) ADD_DEFINITIONS(-D_MQTT)
ENDIF () ENDIF ()

5
cmake/input.inc
View File

@ -47,11 +47,6 @@ IF (${MQTT} MATCHES "false")
MESSAGE(STATUS "build without mqtt module") MESSAGE(STATUS "build without mqtt module")
ENDIF () ENDIF ()
IF (${SYNC} MATCHES "false")
SET(TD_SYNC FALSE)
MESSAGE(STATUS "build without sync module")
ENDIF ()
IF (${RANDOM_FILE_FAIL} MATCHES "true") IF (${RANDOM_FILE_FAIL} MATCHES "true")
SET(TD_RANDOM_FILE_FAIL TRUE) SET(TD_RANDOM_FILE_FAIL TRUE)
MESSAGE(STATUS "build with random-file-fail enabled") MESSAGE(STATUS "build with random-file-fail enabled")

2
cmake/install.inc
View File

@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .) #INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED) IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.15-dist.jar DESTINATION connector/jdbc)
ENDIF () ENDIF ()
ELSEIF (TD_DARWIN) ELSEIF (TD_DARWIN)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

2
cmake/version.inc
View File

@ -4,7 +4,7 @@ PROJECT(TDengine)
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "2.0.9.0") SET(TD_VER_NUMBER "2.0.10.0")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)

BIN
documentation20/webdocs/assets/connector.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 538 KiB

1
documentation20/webdocs/markdowndocs/Documentation-ch.md
View File

@ -84,6 +84,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
- [数据导出](https://www.taosdata.com/cn/documentation20/administrator/#数据导出)：从shell按表导出，也可用taosdump工具做各种导出 - [数据导出](https://www.taosdata.com/cn/documentation20/administrator/#数据导出)：从shell按表导出，也可用taosdump工具做各种导出
- [系统监控](https://www.taosdata.com/cn/documentation20/administrator/#系统监控)：检查系统现有的连接、查询、流式计算，日志和事件等 - [系统监控](https://www.taosdata.com/cn/documentation20/administrator/#系统监控)：检查系统现有的连接、查询、流式计算，日志和事件等
- [文件目录结构](https://www.taosdata.com/cn/documentation20/administrator/#文件目录结构)：TDengine数据文件、配置文件等所在目录 - [文件目录结构](https://www.taosdata.com/cn/documentation20/administrator/#文件目录结构)：TDengine数据文件、配置文件等所在目录
- [参数限制和保留关键字](https://www.taosdata.com/cn/documentation20/administrator/#参数限制和保留关键字)：TDengine的参数限制和保留关键字列表
## [TAOS SQL](https://www.taosdata.com/cn/documentation20/taos-sql) ## [TAOS SQL](https://www.taosdata.com/cn/documentation20/taos-sql)

51
documentation20/webdocs/markdowndocs/Getting Started-ch.md
View File

@ -2,7 +2,35 @@
## 快捷安装 ## 快捷安装
TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版仅能在Linux系统上安装和运行，后续会支持Windows、MAC OS等系统。如果应用需要在Windows或Mac上运行，目前只能使用TDengine的RESTful接口连接服务器。硬件支持X64，后续会支持ARM、龙芯等CPU系统。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。
**应用驱动**
如果应用在Windows和Linux上运行，可使用C/C++/C#/JAVA/Python/Go/Node.js接口连接服务器。如果应用在Mac上运行，目前可以使用RESTful接口连接服务器。
**CPU**
CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。
**服务器**
目前TDengine服务器可以运行在以下平台上：
| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **Win64/32** | **macOS** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** |
| -------------- | --------------------- | ------------------------ | --------------- | ------------ | --------- | --------------- | ------------------------- | --------------------- |
| X64 | ● | ● | | ○/○ | ○ | ○ | ● | ● |
| 树莓派ARM32 | | ● | ● | | | | | |
| 龙芯MIPS64 | | | ● | | | | | |
| 鲲鹏 ARM64 | | ○ | ○ | | | | ● | |
| 申威 Alpha64 | | | ○ | | | ● | | |
| 飞腾ARM64 | | | ○优麒麟 | | | | | |
| 海光X64 | ● | ● | ● | | | ○ | ● | ● |
| 瑞芯微ARM64/32 | | | ○ | | | | | |
| 全志ARM64/32 | | | ○ | | | | | |
| 炬力ARM64/32 | | | ○ | | | | | |
| TI ARM32 | | | ○ | | | | | |
### 通过源码安装 ### 通过源码安装
@ -16,28 +44,27 @@ TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版
服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。 服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。
<ul id='packageList'> - TDengine-server-2.0.9.0-Linux-x64.rpm (4.2M)
<li><a id='tdengine-rpm' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.rpm (5.3M)</a></li> - TDengine-server-2.0.9.0-Linux-x64.deb (2.7M)
<li><a id='tdengine-deb' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.deb (2.5M)</a></li> - TDengine-server-2.0.9.0-Linux-x64.tar.gz (4.5M)
<li><a id='tdengine-tar' style='color:var(--b2)'>TDengine-server-2.0.0.0-Linux-x64.tar.gz (5.3M)</a></li>
</ul>
客户端部分，Linux安装包如下： 客户端部分，Linux安装包如下：
- TDengine-client-2.0.0.0-Linux-x64.tar.gz (3.4M) - TDengine-client-2.0.9.0-Linux-x64.tar.gz(3.0M)
- TDengine-client-2.0.9.0-Windows-x64.exe(2.8M)
- TDengine-client-2.0.9.0-Windows-x86.exe(2.8M)
报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： 报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）：
- TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M) - TDengine-alert-2.0.9.0-Linux-x64.tar.gz (8.1M)
目前，TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which systemctl`命令来检测系统中是否存在`systemd`包: 目前，TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包:
```cmd ```cmd
which systemctl which systemctl
``` ```
如果系统中不存在`systemd`包，请考虑[通过源码安装](#通过源码安装)TDengine。
具体的安装过程，请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>。 具体的安装过程，请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>。
## 轻松启动 ## 轻松启动
@ -60,6 +87,8 @@ systemctl status taosd
- systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo - systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo
- 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。 - 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。
如果系统中不支持`systemd`，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。
## TDengine命令行程序 ## TDengine命令行程序
执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。

6
documentation20/webdocs/markdowndocs/Model-ch.md
View File

@ -6,12 +6,12 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个
## 创建库 ## 创建库
不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如：
```cmd ```cmd
CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4 UPDATE 1;
``` ```
上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL</a> 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4，允许更新数据。详细的语法及参数请见<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL</a>
创建库之后，需要使用SQL命令USE将当前库切换过来，例如： 创建库之后，需要使用SQL命令USE将当前库切换过来，例如：

35
documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
View File

@ -2,7 +2,7 @@
本文档说明TAOS SQL支持的语法规则、主要查询功能、支持的SQL查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的SQL语言的基础。 本文档说明TAOS SQL支持的语法规则、主要查询功能、支持的SQL查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的SQL语言的基础。
TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手，在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上，TAOS SQL并不是也不试图提供SQL标准的语法。此外，由于TDengine针对的时序性结构化数据不提供修改和更新功能，因此在TAO SQL中不提供数据更新和数据删除的相关功能。 TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手，在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上，TAOS SQL并不是也不试图提供SQL标准的语法。此外，由于TDengine针对的时序性结构化数据不提供删除功能，因此在TAO SQL中不提供数据删除的相关功能。
本章节SQL语法遵循如下约定： 本章节SQL语法遵循如下约定：
@ -57,18 +57,29 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
## 数据库管理 ## 数据库管理
- **创建数据库** - **创建数据库**
```mysql
CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep];
```
说明：
1) `KEEP`是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； ```mysql
2) 数据库名最大长度为33； CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1];
3) 一条SQL 语句的最大长度为65480个字符； ```
4) 数据库还有更多与存储相关的配置参数，请参见[系统管理](../administrator/#服务端配置)。 说明：
1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据；
2) UPDATE 标志数据库支持更新相同时间戳数据；
3) 数据库名最大长度为33；
4) 一条SQL 语句的最大长度为65480个字符；
5) 数据库还有更多与存储相关的配置参数，请参见系统管理。
- **显示系统当前参数**
```mysql
SHOW VARIABLES;
```
- **使用数据库** - **使用数据库**
```mysql ```mysql
USE db_name; USE db_name;
``` ```
@ -143,6 +154,12 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。 显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。
- **在线修改显示字符宽度**
```mysql
SET MAX_BINARY_DISPLAY_WIDTH <nn>;
```
- **获取表的结构信息** - **获取表的结构信息**
```mysql ```mysql

66
documentation20/webdocs/markdowndocs/administrator-ch.md
View File

@ -96,6 +96,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
- maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
- telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 - stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为字节。
**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。
@ -156,6 +157,9 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同
客户端配置参数 客户端配置参数
- firstEp: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。 - firstEp: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。
- secondEp: taos 启动时，如果 firstEp 连不上，将尝试连接 secondEp。
- locale - locale
默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
@ -413,3 +417,65 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
## TDengine参数限制与保留关键字
- 数据库名：不能包含“.”以及特殊字符，不能超过32个字符
- 表名：不能包含“.”以及特殊字符，与所属数据库名一起，不能超过192个字符
- 表的列名：不能包含特殊字符，不能超过64个字符
- 表的列数：不能超过1024列
- 记录的最大长度：包括时间戳8 byte，不能超过16KB
- 单条SQL语句默认最大字符串长度：65480 byte
- 数据库副本数：不能超过3
- 用户名：不能超过20个byte
- 用户密码：不能超过15个byte
- 标签(Tags)数量：不能超过128个
- 标签的总长度：不能超过16Kbyte
- 记录条数：仅受存储空间限制
- 表的个数：仅受节点个数限制
- 库的个数：仅受节点个数限制
- 单个库上虚拟节点个数：不能超过64个
目前TDengine有将近200个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下：
| 关键字列表 | | | | |
| ---------- | ----------- | ------------ | ---------- | --------- |
| ABLOCKS | CONNECTION | GT | MINUS | SHOW |
| ABORT | CONNECTIONS | ID | MNODES | SLASH |
| ACCOUNT | COPY | IF | MODULES | SLIDING |
| ACCOUNTS | COUNT | IGNORE | NCHAR | SMALLINT |
| ADD | CREATE | IMMEDIATE | NE | SPREAD |
| AFTER | CTIME | IMPORT | NONE | STAR |
| ALL | DATABASE | IN | NOT | STATEMENT |
| ALTER | DATABASES | INITIALLY | NOTNULL | STDDEV |
| AND | DAYS | INSERT | NOW | STREAM |
| AS | DEFERRED | INSTEAD | OF | STREAMS |
| ASC | DELIMITERS | INTEGER | OFFSET | STRING |
| ATTACH | DESC | INTERVAL | OR | SUM |
| AVG | DESCRIBE | INTO | ORDER | TABLE |
| BEFORE | DETACH | IP | PASS | TABLES |
| BEGIN | DIFF | IS | PERCENTILE | TAG |
| BETWEEN | DIVIDE | ISNULL | PLUS | TAGS |
| BIGINT | DNODE | JOIN | PRAGMA | TBLOCKS |
| BINARY | DNODES | KEEP | PREV | TBNAME |
| BITAND | DOT | KEY | PRIVILEGE | TIMES |
| BITNOT | DOUBLE | KILL | QUERIES | TIMESTAMP |
| BITOR | DROP | LAST | QUERY | TINYINT |
| BOOL | EACH | LE | RAISE | TOP |
| BOTTOM | END | LEASTSQUARES | REM | TRIGGER |
| BY | EQ | LIKE | REPLACE | UMINUS |
| CACHE | EXISTS | LIMIT | REPLICA | UPLUS |
| CASCADE | EXPLAIN | LINEAR | RESET | USE |
| CHANGE | FAIL | LOCAL | RESTRICT | USER |
| CLOG | FILL | LP | ROW | USERS |
| CLUSTER | FIRST | LSHIFT | ROWS | USING |
| COLON | FLOAT | LT | RP | VALUES |
| COLUMN | FOR | MATCH | RSHIFT | VARIABLE |
| COMMA | FROM | MAX | SCORES | VGROUPS |
| COMP | GE | METRIC | SELECT | VIEW |
| CONCAT | GLOB | METRICS | SEMI | WAVG |
| CONFIGS | GRANTS | MIN | SET | WHERE |
| CONFLICT | GROUP | | | |

77
documentation20/webdocs/markdowndocs/connector-ch.md
View File

@ -1,6 +1,21 @@
# 连接器 # 连接器
TDengine提供了丰富的应用程序开发接口，其中包括C/C++、JAVA、Python、RESTful、Go等，便于用户快速开发应用。 TDengine提供了丰富的应用程序开发接口，其中包括C/C++、C# 、Java、Python、Go、Node.js、RESTful 等，便于用户快速开发应用。
![image-connecotr](../assets/connector.png)
目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下：
| | **CPU** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | | |
| ---------------------------- | --------- | --------------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | --------- | --------- |
| | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** |
| **连** **接** **器** | **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | |
| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | |
| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | |
| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | |
| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | |
| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | |
注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
@ -664,22 +679,44 @@ import (
_ "github.com/taosdata/driver-go/taosSql" _ "github.com/taosdata/driver-go/taosSql"
) )
``` ```
**建议Go版本是1.13或以上，并开启模块支持：**
```
go env -w GO111MODULE=on
go env -w GOPROXY=https://goproxy.io,direct
```
### 常用API ### 常用API
* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB`
该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine - sql.Open(DRIVER_NAME string, dataSourceName string) *DB`
**注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。 该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine
* `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。
- `func (db *DB) Exec(query string, args ...interface{}) (Result, error)`
sql.Open内置的方法，用来执行非查询相关SQL sql.Open内置的方法，用来执行非查询相关SQL
- `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)`
* `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)`
sql.Open内置的方法，用来执行查询语句 sql.Open内置的方法，用来执行查询语句
- `func (db *DB) Prepare(query string) (*Stmt, error)`
sql.Open内置的方法，Prepare creates a prepared statement for later queries or executions.
- `func (s *Stmt) Exec(args ...interface{}) (Result, error)`
sql.Open内置的方法，executes a prepared statement with the given arguments and returns a Result summarizing the effect of the statement.
- `func (s *Stmt) Query(args ...interface{}) (*Rows, error)`
sql.Open内置的方法，Query executes a prepared query statement with the given arguments and returns the query results as a *Rows.
- `func (s *Stmt) Close() error`
sql.Open内置的方法，Close closes the statement.
## Node.js Connector ## Node.js Connector
@ -701,32 +738,6 @@ npm install td2.0-connector
- `make` - `make`
- c语言编译器比如[GCC](https://gcc.gnu.org) - c语言编译器比如[GCC](https://gcc.gnu.org)
### macOS
- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
- Xcode
- 然后通过Xcode安装
```
Command Line Tools
```
在
```
Xcode -> Preferences -> Locations
```
目录下可以找到这个工具。或者在终端里执行
```
xcode-select --install
```
- 该步执行后 `gcc` 和 `make`就被安装上了
### Windows ### Windows
#### 安装方法1 #### 安装方法1

5
documentation20/webdocs/markdowndocs/connector-java-ch.md
View File

@ -1,5 +1,7 @@
# Java Connector # Java Connector
Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。
TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
@ -24,7 +26,8 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一
| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
| --- | --- | --- | | --- | --- | --- |
| 2.0.4 | 2.0.0.x 及以上 | 1.8.x | | 2.0.12 及以上 | 2.0.8.0 及以上 | 1.8.x |
| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x |
| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | | 1.0.2 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | | 1.0.1 | 1.6.1.x 及以上 | 1.8.x |

12
documentation20/webdocs/markdowndocs/faq-ch.md
View File

@ -117,7 +117,17 @@ Connection = DriverManager.getConnection(url, properties);
## 16. 怎么报告问题？ ## 16. 如何进行数据迁移？
TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A移动机器B时，注意如下两件事：
- 2.0.0.0 至 2.0.6.x 的版本，重新配置机器B的hostname为机器A的hostname
- 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。
- 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。
## 17. 怎么报告问题？
如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包:
1. /var/log/taos 1. /var/log/taos

2
documentation20/webdocs/markdowndocs/insert-ch.md
View File

@ -24,7 +24,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6,
- 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为1M）。 - 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为1M）。
- TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。 - TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。
- 对同一张表，如果新插入记录的时间戳已经存在，新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。 - 对同一张表，如果新插入记录的时间戳已经存在，默认（没有使用 UPDATE 1 创建数据库）新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。如果在创建数据库时使用 UPDATE 1 选项，插入相同时间戳的新记录将覆盖原有记录。
- 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。 - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。
## Prometheus直接写入 ## Prometheus直接写入

12
packaging/cfg/taos.cfg
View File

@ -29,8 +29,12 @@
# number of threads per CPU core # number of threads per CPU core
# numOfThreadsPerCore 1.0 # numOfThreadsPerCore 1.0
# the proportion of total threads responsible for query # the proportion of total CPU cores available for query processing
# ratioOfQueryThreads 0.5 # 2.0: the query threads will be set to double of the CPU cores.
# 1.0: all CPU cores are available for query processing [default].
# 0.5: only half of the CPU cores are available for query.
# 0.0: only one core available.
# tsRatioOfQueryCores 1.0
# number of management nodes in the system # number of management nodes in the system
# numOfMnodes 3 # numOfMnodes 3
@ -265,5 +269,5 @@
# enable/disable stream (continuous query) # enable/disable stream (continuous query)
# stream 1 # stream 1
# only 50% CPU resources will be used in query processing # in retrieve blocking model, only in 50% query threads will be used in query processing in dnode
# halfCoresForQuery 0 # retrieveBlockingModel 0

4
snap/snapcraft.yaml
View File

@ -1,6 +1,6 @@
name: tdengine name: tdengine
base: core18 base: core18
version: '2.0.9.0' version: '2.0.10.0'
icon: snap/gui/t-dengine.svg icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT. summary: an open-source big data platform designed and optimized for IoT.
description: | description: |
@ -72,7 +72,7 @@ parts:
- usr/bin/taosd - usr/bin/taosd
- usr/bin/taos - usr/bin/taos
- usr/bin/taosdemo - usr/bin/taosdemo
- usr/lib/libtaos.so.2.0.9.0 - usr/lib/libtaos.so.2.0.10.0
- usr/lib/libtaos.so.1 - usr/lib/libtaos.so.1
- usr/lib/libtaos.so - usr/lib/libtaos.so

4
src/CMakeLists.txt
View File

@ -10,9 +10,7 @@ ADD_SUBDIRECTORY(client)
ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(query)
ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(kit)
ADD_SUBDIRECTORY(plugins) ADD_SUBDIRECTORY(plugins)
IF (TD_SYNC) ADD_SUBDIRECTORY(sync)
ADD_SUBDIRECTORY(sync)
ENDIF ()
ADD_SUBDIRECTORY(balance) ADD_SUBDIRECTORY(balance)
ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(mnode)
ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(vnode)

10
src/balance/src/bnMain.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "tref.h"
#include "tsync.h" #include "tsync.h"
#include "tglobal.h" #include "tglobal.h"
#include "dnode.h" #include "dnode.h"
@ -28,7 +29,9 @@
#include "mnodeUser.h" #include "mnodeUser.h"
#include "mnodeVgroup.h" #include "mnodeVgroup.h"
static SBnMgmt tsBnMgmt;; extern int64_t tsDnodeRid;
extern int64_t tsSdbRid;
static SBnMgmt tsBnMgmt;
static void bnMonitorDnodeModule(); static void bnMonitorDnodeModule();
static void bnLock() { static void bnLock() {
@ -529,6 +532,9 @@ void bnCheckStatus() {
void * pIter = NULL; void * pIter = NULL;
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid);
if (dnodeSdb == NULL) return;
while (1) { while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode); pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break; if (pDnode == NULL) break;
@ -543,6 +549,8 @@ void bnCheckStatus() {
} }
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
} }
taosReleaseRef(tsSdbRid, tsDnodeRid);
} }
void bnCheckModules() { void bnCheckModules() {

2
src/client/inc/tscSubquery.h
View File

@ -36,7 +36,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql);
int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); int32_t tscHandleMultivnodeInsert(SSqlObj *pSql);
int32_t tscHandleInsertRetry(SSqlObj* pSql); int32_t tscHandleInsertRetry(SSqlObj* parent, SSqlObj* child);
void tscBuildResFromSubqueries(SSqlObj *pSql); void tscBuildResFromSubqueries(SSqlObj *pSql);
TAOS_ROW doSetResultRowData(SSqlObj *pSql); TAOS_ROW doSetResultRowData(SSqlObj *pSql);

11
src/client/inc/tscUtil.h
View File

@ -110,11 +110,12 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
uint32_t offset); uint32_t offset);
void* tscDestroyBlockArrayList(SArray* pDataBlockList); void* tscDestroyBlockArrayList(SArray* pDataBlockList);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList); int32_t tscMergeTableDataBlocks(SSqlObj* pSql);
int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size, int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList);
STableDataBlocks** dataBlocks);
/** /**
* for the projection query on metric or point interpolation query on metric, * for the projection query on metric or point interpolation query on metric,
@ -275,6 +276,8 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreVnodesToTry(SSqlObj *pSql);
bool hasMoreClauseToTry(SSqlObj* pSql); bool hasMoreClauseToTry(SSqlObj* pSql);
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache);
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);

90
src/client/inc/tsclient.h
View File

@ -37,40 +37,6 @@ extern "C" {
#include "qTsbuf.h" #include "qTsbuf.h"
#include "tcmdtype.h" #include "tcmdtype.h"
#if 0
static UNUSED_FUNC void *u_malloc (size_t __size) {
uint32_t v = rand();
if (v % 5000 <= 0) {
return NULL;
} else {
return malloc(__size);
}
}
static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) {
uint32_t v = rand();
if (v % 5000 <= 0) {
return NULL;
} else {
return calloc(num, __size);
}
}
static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
uint32_t v = rand();
if (v % 5000 <= 0) {
return NULL;
} else {
return realloc(p, __size);
}
}
#define calloc u_calloc
#define malloc u_malloc
#define realloc u_realloc
#endif
// forward declaration // forward declaration
struct SSqlInfo; struct SSqlInfo;
struct SLocalReducer; struct SLocalReducer;
@ -78,7 +44,7 @@ struct SLocalReducer;
// data source from sql string or from file // data source from sql string or from file
enum { enum {
DATA_FROM_SQL_STRING = 1, DATA_FROM_SQL_STRING = 1,
DATA_FROM_DATA_FILE = 2, DATA_FROM_DATA_FILE = 2,
}; };
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
@ -118,10 +84,10 @@ typedef struct STableMetaInfo {
* 1. keep the vgroup index during the multi-vnode super table projection query * 1. keep the vgroup index during the multi-vnode super table projection query
* 2. keep the vgroup index for multi-vnode insertion * 2. keep the vgroup index for multi-vnode insertion
*/ */
int32_t vgroupIndex; int32_t vgroupIndex;
char name[TSDB_TABLE_FNAME_LEN]; // (super) table name char name[TSDB_TABLE_FNAME_LEN]; // (super) table name
char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql
SArray* tagColList; // SArray<SColumn*>, involved tag columns SArray *tagColList; // SArray<SColumn*>, involved tag columns
} STableMetaInfo; } STableMetaInfo;
/* the structure for sql function in select clause */ /* the structure for sql function in select clause */
@ -136,7 +102,7 @@ typedef struct SSqlExpr {
int16_t numOfParams; // argument value of each function int16_t numOfParams; // argument value of each function
tVariant param[3]; // parameters are not more than 3 tVariant param[3]; // parameters are not more than 3
int32_t offset; // sub result column value of arithmetic expression. int32_t offset; // sub result column value of arithmetic expression.
int16_t resColId; // result column id int16_t resColId; // result column id
} SSqlExpr; } SSqlExpr;
typedef struct SColumnIndex { typedef struct SColumnIndex {
@ -204,22 +170,17 @@ typedef struct SParamInfo {
} SParamInfo; } SParamInfo;
typedef struct STableDataBlocks { typedef struct STableDataBlocks {
char tableId[TSDB_TABLE_FNAME_LEN]; char tableId[TSDB_TABLE_FNAME_LEN];
int8_t tsSource; // where does the UNIX timestamp come from, server or client int8_t tsSource; // where does the UNIX timestamp come from, server or client
bool ordered; // if current rows are ordered or not bool ordered; // if current rows are ordered or not
int64_t vgId; // virtual group id int64_t vgId; // virtual group id
int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending
int32_t numOfTables; // number of tables in current submit block int32_t numOfTables; // number of tables in current submit block
int32_t rowSize; // row size for current table int32_t rowSize; // row size for current table
uint32_t nAllocSize; uint32_t nAllocSize;
uint32_t headerSize; // header for table info (uid, tid, submit metadata) uint32_t headerSize; // header for table info (uid, tid, submit metadata)
uint32_t size; uint32_t size;
STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
/*
* the table meta of table, the table meta will be used during submit, keep a ref
* to avoid it to be removed from cache
*/
STableMeta *pTableMeta;
char *pData; char *pData;
// for parameter ('?') binding // for parameter ('?') binding
@ -252,7 +213,7 @@ typedef struct SQueryInfo {
int64_t clauseLimit; // limit for current sub clause int64_t clauseLimit; // limit for current sub clause
int64_t prjOffset; // offset value in the original sql expression, only applied at client side int64_t prjOffset; // offset value in the original sql expression, only applied at client side
int64_t tableLimit; // table limit in case of super table projection query + global order + limit int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit
int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
int16_t resColumnId; // result column id int16_t resColumnId; // result column id
@ -284,10 +245,14 @@ typedef struct {
int32_t numOfParams; int32_t numOfParams;
int8_t dataSourceType; // load data from file or not int8_t dataSourceType; // load data from file or not
int8_t submitSchema; // submit block is built with table schema int8_t submitSchema; // submit block is built with table schema
STagData *pTagData; // NOTE: pTagData->data is used as a variant length array STagData *pTagData; // NOTE: pTagData->data is used as a variant length array
SHashObj *pTableList; // referred table involved in sql
SArray *pDataBlocks; // SArray<STableDataBlocks*> submit data blocks after parsing sql STableMeta **pTableMetaList; // all involved tableMeta list of current insert sql statement.
int32_t numOfTables;
SHashObj *pTableBlockHashList; // data block for each table
SArray *pDataBlocks; // SArray<STableDataBlocks*>. Merged submit block for each vgroup
} SSqlCmd; } SSqlCmd;
typedef struct SResRec { typedef struct SResRec {
@ -382,6 +347,7 @@ typedef struct SSqlObj {
typedef struct SSqlStream { typedef struct SSqlStream {
SSqlObj *pSql; SSqlObj *pSql;
const char* dstTable;
uint32_t streamId; uint32_t streamId;
char listed; char listed;
bool isProject; bool isProject;
@ -408,6 +374,8 @@ typedef struct SSqlStream {
struct SSqlStream *prev, *next; struct SSqlStream *prev, *next;
} SSqlStream; } SSqlStream;
void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable);
int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn); int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
void tscInitMsgsFp(); void tscInitMsgsFp();

66
src/client/src/tscAsync.c
View File

@ -96,7 +96,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa
return; return;
} }
taosNotePrintTsc(sqlstr); nPrintTsc(sqlstr);
SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
if (pSql == NULL) { if (pSql == NULL) {
@ -365,6 +365,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
static void tscProcessAsyncError(SSchedMsg *pMsg) { static void tscProcessAsyncError(SSchedMsg *pMsg) {
void (*fp)() = pMsg->ahandle; void (*fp)() = pMsg->ahandle;
terrno = *(int32_t*) pMsg->msg; terrno = *(int32_t*) pMsg->msg;
tfree(pMsg->msg);
(*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg); (*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg);
} }
@ -410,52 +411,26 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
goto _error; goto _error;
} else {
tscDebug("%p get %s successfully", pSql, msg);
} }
tscDebug("%p get %s successfully", pSql, msg);
if (pSql->pStream == NULL) { if (pSql->pStream == NULL) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// check if it is a sub-query of super table query first, if true, enter another routine // check if it is a sub-query of super table query first, if true, enter another routine
if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY)) { if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding subquery", pSql); tscDebug("%p update table meta in local cache, continue to process sql and send the corresponding query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo); code = tscGetTableMeta(pSql, pTableMetaInfo);
assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
// param already freed by other routine and pSql in tscCache when ctrl + c
if (atomic_load_ptr(&pSql->param) == NULL) {
return;
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
SSqlObj * pParObj = trs->pParentSql;
// NOTE: the vgroupInfo for the queried super table must be existed here.
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
} else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
} }
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
// tscProcessSql can add error into async res // tscProcessSql can add error into async res
tscProcessSql(pSql); tscProcessSql(pSql);
return; return;
@ -465,16 +440,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo); code = tscGetTableMeta(pSql, pTableMetaInfo);
assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
} else {
assert(code == TSDB_CODE_SUCCESS);
} }
// in case of insert, redo parsing the sql string and build new submit data block for two reasons: assert(pCmd->command != TSDB_SQL_INSERT);
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema. if (pCmd->command == TSDB_SQL_SELECT) {
if (pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) {
tscDebug("%p redo parse sql string and proceed", pSql); tscDebug("%p redo parse sql string and proceed", pSql);
pCmd->parseFinished = false; pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd, false); tscResetSqlCmdObj(pCmd, false);
@ -486,17 +460,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
goto _error; goto _error;
} }
if (pCmd->command == TSDB_SQL_INSERT) { tscProcessSql(pSql);
/* } else { // in all other cases, simple retry
* Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
* and send the required submit block according to index value in supporter to server.
*/
pSql->fp = pSql->fetchFp; // restore the fp
tscHandleInsertRetry(pSql);
} else if (pCmd->command == TSDB_SQL_SELECT) { // in case of other query type, continue
tscProcessSql(pSql);
}
}else { // in all other cases, simple retry
tscProcessSql(pSql); tscProcessSql(pSql);
} }
@ -551,6 +516,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (!pSql->cmd.parseFinished) { if (!pSql->cmd.parseFinished) {
tsParseSql(pSql, false); tsParseSql(pSql, false);
} }
(*pSql->fp)(pSql->param, pSql, code); (*pSql->fp)(pSql->param, pSql, code);
return; return;

146
src/client/src/tscFunctionImpl.c
View File

@ -2589,10 +2589,11 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
// all data are null, set it completed // all data are null, set it completed
if (pInfo->numOfElems == 0) { if (pInfo->numOfElems == 0) {
pResInfo->complete = true; pResInfo->complete = true;
} else {
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
} }
pInfo->stage += 1; pInfo->stage += 1;
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
} else { } else {
pResInfo->complete = true; pResInfo->complete = true;
} }
@ -3647,11 +3648,21 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN; pInfo->p.key = INT64_MIN;
pInfo->win = TSWINDOW_INITIALIZER; pInfo->win = TSWINDOW_INITIALIZER;
return true; return true;
} }
static double twa_get_area(SPoint1 s, SPoint1 e) {
if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) {
return (s.val + e.val) * (e.key - s.key) / 2;
}
double x = (s.key * e.val - e.key * s.val)/(e.val - s.val);
double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2;
return val;
}
static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
int32_t notNullElems = 0; int32_t notNullElems = 0;
TSKEY *primaryKey = pCtx->ptsList; TSKEY *primaryKey = pCtx->ptsList;
@ -3662,28 +3673,29 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
int32_t i = index; int32_t i = index;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
SPoint1* last = &pInfo->p;
if (pCtx->start.key != INT64_MIN) { if (pCtx->start.key != INT64_MIN) {
assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
(pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
assert(pInfo->lastKey == INT64_MIN); assert(last->key == INT64_MIN);
pInfo->lastKey = primaryKey[tsIndex + i]; last->key = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->dOutput += twa_get_area(pCtx->start, *last);
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key; pInfo->win.skey = pCtx->start.key;
notNullElems++; notNullElems++;
i += step; i += step;
} else if (pInfo->lastKey == INT64_MIN) { } else if (pInfo->p.key == INT64_MIN) {
pInfo->lastKey = primaryKey[tsIndex + i]; last->key = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey; pInfo->win.skey = last->key;
notNullElems++; notNullElems++;
i += step; i += step;
} }
@ -3697,9 +3709,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3710,9 +3722,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3723,9 +3735,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3736,9 +3748,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = (double) val[i]};
pInfo->lastValue = (double) val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3749,9 +3761,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3762,9 +3774,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3773,20 +3785,19 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
// the last interpolated time window value // the last interpolated time window value
if (pCtx->end.key != INT64_MIN) { if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);
pInfo->lastValue = pCtx->end.val; pInfo->p = pCtx->end;
pInfo->lastKey = pCtx->end.key;
} }
pInfo->win.ekey = pInfo->lastKey; pInfo->win.ekey = pInfo->p.key;
return notNullElems; return notNullElems;
} }
static void twa_function(SQLFunctionCtx *pCtx) { static void twa_function(SQLFunctionCtx *pCtx) {
void * data = GET_INPUT_CHAR(pCtx); void *data = GET_INPUT_CHAR(pCtx);
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// skip null value // skip null value
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
@ -3807,6 +3818,7 @@ static void twa_function(SQLFunctionCtx *pCtx) {
} }
} }
//TODO refactor
static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
@ -3823,23 +3835,23 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
int32_t size = pCtx->size; int32_t size = pCtx->size;
if (pCtx->start.key != INT64_MIN) { if (pCtx->start.key != INT64_MIN) {
assert(pInfo->lastKey == INT64_MIN); assert(pInfo->p.key == INT64_MIN);
pInfo->lastKey = primaryKey[index]; pInfo->p.key = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->dOutput += twa_get_area(pCtx->start, pInfo->p);
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key; pInfo->win.skey = pCtx->start.key;
notNullElems++; notNullElems++;
i += 1; i += 1;
} else if (pInfo->lastKey == INT64_MIN) { } else if (pInfo->p.key == INT64_MIN) {
pInfo->lastKey = primaryKey[index]; pInfo->p.key = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey; pInfo->win.skey = pInfo->p.key;
notNullElems++; notNullElems++;
i += 1; i += 1;
} }
@ -3853,9 +3865,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3866,9 +3878,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3879,9 +3891,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3892,9 +3904,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = (double) val[i]};
pInfo->lastValue = (double) val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3905,9 +3917,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3918,9 +3930,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3929,12 +3941,11 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// the last interpolated time window value // the last interpolated time window value
if (pCtx->end.key != INT64_MIN) { if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);//((pInfo->p.val + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->p.key);
pInfo->lastValue = pCtx->end.val; pInfo->p = pCtx->end;
pInfo->lastKey = pCtx->end.key;
} }
pInfo->win.ekey = pInfo->lastKey; pInfo->win.ekey = pInfo->p.key;
SET_VAL(pCtx, notNullElems, 1); SET_VAL(pCtx, notNullElems, 1);
@ -3965,7 +3976,7 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
pBuf->dOutput += pInput->dOutput; pBuf->dOutput += pInput->dOutput;
pBuf->win = pInput->win; pBuf->win = pInput->win;
pBuf->lastKey = pInput->lastKey; pBuf->p = pInput->p;
} }
SET_VAL(pCtx, numOfNotNull, 1); SET_VAL(pCtx, numOfNotNull, 1);
@ -3992,15 +4003,14 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->hasResult != DATA_SET_FLAG) { if (pInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
return; return;
} }
assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->win.ekey == pInfo->win.skey) { if (pInfo->win.ekey == pInfo->win.skey) {
*(double *)pCtx->aOutputBuf = pInfo->lastValue; *(double *)pCtx->aOutputBuf = pInfo->p.val;
} else { } else {
*(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey);
} }

10
src/client/src/tscLocalMerge.c
View File

@ -726,10 +726,14 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
SSchema p1 = {0}; SSchema p1 = {0};
if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) { if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) {
p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
} else {
p1 = tGetTableNameColumnSchema(); p1 = tGetTableNameColumnSchema();
} else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) {
p1.bytes = pExpr->resBytes;
p1.type = (uint8_t) pExpr->resType;
tstrncpy(p1.name, pExpr->aliasName, tListLen(p1.name));
} else {
p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
} }
int32_t inter = 0; int32_t inter = 0;

97
src/client/src/tscParseInsert.c
View File

@ -686,17 +686,14 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) {
} }
} }
static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **str, SParsedDataColInfo *spd, static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColInfo *spd, int32_t *totalNum) {
int32_t *totalNum) {
SSqlCmd * pCmd = &pSql->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; STableMeta *pTableMeta = pTableMetaInfo->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta);
STableDataBlocks *dataBuf = NULL; STableDataBlocks *dataBuf = NULL;
int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf, NULL);
pTableMeta, &dataBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return ret; return ret;
} }
@ -1058,18 +1055,17 @@ int tsParseInsertSql(SSqlObj *pSql) {
return code; return code;
} }
if (NULL == pCmd->pTableList) { if (NULL == pCmd->pTableBlockHashList) {
pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); pCmd->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES); if (NULL == pCmd->pTableBlockHashList) {
if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY; code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error; goto _clean;
} }
} else { } else {
str = pCmd->curSql; str = pCmd->curSql;
} }
tscDebug("%p create data block list for submit data:%p, pTableList:%p", pSql, pCmd->pDataBlocks, pCmd->pTableList); tscDebug("%p create data block list hashList:%p", pSql, pCmd->pTableBlockHashList);
while (1) { while (1) {
int32_t index = 0; int32_t index = 0;
@ -1091,7 +1087,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
*/ */
if (totalNum == 0) { if (totalNum == 0) {
code = TSDB_CODE_TSC_INVALID_SQL; code = TSDB_CODE_TSC_INVALID_SQL;
goto _error; goto _clean;
} else { } else {
break; break;
} }
@ -1104,11 +1100,11 @@ int tsParseInsertSql(SSqlObj *pSql) {
// Check if the table name available or not // Check if the table name available or not
if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);
goto _error; goto _clean;
} }
if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) { if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) {
@ -1122,12 +1118,12 @@ int tsParseInsertSql(SSqlObj *pSql) {
tscError("%p async insert parse error, code:%s", pSql, tstrerror(code)); tscError("%p async insert parse error, code:%s", pSql, tstrerror(code));
pCmd->curSql = NULL; pCmd->curSql = NULL;
goto _error; goto _clean;
} }
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL); code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL);
goto _error; goto _clean;
} }
index = 0; index = 0;
@ -1136,7 +1132,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
if (sToken.n == 0) { if (sToken.n == 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z);
goto _error; goto _clean;
} }
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
@ -1148,32 +1144,32 @@ int tsParseInsertSql(SSqlObj *pSql) {
tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns);
if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
/* /*
* app here insert data in different vnodes, so we need to set the following * app here insert data in different vnodes, so we need to set the following
* data in another submit procedure using async insert routines * data in another submit procedure using async insert routines
*/ */
code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); code = doParseInsertStatement(pCmd, &str, &spd, &totalNum);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
} else if (sToken.type == TK_FILE) { } else if (sToken.type == TK_FILE) {
if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) { if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
index = 0; index = 0;
sToken = tStrGetToken(str, &index, false, 0, NULL); sToken = tStrGetToken(str, &index, false, 0, NULL);
if (sToken.type != TK_STRING && sToken.type != TK_ID) { if (sToken.type != TK_STRING && sToken.type != TK_ID) {
code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
goto _error; goto _clean;
} }
str += index; str += index;
if (sToken.n == 0) { if (sToken.n == 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
goto _error; goto _clean;
} }
strncpy(pCmd->payload, sToken.z, sToken.n); strncpy(pCmd->payload, sToken.z, sToken.n);
@ -1183,7 +1179,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
wordexp_t full_path; wordexp_t full_path;
if (wordexp(pCmd->payload, &full_path, 0) != 0) { if (wordexp(pCmd->payload, &full_path, 0) != 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z);
goto _error; goto _clean;
} }
tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize); tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize);
@ -1195,7 +1191,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
SSchema * pSchema = tscGetTableSchema(pTableMeta); SSchema * pSchema = tscGetTableSchema(pTableMeta);
if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
SParsedDataColInfo spd = {0}; SParsedDataColInfo spd = {0};
@ -1230,7 +1226,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
if (spd.hasVal[t] == true) { if (spd.hasVal[t] == true) {
code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z);
goto _error; goto _clean;
} }
spd.hasVal[t] = true; spd.hasVal[t] = true;
@ -1241,13 +1237,13 @@ int tsParseInsertSql(SSqlObj *pSql) {
if (!findColumnIndex) { if (!findColumnIndex) {
code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column name", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column name", sToken.z);
goto _error; goto _clean;
} }
} }
if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) { if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) {
code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z);
goto _error; goto _clean;
} }
index = 0; index = 0;
@ -1256,16 +1252,16 @@ int tsParseInsertSql(SSqlObj *pSql) {
if (sToken.type != TK_VALUES) { if (sToken.type != TK_VALUES) {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z);
goto _error; goto _clean;
} }
code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); code = doParseInsertStatement(pCmd, &str, &spd, &totalNum);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
} else { } else {
code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE are required", sToken.z); code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE are required", sToken.z);
goto _error; goto _clean;
} }
} }
@ -1274,25 +1270,18 @@ int tsParseInsertSql(SSqlObj *pSql) {
goto _clean; goto _clean;
} }
if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { // merge according to vgId if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { if ((code = tscMergeTableDataBlocks(pSql)) != TSDB_CODE_SUCCESS) {
goto _error; goto _clean;
} }
} }
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
goto _clean; goto _clean;
_error:
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
_clean: _clean:
taosHashCleanup(pCmd->pTableList); pCmd->curSql = NULL;
pCmd->pTableList = NULL;
pCmd->curSql = NULL;
pCmd->parseFinished = 1; pCmd->parseFinished = 1;
return code; return code;
} }
@ -1373,6 +1362,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
pSql->parseRetry++; pSql->parseRetry++;
ret = tscToSQLCmd(pSql, &SQLInfo); ret = tscToSQLCmd(pSql, &SQLInfo);
} }
SQLInfoDestroy(&SQLInfo); SQLInfoDestroy(&SQLInfo);
} }
@ -1399,7 +1389,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL); return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL);
} }
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { if ((code = tscMergeTableDataBlocks(pSql)) != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -1456,18 +1446,21 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
int32_t count = 0; int32_t count = 0;
int32_t maxRows = 0; int32_t maxRows = 0;
tscDestroyBlockArrayList(pSql->cmd.pDataBlocks); tfree(pCmd->pTableMetaList);
pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
if (pCmd->pTableBlockHashList == NULL) {
pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
}
STableDataBlocks *pTableDataBlock = NULL; STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE,
sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
// return ret; // return ret;
} }
taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock);
tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows);
char *tokenBuf = calloc(1, 4096); char *tokenBuf = calloc(1, 4096);
while ((readLen = tgetline(&line, &n, fp)) != -1) { while ((readLen = tgetline(&line, &n, fp)) != -1) {
@ -1529,8 +1522,6 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport));
SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES);
pCmd->count = 1; pCmd->count = 1;
FILE *fp = fopen(pCmd->payload, "r"); FILE *fp = fopen(pCmd->payload, "r");

4
src/client/src/tscPrepare.c
View File

@ -800,9 +800,9 @@ static int insertStmtExecute(STscStmt* stmt) {
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
assert(pCmd->numOfClause == 1); assert(pCmd->numOfClause == 1);
if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) {
// merge according to vgid // merge according to vgid
int code = tscMergeTableDataBlocks(stmt->pSql, pCmd->pDataBlocks); int code = tscMergeTableDataBlocks(stmt->pSql);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }

5
src/client/src/tscProfile.c
View File

@ -262,6 +262,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
SSqlStream *pStream = pObj->streamList; SSqlStream *pStream = pObj->streamList;
while (pStream) { while (pStream) {
tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql)); tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
if (pStream->dstTable == NULL) {
pSdesc->dstTable[0] = 0;
} else {
tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable));
}
pSdesc->streamId = htonl(pStream->streamId); pSdesc->streamId = htonl(pStream->streamId);
pSdesc->num = htobe64(pStream->num); pSdesc->num = htobe64(pStream->num);

47
src/client/src/tscSQLParser.c
View File

@ -1310,7 +1310,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
SColumnIndex index = {.tableIndex = tableIndex}; SColumnIndex index = {.tableIndex = tableIndex};
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double),
-1000, sizeof(double), false); getNewResColId(pQueryInfo), sizeof(double), false);
char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z;
size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1);
@ -3282,7 +3282,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC
((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0);
if (pColFilter->filterstr) { if (pColFilter->filterstr) {
if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { if (pExpr->nSQLOptr != TK_EQ
&& pExpr->nSQLOptr != TK_NE
&& pExpr->nSQLOptr != TK_ISNULL
&& pExpr->nSQLOptr != TK_NOTNULL
&& pExpr->nSQLOptr != TK_LIKE
) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
} else { } else {
@ -5312,7 +5317,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
// keep original limitation value in globalLimit // keep original limitation value in globalLimit
pQueryInfo->clauseLimit = pQueryInfo->limit.limit; pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
pQueryInfo->prjOffset = pQueryInfo->limit.offset; pQueryInfo->prjOffset = pQueryInfo->limit.offset;
pQueryInfo->tableLimit = -1; pQueryInfo->vgroupLimit = -1;
if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
/* /*
@ -5322,7 +5327,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
* than or equal to the value of limit. * than or equal to the value of limit.
*/ */
if (pQueryInfo->limit.limit > 0) { if (pQueryInfo->limit.limit > 0) {
pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; pQueryInfo->vgroupLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset;
pQueryInfo->limit.limit = -1; pQueryInfo->limit.limit = -1;
} }
@ -5908,25 +5913,33 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
if (pExprList->nExpr != 1) { if (pExprList->nExpr != 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }
bool server_status = false;
tSQLExpr* pExpr = pExprList->a[0].pNode; tSQLExpr* pExpr = pExprList->a[0].pNode;
if (pExpr->operand.z == NULL) { if (pExpr->operand.z == NULL) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); //handle 'select 1'
if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) {
server_status = true;
} else {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
} }
// TODO redefine the function // TODO redefine the function
SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, SDNodeDynConfOption functionsInfo[5] = {{"database()", 10},
{"server_version()", 16}, {"server_version()", 16},
{"server_status()", 15}, {"server_status()", 15},
{"client_version()", 16}, {"client_version()", 16},
{"current_user()", 14}}; {"current_user()", 14}};
int32_t index = -1; int32_t index = -1;
for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { if (server_status == true) {
if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && index = 2;
functionsInfo[i].len == pExpr->operand.n) { } else {
index = i; for (int32_t i = 0; i < tListLen(functionsInfo); ++i) {
break; if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 &&
functionsInfo[i].len == pExpr->operand.n) {
index = i;
break;
}
} }
} }

67
src/client/src/tscServer.c
View File

@ -152,7 +152,13 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
SRpcEpSet * epSet = &pRsp->epSet; SRpcEpSet * epSet = &pRsp->epSet;
if (epSet->numOfEps > 0) { if (epSet->numOfEps > 0) {
tscEpSetHtons(epSet); tscEpSetHtons(epSet);
tscUpdateMgmtEpSet(pSql, epSet); if (!tscEpSetIsEqual(&pSql->pTscObj->tscCorMgmtEpSet->epSet, epSet)) {
tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse);
for (int8_t i = 0; i < epSet->numOfEps; i++) {
tscTrace("endpoint %d: fqdn = %s, port=%d", i, epSet->fqdn[i], epSet->port[i]);
}
tscUpdateMgmtEpSet(pSql, epSet);
}
} }
pSql->pTscObj->connId = htonl(pRsp->connId); pSql->pTscObj->connId = htonl(pRsp->connId);
@ -280,19 +286,19 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
} }
int32_t cmd = pCmd->command; int32_t cmd = pCmd->command;
if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) &&
// set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
pSql->cmd.submitSchema = 1;
}
if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_UPDATE_TAGS_VAL) &&
(rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID ||
rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID ||
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL ||
rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) {
rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) {
tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
// set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
pSql->cmd.submitSchema = 1;
}
pSql->res.code = rpcMsg->code; // keep the previous error code pSql->res.code = rpcMsg->code; // keep the previous error code
if (pSql->retry > pSql->maxRetry) { if (pSql->retry > pSql->maxRetry) {
tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
@ -451,10 +457,10 @@ int tscProcessSql(SSqlObj *pSql) {
int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
pRetrieveMsg->free = htons(pQueryInfo->type); pRetrieveMsg->free = htons(pQueryInfo->type);
pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
// todo valid the vgroupId at the client side // todo valid the vgroupId at the client side
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -681,7 +687,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->queryType = htonl(pQueryInfo->type); pQueryMsg->queryType = htonl(pQueryInfo->type);
pQueryMsg->tableLimit = htobe64(pQueryInfo->tableLimit); pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit);
size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number
@ -1394,6 +1400,43 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
//int tscBuildCancelQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// SCancelQueryMsg *pCancelMsg = (SCancelQueryMsg*) pSql->cmd.payload;
// pCancelMsg->qhandle = htobe64(pSql->res.qhandle);
//
// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
//
// if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
// int32_t vgIndex = pTableMetaInfo->vgroupIndex;
// if (pTableMetaInfo->pVgroupTables == NULL) {
// SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList;
// assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
//
// pCancelMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex);
// } else {
// int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
// assert(vgIndex >= 0 && vgIndex < numOfVgroups);
//
// SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex);
//
// pCancelMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId);
// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex);
// }
// } else {
// STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
// pCancelMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
// tscDebug("%p build cancel query msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId);
// }
//
// pSql->cmd.payloadLen = sizeof(SCancelQueryMsg);
// pSql->cmd.msgType = TSDB_MSG_TYPE_CANCEL_QUERY;
//
// pCancelMsg->header.contLen = htonl(sizeof(SCancelQueryMsg));
// return TSDB_CODE_SUCCESS;
//}
int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SAlterDbMsg); pCmd->payloadLen = sizeof(SAlterDbMsg);

8
src/client/src/tscSql.c
View File

@ -344,7 +344,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES
return NULL; return NULL;
} }
taosNotePrintTsc(sqlstr); nPrintTsc(sqlstr);
SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) { if (pSql == NULL) {
@ -900,9 +900,9 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
strtolower(pSql->sqlstr, sql); strtolower(pSql->sqlstr, sql);
pCmd->curSql = NULL; pCmd->curSql = NULL;
if (NULL != pCmd->pTableList) { if (NULL != pCmd->pTableBlockHashList) {
taosHashCleanup(pCmd->pTableList); taosHashCleanup(pCmd->pTableBlockHashList);
pCmd->pTableList = NULL; pCmd->pTableBlockHashList = NULL;
} }
pSql->fp = asyncCallback; pSql->fp = asyncCallback;

4
src/client/src/tscStream.c
View File

@ -535,6 +535,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) {
pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
} }
void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) {
pStream->dstTable = dstTable;
}
TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
int64_t stime, void *param, void (*callback)(void *)) { int64_t stime, void *param, void (*callback)(void *)) {
STscObj *pObj = (STscObj *)taos; STscObj *pObj = (STscObj *)taos;

128
src/client/src/tscSubquery.c
View File

@ -2149,6 +2149,38 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
} }
} }
static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) {
if (pParentObj->retry > pParentObj->maxRetry) {
tscError("%p max retry reached, abort the retry effort", pParentObj)
return false;
}
for (int32_t i = 0; i < numOfSub; ++i) {
int32_t code = pParentObj->pSubs[i]->res.code;
if (code == TSDB_CODE_SUCCESS) {
continue;
}
if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE && code != TSDB_CODE_TDB_INVALID_TABLE_ID &&
code != TSDB_CODE_VND_INVALID_VGROUP_ID && code != TSDB_CODE_RPC_NETWORK_UNAVAIL &&
code != TSDB_CODE_APP_NOT_READY) {
pParentObj->res.code = code;
return false;
}
}
return true;
}
static void doFreeInsertSupporter(SSqlObj* pSqlObj) {
assert(pSqlObj != NULL && pSqlObj->subState.numOfSub > 0);
for(int32_t i = 0; i < pSqlObj->subState.numOfSub; ++i) {
SSqlObj* pSql = pSqlObj->pSubs[i];
tfree(pSql->param);
}
}
static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) { static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
SInsertSupporter *pSupporter = (SInsertSupporter *)param; SInsertSupporter *pSupporter = (SInsertSupporter *)param;
SSqlObj* pParentObj = pSupporter->pSql; SSqlObj* pParentObj = pSupporter->pSql;
@ -2163,23 +2195,81 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
assert(pSql != NULL && pSql->res.code == numOfRows); assert(pSql != NULL && pSql->res.code == numOfRows);
pParentObj->res.code = pSql->res.code; pParentObj->res.code = pSql->res.code;
}
tfree(pSupporter); // set the flag in the parent sqlObj
if (pSql->cmd.submitSchema) {
pParentObj->cmd.submitSchema = 1;
}
}
if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) { if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
return; return;
} }
tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows);
// restore user defined fp // restore user defined fp
pParentObj->fp = pParentObj->fetchFp; pParentObj->fp = pParentObj->fetchFp;
int32_t numOfSub = pParentObj->subState.numOfSub;
// todo remove this parameter in async callback function definition. if (pParentObj->res.code == TSDB_CODE_SUCCESS) {
// all data has been sent to vnode, call user function tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows);
int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows; doFreeInsertSupporter(pParentObj);
(*pParentObj->fp)(pParentObj->param, pParentObj, v);
// todo remove this parameter in async callback function definition.
// all data has been sent to vnode, call user function
int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows;
(*pParentObj->fp)(pParentObj->param, pParentObj, v);
} else {
if (!needRetryInsert(pParentObj, numOfSub)) {
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj);
return;
}
int32_t numOfFailed = 0;
for(int32_t i = 0; i < numOfSub; ++i) {
SSqlObj* pSql = pParentObj->pSubs[i];
if (pSql->res.code != TSDB_CODE_SUCCESS) {
numOfFailed += 1;
// clean up tableMeta in cache
tscFreeQueryInfo(&pSql->cmd, true);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0);
STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0);
tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL);
tscDebug("%p, failed sub:%d, %p", pParentObj, i, pSql);
}
}
tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj,
pParentObj->res.numOfRows, numOfFailed, numOfSub);
tscDebug("%p cleanup %d tableMeta in cache", pParentObj, pParentObj->cmd.numOfTables);
for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) {
taosCacheRelease(tscMetaCache, (void**)&(pParentObj->cmd.pTableMetaList[i]), true);
}
pParentObj->cmd.parseFinished = false;
pParentObj->subState.numOfRemain = numOfFailed;
tscResetSqlCmdObj(&pParentObj->cmd, false);
// in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema.
tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry++);
int32_t code = tsParseSql(pParentObj, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
if (code != TSDB_CODE_SUCCESS) {
pParentObj->res.code = code;
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj);
return;
}
tscDoQuery(pParentObj);
}
} }
/** /**
@ -2187,19 +2277,19 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
* @param pSql * @param pSql
* @return * @return
*/ */
int32_t tscHandleInsertRetry(SSqlObj* pSql) { int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) {
assert(pSql != NULL && pSql->param != NULL); assert(pSql != NULL && pSql->param != NULL);
SSqlCmd* pCmd = &pSql->cmd; // SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param;
assert(pSupporter->index < pSupporter->pSql->subState.numOfSub); assert(pSupporter->index < pSupporter->pSql->subState.numOfSub);
STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.pDataBlocks, pSupporter->index);
int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
// free the data block created from insert sql string // free the data block created from insert sql string
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); // pCmd->pDataBlocks = tscDestroyBlockArrayList(pParent->cmd.pDataBlocks);
if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql); tscQueueAsyncRes(pSql);
@ -2213,6 +2303,20 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
// it is the failure retry insert
if (pSql->pSubs != NULL) {
for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
SSqlObj* pSub = pSql->pSubs[i];
tscDebug("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, i);
if (pSub->res.code != TSDB_CODE_SUCCESS) {
tscHandleInsertRetry(pSql, pSub);
}
}
return TSDB_CODE_SUCCESS;
}
pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks); pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks);
assert(pSql->subState.numOfSub > 0); assert(pSql->subState.numOfSub > 0);

9
src/client/src/tscSystem.c
View File

@ -17,6 +17,7 @@
#include "taosmsg.h" #include "taosmsg.h"
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "tnote.h"
#include "tsystem.h" #include "tsystem.h"
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h" #include "tutil.h"
@ -41,7 +42,6 @@ int tscRefId = -1;
int tscNumOfThreads; int tscNumOfThreads;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT; static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); //void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
@ -78,7 +78,6 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon
return 0; return 0;
} }
void taos_init_imp(void) { void taos_init_imp(void) {
char temp[128] = {0}; char temp[128] = {0};
@ -104,6 +103,7 @@ void taos_init_imp(void) {
taosReadGlobalCfg(); taosReadGlobalCfg();
taosCheckGlobalCfg(); taosCheckGlobalCfg();
taosInitNotes();
rpcInit(); rpcInit();
tscDebug("starting to initialize TAOS client ..."); tscDebug("starting to initialize TAOS client ...");
@ -111,11 +111,6 @@ void taos_init_imp(void) {
} }
taosSetCoreDump(); taosSetCoreDump();
if (tsTscEnableRecordSql != 0) {
taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
}
tscInitMsgsFp(); tscInitMsgsFp();
int queueSize = tsMaxConnections*2; int queueSize = tsMaxConnections*2;

87
src/client/src/tscUtil.c
View File

@ -333,13 +333,15 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { if (isNull(p, TSDB_DATA_TYPE_NCHAR)) {
memcpy(dst, p, varDataTLen(p)); memcpy(dst, p, varDataTLen(p));
} else { } else if (varDataLen(p) > 0) {
int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst));
varDataSetLen(dst, length); varDataSetLen(dst, length);
if (length == 0) { if (length == 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p);
} }
} else {
varDataSetLen(dst, 0);
} }
p += pInfo->field.bytes; p += pInfo->field.bytes;
@ -377,7 +379,7 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) {
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
} }
static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
if (pCmd == NULL || pCmd->numOfClause == 0) { if (pCmd == NULL || pCmd->numOfClause == 0) {
return; return;
} }
@ -404,11 +406,17 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
pCmd->parseFinished = 0; pCmd->parseFinished = 0;
pCmd->autoCreated = 0; pCmd->autoCreated = 0;
taosHashCleanup(pCmd->pTableList); for(int32_t i = 0; i < pCmd->numOfTables; ++i) {
pCmd->pTableList = NULL; if (pCmd->pTableMetaList && pCmd->pTableMetaList[i]) {
taosCacheRelease(tscMetaCache, (void**)&(pCmd->pTableMetaList[i]), false);
}
}
pCmd->numOfTables = 0;
tfree(pCmd->pTableMetaList);
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList);
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
tscFreeQueryInfo(pCmd, removeFromCache); tscFreeQueryInfo(pCmd, removeFromCache);
} }
@ -575,6 +583,21 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) {
return NULL; return NULL;
} }
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable) {
if (pBlockHashTable == NULL) {
return NULL;
}
STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL);
while(p) {
tscDestroyDataBlock(*p);
p = taosHashIterate(pBlockHashTable, p);
}
taosHashCleanup(pBlockHashTable);
return NULL;
}
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
assert(pDataBlock->pTableMeta != NULL); assert(pDataBlock->pTableMeta != NULL);
@ -671,9 +694,8 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size, int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) {
STableDataBlocks** dataBlocks) {
*dataBlocks = NULL; *dataBlocks = NULL;
STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id));
@ -688,7 +710,9 @@ int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t
} }
taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES);
taosArrayPush(pDataBlockList, dataBlocks); if (pBlockList) {
taosArrayPush(pBlockList, dataBlocks);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -769,22 +793,37 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) {
return result; return result;
} }
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { static void extractTableMeta(SSqlCmd* pCmd) {
pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList);
pCmd->pTableMetaList = calloc(pCmd->numOfTables, POINTER_BYTES);
STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL);
int32_t i = 0;
while(p1) {
STableDataBlocks* pBlocks = *p1;
pCmd->pTableMetaList[i++] = taosCacheTransfer(tscMetaCache, (void**) &pBlocks->pTableMeta);
p1 = taosHashIterate(pCmd->pTableBlockHashList, p1);
}
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList);
}
int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
size_t total = taosArrayGetSize(pTableDataBlockList); STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL);
for (int32_t i = 0; i < total; ++i) {
STableDataBlocks* pOneTableBlock = *p;
while(pOneTableBlock) {
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format // the maximum expanded size in byte when a row-wise data is converted to SDataRow format
STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i);
int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta);
STableDataBlocks* dataBuf = NULL; STableDataBlocks* dataBuf = NULL;
int32_t ret = int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
@ -839,14 +878,19 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
// the length does not include the SSubmitBlk structure // the length does not include the SSubmitBlk structure
pBlocks->dataLen = htonl(finalLen); pBlocks->dataLen = htonl(finalLen);
dataBuf->numOfTables += 1; dataBuf->numOfTables += 1;
p = taosHashIterate(pCmd->pTableBlockHashList, p);
if (p == NULL) {
break;
}
pOneTableBlock = *p;
} }
tscDestroyBlockArrayList(pTableDataBlockList); extractTableMeta(pCmd);
// free the table data blocks; // free the table data blocks;
pCmd->pDataBlocks = pVnodeDataBlockList; pCmd->pDataBlocks = pVnodeDataBlockList;
// tscFreeUnusedDataBlocks(pCmd->pDataBlocks);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2006,7 +2050,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pnCmd->numOfClause = 0; pnCmd->numOfClause = 0;
pnCmd->clauseIndex = 0; pnCmd->clauseIndex = 0;
pnCmd->pDataBlocks = NULL; pnCmd->pDataBlocks = NULL;
pnCmd->numOfTables = 0;
pnCmd->parseFinished = 1; pnCmd->parseFinished = 1;
pnCmd->pTableMetaList = NULL;
pnCmd->pTableBlockHashList = NULL;
if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -2023,6 +2071,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pNewQueryInfo->limit = pQueryInfo->limit; pNewQueryInfo->limit = pQueryInfo->limit;
pNewQueryInfo->slimit = pQueryInfo->slimit; pNewQueryInfo->slimit = pQueryInfo->slimit;
pNewQueryInfo->order = pQueryInfo->order; pNewQueryInfo->order = pQueryInfo->order;
pNewQueryInfo->vgroupLimit = pQueryInfo->vgroupLimit;
pNewQueryInfo->tsBuf = NULL; pNewQueryInfo->tsBuf = NULL;
pNewQueryInfo->fillType = pQueryInfo->fillType; pNewQueryInfo->fillType = pQueryInfo->fillType;
pNewQueryInfo->fillVal = NULL; pNewQueryInfo->fillVal = NULL;

5
src/common/inc/tglobal.h
View File

@ -35,6 +35,7 @@ extern int32_t tsNumOfMnodes;
extern int32_t tsEnableVnodeBak; extern int32_t tsEnableVnodeBak;
extern int32_t tsEnableTelemetryReporting; extern int32_t tsEnableTelemetryReporting;
extern char tsEmail[]; extern char tsEmail[];
extern char tsArbitrator[];
// common // common
extern int tsRpcTimer; extern int tsRpcTimer;
@ -45,7 +46,7 @@ extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl; extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore; extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryThreads; // todo remove it extern float tsRatioOfQueryCores;
extern int8_t tsDaylight; extern int8_t tsDaylight;
extern char tsTimezone[]; extern char tsTimezone[];
extern char tsLocale[]; extern char tsLocale[];
@ -56,7 +57,7 @@ extern char tsTempDir[];
//query buffer management //query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing
extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing extern int32_t tsRetrieveBlockingModel; // only 50% will be used in query processing
// client // client
extern int32_t tsTableMetaKeepTimer; extern int32_t tsTableMetaKeepTimer;

2
src/common/inc/tname.h
View File

@ -25,6 +25,8 @@ void extractTableName(const char *tableId, char *name);
char* extractDBName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name);
size_t tableIdPrefix(const char* name, char* prefix, int32_t len);
void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
SSchema tGetTableNameColumnSchema(); SSchema tGetTableNameColumnSchema();

32
src/common/src/tglobal.c
View File

@ -52,7 +52,7 @@ int32_t tsMaxConnections = 5000;
int32_t tsShellActivityTimer = 3; // second int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f; float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 1; int32_t tsNumOfCommitThreads = 1;
float tsRatioOfQueryThreads = 0.5f; float tsRatioOfQueryCores = 1.0f;
int8_t tsDaylight = 0; int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0};
@ -107,8 +107,8 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
// positive value (in MB) // positive value (in MB)
int32_t tsQueryBufferSize = -1; int32_t tsQueryBufferSize = -1;
// only 50% cpu will be used in query processing in dnode // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
int32_t tsHalfCoresForQuery = 0; int32_t tsRetrieveBlockingModel = 0;
// db parameters // db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
@ -203,8 +203,8 @@ int32_t tsVersion = 0;
// log // log
int32_t tsNumOfLogLines = 10000000; int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 135; int32_t mDebugFlag = 131;
int32_t sdbDebugFlag = 135; int32_t sdbDebugFlag = 131;
int32_t dDebugFlag = 135; int32_t dDebugFlag = 135;
int32_t vDebugFlag = 135; int32_t vDebugFlag = 135;
int32_t cDebugFlag = 131; int32_t cDebugFlag = 131;
@ -220,7 +220,7 @@ int32_t debugFlag = 0;
int32_t sDebugFlag = 135; int32_t sDebugFlag = 135;
int32_t wDebugFlag = 135; int32_t wDebugFlag = 135;
int32_t tsdbDebugFlag = 131; int32_t tsdbDebugFlag = 131;
int32_t cqDebugFlag = 135; int32_t cqDebugFlag = 131;
int32_t (*monStartSystemFp)() = NULL; int32_t (*monStartSystemFp)() = NULL;
void (*monStopSystemFp)() = NULL; void (*monStopSystemFp)() = NULL;
@ -416,7 +416,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "arbitrator"; cfg.option = "arbitrator";
cfg.ptr = tsArbitrator; cfg.ptr = tsArbitrator;
cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = TSDB_EP_LEN; cfg.ptrLength = TSDB_EP_LEN;
@ -444,12 +444,12 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "ratioOfQueryThreads"; cfg.option = "ratioOfQueryCores";
cfg.ptr = &tsRatioOfQueryThreads; cfg.ptr = &tsRatioOfQueryCores;
cfg.valType = TAOS_CFG_VTYPE_FLOAT; cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0.1f; cfg.minValue = 0.0f;
cfg.maxValue = 0.9f; cfg.maxValue = 2.0f;
cfg.ptrLength = 0; cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
@ -887,8 +887,8 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_BYTE; cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "halfCoresForQuery"; cfg.option = "retrieveBlockingModel";
cfg.ptr = &tsHalfCoresForQuery; cfg.ptr = &tsRetrieveBlockingModel;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0; cfg.minValue = 0;
@ -901,7 +901,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "timezone"; cfg.option = "timezone";
cfg.ptr = tsTimezone; cfg.ptr = tsTimezone;
cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = tListLen(tsTimezone); cfg.ptrLength = tListLen(tsTimezone);
@ -911,7 +911,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "locale"; cfg.option = "locale";
cfg.ptr = tsLocale; cfg.ptr = tsLocale;
cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = tListLen(tsLocale); cfg.ptrLength = tListLen(tsLocale);
@ -921,7 +921,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "charset"; cfg.option = "charset";
cfg.ptr = tsCharset; cfg.ptr = tsCharset;
cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0; cfg.minValue = 0;
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = tListLen(tsCharset); cfg.ptrLength = tListLen(tsCharset);

7
src/common/src/tname.c
View File

@ -39,6 +39,13 @@ char* extractDBName(const char* tableId, char* name) {
return strncpy(name, &tableId[offset1 + 1], len); return strncpy(name, &tableId[offset1 + 1], len);
} }
size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
tstrncpy(prefix, name, len);
strcat(prefix, TS_PATH_DELIMITER);
return strlen(prefix);
}
SSchema tGetTableNameColumnSchema() { SSchema tGetTableNameColumnSchema() {
SSchema s = {0}; SSchema s = {0};
s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE;

2
src/connector/jdbc/CMakeLists.txt
View File

@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
POST_BUILD POST_BUILD
COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.15-dist.jar ${LIBRARY_OUTPUT_PATH}
COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMENT "build jdbc driver") COMMENT "build jdbc driver")
ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

3
src/connector/jdbc/deploy-pom.xml
View File

@ -5,7 +5,7 @@
<groupId>com.taosdata.jdbc</groupId> <groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId> <artifactId>taos-jdbcdriver</artifactId>
<version>2.0.14</version> <version>2.0.15</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>JDBCDriver</name> <name>JDBCDriver</name>
@ -36,7 +36,6 @@
</developer> </developer>
</developers> </developers>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>commons-logging</groupId> <groupId>commons-logging</groupId>

2
src/connector/jdbc/pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.taosdata.jdbc</groupId> <groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId> <artifactId>taos-jdbcdriver</artifactId>
<version>2.0.14</version> <version>2.0.15</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>JDBCDriver</name> <name>JDBCDriver</name>
<url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url> <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>

8
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
View File

@ -81,7 +81,7 @@ public class TSDBStatement implements Statement {
} }
if (!this.connector.isUpdateQuery(pSql)) { if (!this.connector.isUpdateQuery(pSql)) {
TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer); TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer);
res.setBatchFetch(this.connection.getBatchFetch()); res.setBatchFetch(this.connection.getBatchFetch());
return res; return res;
} else { } else {
@ -125,7 +125,8 @@ public class TSDBStatement implements Statement {
} }
public int getMaxFieldSize() throws SQLException { public int getMaxFieldSize() throws SQLException {
throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); return 0;
// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
} }
public void setMaxFieldSize(int max) throws SQLException { public void setMaxFieldSize(int max) throws SQLException {
@ -218,7 +219,8 @@ public class TSDBStatement implements Statement {
} }
public int getFetchDirection() throws SQLException { public int getFetchDirection() throws SQLException {
throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); return ResultSet.FETCH_FORWARD;
// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
} }
/* /*

15
src/cq/src/cqMain.c
View File

@ -57,6 +57,7 @@ typedef struct SCqObj {
uint64_t uid; uint64_t uid;
int32_t tid; // table ID int32_t tid; // table ID
int32_t rowSize; // bytes of a row int32_t rowSize; // bytes of a row
char * dstTable;
char * sqlStr; // SQL string char * sqlStr; // SQL string
STSchema * pSchema; // pointer to schema array STSchema * pSchema; // pointer to schema array
void * pStream; void * pStream;
@ -160,7 +161,7 @@ void cqStop(void *handle) {
return; return;
} }
SCqContext *pContext = handle; SCqContext *pContext = handle;
cInfo("vgId:%d, stop all CQs", pContext->vgId); cDebug("vgId:%d, stop all CQs", pContext->vgId);
if (pContext->dbConn == NULL || pContext->master == 0) return; if (pContext->dbConn == NULL || pContext->master == 0) return;
pthread_mutex_lock(&pContext->mutex); pthread_mutex_lock(&pContext->mutex);
@ -185,7 +186,7 @@ void cqStop(void *handle) {
pthread_mutex_unlock(&pContext->mutex); pthread_mutex_unlock(&pContext->mutex);
} }
void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema) {
if (tsEnableStream == 0) { if (tsEnableStream == 0) {
return NULL; return NULL;
} }
@ -195,9 +196,11 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
if (pObj == NULL) return NULL; if (pObj == NULL) return NULL;
pObj->uid = uid; pObj->uid = uid;
pObj->tid = tid; pObj->tid = sid;
pObj->sqlStr = malloc(strlen(sqlStr)+1); if (dstTable != NULL) {
strcpy(pObj->sqlStr, sqlStr); pObj->dstTable = strdup(dstTable);
}
pObj->sqlStr = strdup(sqlStr);
pObj->pSchema = tdDupSchema(pSchema); pObj->pSchema = tdDupSchema(pSchema);
pObj->rowSize = schemaTLen(pSchema); pObj->rowSize = schemaTLen(pSchema);
@ -247,6 +250,7 @@ void cqDrop(void *handle) {
cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr);
tdFreeSchema(pObj->pSchema); tdFreeSchema(pObj->pSchema);
free(pObj->dstTable);
free(pObj->sqlStr); free(pObj->sqlStr);
free(pObj); free(pObj);
@ -292,6 +296,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
if (pObj->pStream == NULL) { if (pObj->pStream == NULL) {
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
if (pObj->pStream) { if (pObj->pStream) {
tscSetStreamDestTable(pObj->pStream, pObj->dstTable);
pContext->num++; pContext->num++;
cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else { } else {

2
src/cq/test/cqtest.c
View File

@ -70,7 +70,7 @@ int main(int argc, char *argv[]) {
tdDestroyTSchemaBuilder(&schemaBuilder); tdDestroyTSchemaBuilder(&schemaBuilder);
for (int sid =1; sid<10; ++sid) { for (int sid =1; sid<10; ++sid) {
cqCreate(pCq, sid, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema);
} }
tdFreeSchema(pSchema); tdFreeSchema(pSchema);

6
src/dnode/CMakeLists.txt
View File

@ -12,7 +12,7 @@ AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX) IF (TD_LINUX)
ADD_EXECUTABLE(taosd ${SRC}) ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4) TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
IF (TD_SOMODE_STATIC) IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static) TARGET_LINK_LIBRARIES(taosd taos_static)
@ -32,10 +32,6 @@ IF (TD_LINUX)
TARGET_LINK_LIBRARIES(taosd mqtt) TARGET_LINK_LIBRARIES(taosd mqtt)
ENDIF () ENDIF ()
IF (TD_SYNC)
TARGET_LINK_LIBRARIES(taosd balance sync)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd") SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target") SET(PREPARE_ENV_TARGET "prepare_env_target")
ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

1
src/dnode/inc/dnodeCfg.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitCfg(); int32_t dnodeInitCfg();
void dnodeCleanupCfg(); void dnodeCleanupCfg();

1
src/dnode/inc/dnodeCheck.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitCheck(); int32_t dnodeInitCheck();
void dnodeCleanupCheck(); void dnodeCleanupCheck();

3
src/dnode/inc/dnodeEps.h
View File

@ -19,8 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
#include "taosmsg.h"
int32_t dnodeInitEps(); int32_t dnodeInitEps();
void dnodeCleanupEps(); void dnodeCleanupEps();

15
src/dnode/inc/dnodeInt.h
View File

@ -19,8 +19,13 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "taoserror.h"
#include "taosmsg.h"
#include "tlog.h" #include "tlog.h"
#include "trpc.h"
#include "tglobal.h"
#include "dnode.h"
#include "vnode.h"
extern int32_t dDebugFlag; extern int32_t dDebugFlag;
@ -31,6 +36,14 @@ extern int32_t dDebugFlag;
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
typedef enum {
TSDB_RUN_STATUS_INITIALIZE,
TSDB_RUN_STATUS_RUNING,
TSDB_RUN_STATUS_STOPPED
} SRunStatus;
SRunStatus dnodeGetRunStatus();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
src/dnode/inc/dnodeMInfos.h
View File

@ -19,8 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
#include "taosmsg.h"
int32_t dnodeInitMInfos(); int32_t dnodeInitMInfos();
void dnodeCleanupMInfos(); void dnodeCleanupMInfos();
@ -29,6 +28,10 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet);
void dnodeGetMInfos(SMInfos *pMinfos); void dnodeGetMInfos(SMInfos *pMinfos);
bool dnodeIsMasterEp(char *ep); bool dnodeIsMasterEp(char *ep);
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
void dnodeGetEpSetForShell(SRpcEpSet *epSet);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
src/dnode/inc/dnodeMPeer.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitMPeer(); int32_t dnodeInitMPeer();
void dnodeCleanupMPeer(); void dnodeCleanupMPeer();

1
src/dnode/inc/dnodeMRead.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitMRead(); int32_t dnodeInitMRead();
void dnodeCleanupMRead(); void dnodeCleanupMRead();

1
src/dnode/inc/dnodeMWrite.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitMWrite(); int32_t dnodeInitMWrite();
void dnodeCleanupMWrite(); void dnodeCleanupMWrite();

1
src/dnode/inc/dnodeMain.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitSystem(); int32_t dnodeInitSystem();
void dnodeCleanUpSystem(); void dnodeCleanUpSystem();

3
src/dnode/inc/dnodeModule.h
View File

@ -19,10 +19,11 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitModules(); int32_t dnodeInitModules();
void dnodeStartModules();
void dnodeCleanupModules(); void dnodeCleanupModules();
bool dnodeStartMnode(SMInfos *pMinfos);
void dnodeProcessModuleStatus(uint32_t moduleStatus); void dnodeProcessModuleStatus(uint32_t moduleStatus);
#ifdef __cplusplus #ifdef __cplusplus

1
src/dnode/inc/dnodePeer.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitServer(); int32_t dnodeInitServer();
void dnodeCleanupServer(); void dnodeCleanupServer();

1
src/dnode/inc/dnodeShell.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitShell(); int32_t dnodeInitShell();
void dnodeCleanupShell(); void dnodeCleanupShell();

33
src/dnode/inc/dnodeStep.h Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_DNODE_STEP_H
#define TDENGINE_DNODE_STEP_H
#ifdef __cplusplus
extern "C" {
#endif
#include "dnodeInt.h"
int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize);
void dnodeReportStep(char *name, char *desc, int8_t finished);
void dnodeSendStartupStep(SRpcMsg *pMsg);
#ifdef __cplusplus
}
#endif
#endif

1
src/dnode/inc/dnodeTelemetry.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitTelemetry(); int32_t dnodeInitTelemetry();
void dnodeCleanupTelemetry(); void dnodeCleanupTelemetry();

32
src/dnode/inc/dnodeVMgmt.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_DNODE_VMGMT_H
#define TDENGINE_DNODE_VMGMT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "dnodeInt.h"
int32_t dnodeInitVMgmt();
void dnodeCleanupVMgmt();
void dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg);
#ifdef __cplusplus
}
#endif
#endif

7
src/dnode/inc/dnodeVRead.h
View File

@ -19,12 +19,15 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitVRead(); int32_t dnodeInitVRead();
void dnodeCleanupVRead(); void dnodeCleanupVRead();
void dnodeDispatchToVReadQueue(SRpcMsg *pMsg); void dnodeDispatchToVReadQueue(SRpcMsg *pMsg);
void * dnodeAllocVReadQueue(void *pVnode); void * dnodeAllocVQueryQueue(void *pVnode);
void dnodeFreeVReadQueue(void *pRqueue); void * dnodeAllocVFetchQueue(void *pVnode);
void dnodeFreeVQueryQueue(void *pQqueue);
void dnodeFreeVFetchQueue(void *pFqueue);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
src/dnode/inc/dnodeVWrite.h
View File

@ -19,6 +19,7 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "dnodeInt.h"
int32_t dnodeInitVWrite(); int32_t dnodeInitVWrite();
void dnodeCleanupVWrite(); void dnodeCleanupVWrite();

34
src/dnode/inc/dnodeVnodes.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_DNODE_VNODES_H
#define TDENGINE_DNODE_VNODES_H
#ifdef __cplusplus
extern "C" {
#endif
#include "dnodeInt.h"
int32_t dnodeInitVnodes();
void dnodeCleanupVnodes();
int32_t dnodeInitStatusTimer();
void dnodeCleanupStatusTimer();
void dnodeSendStatusMsgToMnode();
#ifdef __cplusplus
}
#endif
#endif

3
src/dnode/src/dnodeCfg.c
View File

@ -16,9 +16,6 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "tglobal.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeCfg.h" #include "dnodeCfg.h"
static SDnodeCfg tsCfg = {0}; static SDnodeCfg tsCfg = {0};

2
src/dnode/src/dnodeCheck.c
View File

@ -15,8 +15,6 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "tglobal.h"
#include "dnodeInt.h"
#include "dnodeCheck.h" #include "dnodeCheck.h"
typedef struct { typedef struct {

12
src/dnode/src/dnodeEps.c
View File

@ -16,10 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "tglobal.h"
#include "hash.h" #include "hash.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeEps.h" #include "dnodeEps.h"
static SDnodeEps *tsEps = NULL; static SDnodeEps *tsEps = NULL;
@ -133,7 +130,7 @@ static void dnodePrintEps(SDnodeEps *eps) {
dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
for (int32_t i = 0; i < eps->dnodeNum; i++) { for (int32_t i = 0; i < eps->dnodeNum; i++) {
SDnodeEp *ep = &eps->dnodeEps[i]; SDnodeEp *ep = &eps->dnodeEps[i];
dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
} }
} }
@ -236,7 +233,14 @@ PRASE_EPS_OVER:
dnodeResetEps(eps); dnodeResetEps(eps);
if (eps) free(eps); if (eps) free(eps);
#if 0
dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
#else
if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) {
dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
return -1;
}
#endif
terrno = 0; terrno = 0;
return 0; return 0;

52
src/dnode/src/dnodeMInfos.c
View File

@ -16,10 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeMInfos.h" #include "dnodeMInfos.h"
static SMInfos tsMInfos; static SMInfos tsMInfos;
@ -157,12 +154,13 @@ static void dnodeResetMInfos(SMInfos *pMinfos) {
} }
static int32_t dnodeReadMInfos() { static int32_t dnodeReadMInfos() {
int32_t len = 0; int32_t len = 0;
int32_t maxLen = 2000; int32_t maxLen = 2000;
char * content = calloc(1, maxLen + 1); char * content = calloc(1, maxLen + 1);
cJSON * root = NULL; cJSON * root = NULL;
FILE * fp = NULL; FILE * fp = NULL;
SMInfos minfos = {0}; SMInfos minfos = {0};
bool nodeChanged = false;
char file[TSDB_FILENAME_LEN + 20] = {0}; char file[TSDB_FILENAME_LEN + 20] = {0};
sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
@ -221,14 +219,19 @@ static int32_t dnodeReadMInfos() {
dError("failed to read mnodeEpSet.json, nodeId not found"); dError("failed to read mnodeEpSet.json, nodeId not found");
goto PARSE_MINFOS_OVER; goto PARSE_MINFOS_OVER;
} }
minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("failed to read mnodeEpSet.json, nodeName not found"); dError("failed to read mnodeEpSet.json, nodeName not found");
goto PARSE_MINFOS_OVER; goto PARSE_MINFOS_OVER;
} }
strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
SMInfo *pMinfo = &minfos.mnodeInfos[i];
pMinfo->mnodeId = nodeId->valueint;
tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp);
if (changed) nodeChanged = changed;
} }
dInfo("read file %s successed", file); dInfo("read file %s successed", file);
@ -245,6 +248,11 @@ PARSE_MINFOS_OVER:
dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
} }
dnodeResetMInfos(&minfos); dnodeResetMInfos(&minfos);
if (nodeChanged) {
dnodeWriteMInfos();
}
return 0; return 0;
} }
@ -286,3 +294,25 @@ static int32_t dnodeWriteMInfos() {
dInfo("successed to write %s", file); dInfo("successed to write %s", file);
return 0; return 0;
} }
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(rpcMsg->handle, &connInfo);
SRpcEpSet epSet = {0};
if (forShell) {
dnodeGetEpSetForShell(&epSet);
} else {
dnodeGetEpSetForPeer(&epSet);
}
dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
epSet.port[i] = htons(epSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &epSet);
}

9
src/dnode/src/dnodeMPeer.c
View File

@ -15,16 +15,11 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tutil.h"
#include "tqueue.h" #include "tqueue.h"
#include "twal.h" #include "twal.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h" #include "dnodeVMgmt.h"
#include "dnodeInt.h" #include "dnodeMInfos.h"
#include "dnodeMgmt.h"
#include "dnodeMWrite.h" #include "dnodeMWrite.h"
typedef struct { typedef struct {

9
src/dnode/src/dnodeMRead.c
View File

@ -15,16 +15,11 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tutil.h"
#include "tqueue.h" #include "tqueue.h"
#include "twal.h" #include "twal.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h" #include "dnodeVMgmt.h"
#include "dnodeInt.h" #include "dnodeMInfos.h"
#include "dnodeMgmt.h"
#include "dnodeMRead.h" #include "dnodeMRead.h"
typedef struct { typedef struct {

14
src/dnode/src/dnodeMWrite.c
View File

@ -15,17 +15,11 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tutil.h"
#include "ttimer.h" #include "ttimer.h"
#include "tqueue.h" #include "tqueue.h"
#include "twal.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h" #include "dnodeVMgmt.h"
#include "dnodeInt.h" #include "dnodeMInfos.h"
#include "dnodeMgmt.h"
#include "dnodeMWrite.h" #include "dnodeMWrite.h"
typedef struct { typedef struct {
@ -127,7 +121,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
dnodeSendRedirectMsg(pMsg, true); dnodeSendRedirectMsg(pMsg, true);
} else { } else {
SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, dTrace("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
} }
@ -136,7 +130,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
} }
static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, dTrace("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
mnodeCleanupMsg(pWrite); mnodeCleanupMsg(pWrite);

109
src/dnode/src/dnodeMain.c
View File

@ -16,15 +16,14 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taos.h" #include "taos.h"
#include "tutil.h" #include "tnote.h"
#include "ttimer.h"
#include "tconfig.h" #include "tconfig.h"
#include "tglobal.h"
#include "tfile.h" #include "tfile.h"
#include "twal.h" #include "twal.h"
#include "trpc.h" // #include "tfs.h"
#include "dnode.h" #include "tsync.h"
#include "dnodeInt.h" #include "dnodeStep.h"
#include "dnodeMgmt.h"
#include "dnodePeer.h" #include "dnodePeer.h"
#include "dnodeModule.h" #include "dnodeModule.h"
#include "dnodeEps.h" #include "dnodeEps.h"
@ -33,50 +32,47 @@
#include "dnodeCheck.h" #include "dnodeCheck.h"
#include "dnodeVRead.h" #include "dnodeVRead.h"
#include "dnodeVWrite.h" #include "dnodeVWrite.h"
#include "dnodeVMgmt.h"
#include "dnodeVnodes.h"
#include "dnodeMRead.h" #include "dnodeMRead.h"
#include "dnodeMWrite.h" #include "dnodeMWrite.h"
#include "dnodeMPeer.h" #include "dnodeMPeer.h"
#include "dnodeShell.h" #include "dnodeShell.h"
#include "dnodeTelemetry.h" #include "dnodeTelemetry.h"
void *tsDnodeTmr = NULL;
static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
static int32_t dnodeInitStorage(); static int32_t dnodeInitStorage();
static void dnodeCleanupStorage(); static void dnodeCleanupStorage();
static void dnodeSetRunStatus(SRunStatus status); static void dnodeSetRunStatus(SRunStatus status);
static void dnodeCheckDataDirOpenned(char *dir); static void dnodeCheckDataDirOpenned(char *dir);
static int32_t dnodeInitComponents();
static void dnodeCleanupComponents(int32_t stepId);
static int dnodeCreateDir(const char *dir); static int dnodeCreateDir(const char *dir);
typedef struct { static SStep tsDnodeSteps[] = {
const char *const name; {"dnode-tfile", tfInit, tfCleanup},
int32_t (*init)(); {"dnode-rpc", rpcInit, rpcCleanup},
void (*cleanup)(); {"dnode-globalcfg", taosCheckGlobalCfg, NULL},
} SDnodeComponent; {"dnode-storage", dnodeInitStorage, dnodeCleanupStorage},
{"dnode-cfg", dnodeInitCfg, dnodeCleanupCfg},
static const SDnodeComponent tsDnodeComponents[] = { {"dnode-eps", dnodeInitEps, dnodeCleanupEps},
{"tfile", tfInit, tfCleanup}, {"dnode-minfos", dnodeInitMInfos, dnodeCleanupMInfos},
{"rpc", rpcInit, rpcCleanup}, {"dnode-wal", walInit, walCleanUp},
{"storage", dnodeInitStorage, dnodeCleanupStorage}, {"dnode-sync", syncInit, syncCleanUp},
{"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, {"dnode-check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
{"dnodeeps", dnodeInitEps, dnodeCleanupEps}, {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead},
{"globalcfg" ,taosCheckGlobalCfg, NULL}, {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite},
{"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt},
{"wal", walInit, walCleanUp}, {"dnode-mread", dnodeInitMRead, dnodeCleanupMRead},
{"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! {"dnode-mwrite", dnodeInitMWrite, dnodeCleanupMWrite},
{"vread", dnodeInitVRead, dnodeCleanupVRead}, {"dnode-mpeer", dnodeInitMPeer, dnodeCleanupMPeer},
{"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"dnode-client", dnodeInitClient, dnodeCleanupClient},
{"mread", dnodeInitMRead, dnodeCleanupMRead}, {"dnode-server", dnodeInitServer, dnodeCleanupServer},
{"mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes},
{"mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, {"dnode-modules", dnodeInitModules, dnodeCleanupModules},
{"client", dnodeInitClient, dnodeCleanupClient}, {"dnode-shell", dnodeInitShell, dnodeCleanupShell},
{"server", dnodeInitServer, dnodeCleanupServer}, {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer},
{"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry},
{"modules", dnodeInitModules, dnodeCleanupModules},
{"mgmt-tmr", dnodeInitMgmtTimer, dnodeCleanupMgmtTimer},
{"shell", dnodeInitShell, dnodeCleanupShell},
{"telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry},
}; };
static int dnodeCreateDir(const char *dir) { static int dnodeCreateDir(const char *dir) {
@ -87,24 +83,31 @@ static int dnodeCreateDir(const char *dir) {
return 0; return 0;
} }
static void dnodeCleanupComponents(int32_t stepId) { static void dnodeCleanupComponents() {
for (int32_t i = stepId; i >= 0; i--) { int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
if (tsDnodeComponents[i].cleanup) { dnodeStepCleanup(tsDnodeSteps, stepSize);
(*tsDnodeComponents[i].cleanup)();
}
}
} }
static int32_t dnodeInitComponents() { static int32_t dnodeInitComponents() {
int32_t code = 0; int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
for (int32_t i = 0; i < sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]); i++) { return dnodeStepInit(tsDnodeSteps, stepSize);
if (tsDnodeComponents[i].init() != 0) { }
dnodeCleanupComponents(i);
code = -1; static int32_t dnodeInitTmr() {
break; tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
} if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
return -1;
}
return 0;
}
static void dnodeCleanupTmr() {
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
} }
return code;
} }
int32_t dnodeInitSystem() { int32_t dnodeInitSystem() {
@ -115,6 +118,8 @@ int32_t dnodeInitSystem() {
taosInitGlobalCfg(); taosInitGlobalCfg();
taosReadGlobalLogCfg(); taosReadGlobalLogCfg();
taosSetCoreDump(); taosSetCoreDump();
taosInitNotes();
dnodeInitTmr();
signal(SIGPIPE, SIG_IGN); signal(SIGPIPE, SIG_IGN);
if (dnodeCreateDir(tsLogDir) < 0) { if (dnodeCreateDir(tsLogDir) < 0) {
@ -140,7 +145,6 @@ int32_t dnodeInitSystem() {
return -1; return -1;
} }
dnodeStartModules();
dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
dInfo("TDengine is initialized successfully"); dInfo("TDengine is initialized successfully");
@ -151,7 +155,8 @@ int32_t dnodeInitSystem() {
void dnodeCleanUpSystem() { void dnodeCleanUpSystem() {
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1); dnodeCleanupTmr();
dnodeCleanupComponents();
taos_cleanup(); taos_cleanup();
taosCloseLog(); taosCloseLog();
} }

572
src/dnode/src/dnodeMgmt.c
View File

@ -1,572 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "cJSON.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "ttimer.h"
#include "tsdb.h"
#include "twal.h"
#include "tqueue.h"
#include "tsync.h"
#include "ttimer.h"
#include "tbn.h"
#include "tglobal.h"
#include "dnode.h"
#include "vnode.h"
#include "mnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
#include "dnodeEps.h"
#include "dnodeCfg.h"
#include "dnodeMInfos.h"
#include "dnodeVRead.h"
#include "dnodeVWrite.h"
#include "dnodeModule.h"
typedef struct {
pthread_t thread;
int32_t threadIndex;
int32_t failed;
int32_t opened;
int32_t vnodeNum;
int32_t * vnodeList;
} SOpenVnodeThread;
typedef struct {
SRpcMsg rpcMsg;
char pCont[];
} SMgmtMsg;
void * tsDnodeTmr = NULL;
static void * tsStatusTimer = NULL;
static uint32_t tsRebootTime;
static taos_qset tsMgmtQset = NULL;
static taos_queue tsMgmtQueue = NULL;
static pthread_t tsQthread;
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void *dnodeProcessMgmtQueue(void *param);
static int32_t dnodeOpenVnodes();
static void dnodeCloseVnodes();
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
int32_t dnodeInitMgmt() {
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
tsRebootTime = taosGetTimestampSec();
int32_t code = vnodeInitResources();
if (code != TSDB_CODE_SUCCESS) {
dnodeCleanupMgmt();
return -1;
}
// create the queue and thread to handle the message
tsMgmtQset = taosOpenQset();
if (tsMgmtQset == NULL) {
dError("failed to create the mgmt queue set");
dnodeCleanupMgmt();
return -1;
}
tsMgmtQueue = taosOpenQueue();
if (tsMgmtQueue == NULL) {
dError("failed to create the mgmt queue");
dnodeCleanupMgmt();
return -1;
}
taosAddIntoQset(tsMgmtQset, tsMgmtQueue, NULL);
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
code = pthread_create(&tsQthread, &thAttr, dnodeProcessMgmtQueue, NULL);
pthread_attr_destroy(&thAttr);
if (code != 0) {
dError("failed to create thread to process mgmt queue, reason:%s", strerror(errno));
dnodeCleanupMgmt();
return -1;
}
code = dnodeOpenVnodes();
if (code != TSDB_CODE_SUCCESS) {
dnodeCleanupMgmt();
return -1;
}
dInfo("dnode mgmt is initialized");
return TSDB_CODE_SUCCESS;
}
int32_t dnodeInitMgmtTimer() {
tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
if (tsDnodeTmr == NULL) {
dError("failed to init dnode timer");
dnodeCleanupMgmt();
return -1;
}
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
dInfo("dnode mgmt timer is initialized");
return TSDB_CODE_SUCCESS;
}
void dnodeSendStatusMsgToMnode() {
if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
dInfo("force send status msg to mnode");
taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
}
}
void dnodeCleanupMgmtTimer() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
if (tsDnodeTmr != NULL) {
taosTmrCleanUp(tsDnodeTmr);
tsDnodeTmr = NULL;
}
}
void dnodeCleanupMgmt() {
dnodeCleanupMgmtTimer();
dnodeCloseVnodes();
if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);
if (tsQthread) pthread_join(tsQthread, NULL);
if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue);
if (tsMgmtQset) taosCloseQset(tsMgmtQset);
tsMgmtQset = NULL;
tsMgmtQueue = NULL;
vnodeCleanupResources();
}
static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
int32_t size = sizeof(SMgmtMsg) + pMsg->contLen;
SMgmtMsg *pMgmt = taosAllocateQitem(size);
if (pMgmt == NULL) {
return TSDB_CODE_DND_OUT_OF_MEMORY;
}
pMgmt->rpcMsg = *pMsg;
pMgmt->rpcMsg.pCont = pMgmt->pCont;
memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
return TSDB_CODE_SUCCESS;
}
void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
int32_t code = dnodeWriteToMgmtQueue(pMsg);
if (code != TSDB_CODE_SUCCESS) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
static void *dnodeProcessMgmtQueue(void *param) {
SMgmtMsg *pMgmt;
SRpcMsg * pMsg;
SRpcMsg rsp = {0};
int32_t qtype;
void * handle;
while (1) {
if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
break;
}
pMsg = &pMgmt->rpcMsg;
dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
} else {
rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
}
rsp.handle = pMsg->handle;
rsp.pCont = NULL;
rpcSendResponse(&rsp);
taosFreeQitem(pMsg);
}
return NULL;
}
static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
DIR *dir = opendir(tsVnodeDir);
if (dir == NULL) {
return TSDB_CODE_DND_NO_WRITE_ACCESS;
}
*numOfVnodes = 0;
struct dirent *de = NULL;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
if (de->d_type & DT_DIR) {
if (strncmp("vnode", de->d_name, 5) != 0) continue;
int32_t vnode = atoi(de->d_name + 5);
if (vnode == 0) continue;
(*numOfVnodes)++;
if (*numOfVnodes >= TSDB_MAX_VNODES) {
dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
continue;
} else {
vnodeList[*numOfVnodes - 1] = vnode;
}
}
}
closedir(dir);
return TSDB_CODE_SUCCESS;
}
static void *dnodeOpenVnode(void *param) {
SOpenVnodeThread *pThread = param;
char vnodeDir[TSDB_FILENAME_LEN * 3];
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
int32_t vgId = pThread->vnodeList[v];
snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId);
if (vnodeOpen(vgId, vnodeDir) < 0) {
dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
pThread->failed++;
} else {
dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
pThread->opened++;
}
}
dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
return NULL;
}
static int32_t dnodeOpenVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES] = {0};
int32_t numOfVnodes = 0;
int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
return status;
}
int32_t threadNum = tsNumOfCores;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t;
threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
}
for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum;
SOpenVnodeThread *pThread = &threads[t];
pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
}
dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum == 0) continue;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
}
pthread_attr_destroy(&thAttr);
}
int32_t openVnodes = 0;
int32_t failedVnodes = 0;
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum > 0 && pThread->thread) {
pthread_join(pThread->thread, NULL);
}
openVnodes += pThread->opened;
failedVnodes += pThread->failed;
free(pThread->vnodeList);
}
free(threads);
dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes);
return TSDB_CODE_SUCCESS;
}
static void dnodeCloseVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES]= {0};
int32_t numOfVnodes = 0;
int32_t status;
status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
return;
}
for (int32_t i = 0; i < numOfVnodes; ++i) {
vnodeClose(vnodeList[i]);
}
dInfo("total vnodes:%d are all closed", numOfVnodes);
}
static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize);
pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks);
pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile);
pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1);
pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2);
pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep);
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
}
return pCreate;
}
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
vnodeRelease(pVnode);
return TSDB_CODE_SUCCESS;
} else {
dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
return vnodeCreate(pCreate);
}
}
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
int32_t code = vnodeAlter(pVnode, pAlter);
vnodeRelease(pVnode);
return code;
} else {
dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
return TSDB_CODE_VND_INVALID_VGROUP_ID;
}
}
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
SDropVnodeMsg *pDrop = rpcMsg->pCont;
pDrop->vgId = htonl(pDrop->vgId);
return vnodeDrop(pDrop->vgId);
}
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
// SAlterStreamMsg *pStream = pCont;
// pStream->uid = htobe64(pStream->uid);
// pStream->stime = htobe64(pStream->stime);
// pStream->vnode = htonl(pStream->vnode);
// pStream->sid = htonl(pStream->sid);
// pStream->status = htonl(pStream->status);
//
// int32_t code = dnodeCreateStream(pStream);
return 0;
}
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
SCfgDnodeMsg *pCfg = pMsg->pCont;
return taosCfgDynamicOptions(pCfg->config);
}
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) {
dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
}
if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
}
dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
}
dnodeStartMnode(&pCfg->mnodes);
return TSDB_CODE_SUCCESS;
}
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
SStatusRsp *pStatusRsp = pMsg->pCont;
SMInfos *pMinfos = &pStatusRsp->mnodes;
dnodeUpdateMInfos(pMinfos);
SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
pCfg->moduleStatus = htonl(pCfg->moduleStatus);
pCfg->dnodeId = htonl(pCfg->dnodeId);
dnodeUpdateCfg(pCfg);
vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
dnodeUpdateEps(pEps);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
pStatus->dnodeId = htonl(dnodeGetDnodeId());
pStatus->version = htonl(tsVersion);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
// fill cluster cfg parameters
pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes);
pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance);
pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold);
pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode);
pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb);
tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
pStatus->clusterCfg.checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
SRpcEpSet epSet;
dnodeGetEpSetForPeer(&epSet);
dnodeSendMsgToDnode(&epSet, &rpcMsg);
}
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(rpcMsg->handle, &connInfo);
SRpcEpSet epSet = {0};
if (forShell) {
dnodeGetEpSetForShell(&epSet);
} else {
dnodeGetEpSetForPeer(&epSet);
}
dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
for (int i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
epSet.port[i] = htons(epSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &epSet);
}

31
src/dnode/src/dnodeModule.c
View File

@ -15,15 +15,10 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "http.h" #include "http.h"
#include "tmqtt.h" #include "tmqtt.h"
#include "monitor.h" #include "monitor.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeModule.h" #include "dnodeModule.h"
typedef struct { typedef struct {
@ -102,6 +97,20 @@ void dnodeCleanupModules() {
} }
} }
static int32_t dnodeStartModules() {
for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) {
if (tsModule[module].enable && tsModule[module].startFp) {
int32_t code = (*tsModule[module].startFp)();
if (code != 0) {
dError("failed to start module:%s, code:%d", tsModule[module].name, code);
return code;
}
}
}
return 0;
}
int32_t dnodeInitModules() { int32_t dnodeInitModules() {
dnodeAllocModules(); dnodeAllocModules();
@ -115,17 +124,7 @@ int32_t dnodeInitModules() {
} }
dInfo("dnode modules is initialized"); dInfo("dnode modules is initialized");
return 0; return dnodeStartModules();
}
void dnodeStartModules() {
for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) {
if (tsModule[module].enable && tsModule[module].startFp) {
if ((*tsModule[module].startFp)() != 0) {
dError("failed to start module:%s", tsModule[module].name);
}
}
}
} }
void dnodeProcessModuleStatus(uint32_t moduleStatus) { void dnodeProcessModuleStatus(uint32_t moduleStatus) {

24
src/dnode/src/dnodePeer.c
View File

@ -21,15 +21,12 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h" #include "dnodeVMgmt.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
#include "dnodeVWrite.h" #include "dnodeVWrite.h"
#include "dnodeMPeer.h" #include "dnodeMPeer.h"
#include "dnodeMInfos.h" #include "dnodeMInfos.h"
#include "dnodeStep.h"
static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
@ -44,12 +41,12 @@ int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
@ -93,12 +90,13 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
}; };
if (pMsg->pCont == NULL) return; if (pMsg->pCont == NULL) return;
if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg);
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_APP_NOT_READY; rspMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rspMsg); rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
dDebug("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
return; return;
} }
@ -153,7 +151,7 @@ void dnodeCleanupClient() {
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) {
if (pMsg == NULL || pMsg->pCont == NULL) return; if (pMsg == NULL || pMsg->pCont == NULL) return;
dDebug("msg:%p is ignored since dnode is stopping", pMsg); dTrace("msg:%p is ignored since dnode is stopping", pMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
return; return;
} }

29
src/dnode/src/dnodeShell.c
View File

@ -15,20 +15,14 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "tutil.h"
#include "http.h" #include "http.h"
#include "mnode.h" #include "mnode.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeVRead.h" #include "dnodeVRead.h"
#include "dnodeVWrite.h" #include "dnodeVWrite.h"
#include "dnodeMRead.h" #include "dnodeMRead.h"
#include "dnodeMWrite.h" #include "dnodeMWrite.h"
#include "dnodeShell.h" #include "dnodeShell.h"
#include "dnodeStep.h"
static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
@ -74,8 +68,9 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0;
if (numOfThreads < 1) { if (numOfThreads < 1) {
numOfThreads = 1; numOfThreads = 1;
} }
@ -142,7 +137,23 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
} }
} }
static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
if (strcmp(user, "nettestinternal") == 0) {
char pass[32] = {0};
taosEncryptPass((uint8_t *)user, strlen(user), pass);
*spi = 0;
*encrypt = 0;
*ckey = 0;
memcpy(secret, pass, TSDB_KEY_LEN);
dTrace("nettest user is authorized");
return 0;
}
return -1;
}
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0;
int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
if (code != TSDB_CODE_APP_NOT_READY) return code; if (code != TSDB_CODE_APP_NOT_READY) return code;

73
src/dnode/src/dnodeStep.c Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "dnodeStep.h"
static SStartupStep tsStartupStep;
void dnodeReportStep(char *name, char *desc, int8_t finished) {
tstrncpy(tsStartupStep.name, name, sizeof(tsStartupStep.name));
tstrncpy(tsStartupStep.desc, desc, sizeof(tsStartupStep.desc));
tsStartupStep.finished = finished;
}
void dnodeSendStartupStep(SRpcMsg *pMsg) {
dInfo("nettest msg is received, cont:%s", (char *)pMsg->pCont);
SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep));
memcpy(pStep, &tsStartupStep, sizeof(SStartupStep));
dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished);
SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pMsg->pCont);
}
void taosStepCleanupImp(SStep *pSteps, int32_t stepId) {
for (int32_t step = stepId; step >= 0; step--) {
SStep *pStep = pSteps + step;
dDebug("step:%s will cleanup", pStep->name);
if (pStep->cleanupFp != NULL) {
(*pStep->cleanupFp)();
}
}
}
int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) {
for (int32_t step = 0; step < stepSize; step++) {
SStep *pStep = pSteps + step;
if (pStep->initFp == NULL) continue;
dnodeReportStep(pStep->name, "Start initialization", 0);
int32_t code = (*pStep->initFp)();
if (code != 0) {
dDebug("step:%s will init", pStep->name);
taosStepCleanupImp(pSteps, step);
return code;
}
dnodeReportStep(pStep->name, "Initialization complete", step + 1 >= stepSize);
}
return 0;
}
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) {
return taosStepCleanupImp(pSteps, stepSize - 1);
}

22
src/dnode/src/dnodeSystem.c
View File

@ -16,15 +16,15 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "tgrant.h" #include "tgrant.h"
#include "tutil.h" #include "tconfig.h"
#include "tglobal.h"
#include "dnodeInt.h"
#include "dnodeMain.h" #include "dnodeMain.h"
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
static tsem_t exitSem; static tsem_t exitSem;
int32_t main(int32_t argc, char *argv[]) { int32_t main(int32_t argc, char *argv[]) {
int dump_config = 0;
// Set global configuration file // Set global configuration file
for (int32_t i = 1; i < argc; ++i) { for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) { if (strcmp(argv[i], "-c") == 0) {
@ -38,6 +38,8 @@ int32_t main(int32_t argc, char *argv[]) {
printf("'-c' requires a parameter, default:%s\n", configDir); printf("'-c' requires a parameter, default:%s\n", configDir);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} else if (strcmp(argv[i], "-C") == 0) {
dump_config = 1;
} else if (strcmp(argv[i], "-V") == 0) { } else if (strcmp(argv[i], "-V") == 0) {
#ifdef _ACCT #ifdef _ACCT
char *versionStr = "enterprise"; char *versionStr = "enterprise";
@ -90,6 +92,20 @@ int32_t main(int32_t argc, char *argv[]) {
#endif #endif
} }
if (0 != dump_config) {
tscEmbedded = 1;
taosInitGlobalCfg();
taosReadGlobalLogCfg();
if (!taosReadGlobalCfg()) {
printf("TDengine read global config failed");
exit(EXIT_FAILURE);
}
taosDumpGlobalCfg();
exit(EXIT_SUCCESS);
}
if (tsem_init(&exitSem, 0, 0) != 0) { if (tsem_init(&exitSem, 0, 0) != 0) {
printf("failed to create exit semphore\n"); printf("failed to create exit semphore\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);

5
src/dnode/src/dnodeTelemetry.c
View File

@ -15,9 +15,6 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tutil.h"
#include "osTime.h" #include "osTime.h"
#include "tsocket.h" #include "tsocket.h"
#include "tbuffer.h" #include "tbuffer.h"
@ -32,8 +29,6 @@
#include "mnodeTable.h" #include "mnodeTable.h"
#include "mnodeSdb.h" #include "mnodeSdb.h"
#include "mnodeAcct.h" #include "mnodeAcct.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeTelemetry.h" #include "dnodeTelemetry.h"
static tsem_t tsExitSem; static tsem_t tsExitSem;

219
src/dnode/src/dnodeVMgmt.c Normal file
View File

@ -0,0 +1,219 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tqueue.h"
#include "tworker.h"
#include "dnodeVMgmt.h"
typedef struct {
SRpcMsg rpcMsg;
char pCont[];
} SMgmtMsg;
static SWorkerPool tsVMgmtWP;
static taos_queue tsVMgmtQueue = NULL;
static void * dnodeProcessMgmtQueue(void *param);
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
int32_t dnodeInitVMgmt() {
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
int32_t code = vnodeInitMgmt();
if (code != TSDB_CODE_SUCCESS) return -1;
tsVMgmtWP.name = "vmgmt";
tsVMgmtWP.workerFp = dnodeProcessMgmtQueue;
tsVMgmtWP.min = 1;
tsVMgmtWP.max = 1;
if (tWorkerInit(&tsVMgmtWP) != 0) return -1;
tsVMgmtQueue = tWorkerAllocQueue(&tsVMgmtWP, NULL);
dInfo("dnode vmgmt is initialized");
return TSDB_CODE_SUCCESS;
}
void dnodeCleanupVMgmt() {
tWorkerFreeQueue(&tsVMgmtWP, tsVMgmtQueue);
tWorkerCleanup(&tsVMgmtWP);
tsVMgmtQueue = NULL;
vnodeCleanupMgmt();
}
static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
int32_t size = sizeof(SMgmtMsg) + pMsg->contLen;
SMgmtMsg *pMgmt = taosAllocateQitem(size);
if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
pMgmt->rpcMsg = *pMsg;
pMgmt->rpcMsg.pCont = pMgmt->pCont;
memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
taosWriteQitem(tsVMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
return TSDB_CODE_SUCCESS;
}
void dnodeDispatchToVMgmtQueue(SRpcMsg *pMsg) {
int32_t code = dnodeWriteToMgmtQueue(pMsg);
if (code != TSDB_CODE_SUCCESS) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
static void *dnodeProcessMgmtQueue(void *wparam) {
SWorker * pWorker = wparam;
SWorkerPool *pPool = pWorker->pPool;
SMgmtMsg * pMgmt;
SRpcMsg * pMsg;
SRpcMsg rsp = {0};
int32_t qtype;
void * handle;
while (1) {
if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pMgmt, &handle) == 0) {
dDebug("qdnode mgmt got no message from qset:%p, , exit", pPool->qset);
break;
}
pMsg = &pMgmt->rpcMsg;
dTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
} else {
rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
}
dTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) {
rsp.handle = pMsg->handle;
rsp.pCont = NULL;
rpcSendResponse(&rsp);
}
taosFreeQitem(pMsg);
}
return NULL;
}
static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize);
pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks);
pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile);
pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1);
pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2);
pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep);
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
}
return pCreate;
}
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
vnodeRelease(pVnode);
return TSDB_CODE_SUCCESS;
} else {
dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
return vnodeCreate(pCreate);
}
}
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
int32_t code = vnodeAlter(pVnode, pAlter);
vnodeRelease(pVnode);
return code;
} else {
dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
return TSDB_CODE_VND_INVALID_VGROUP_ID;
}
}
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
SDropVnodeMsg *pDrop = rpcMsg->pCont;
pDrop->vgId = htonl(pDrop->vgId);
return vnodeDrop(pDrop->vgId);
}
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
return 0;
}
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
SCfgDnodeMsg *pCfg = pMsg->pCont;
return taosCfgDynamicOptions(pCfg->config);
}
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) {
dDebug("dnode:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
}
if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
}
dDebug("dnode:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
}
dnodeStartMnode(&pCfg->mnodes);
return TSDB_CODE_SUCCESS;
}

135
src/dnode/src/dnodeVRead.c
View File

@ -15,71 +15,40 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "tqueue.h" #include "tqueue.h"
#include "vnode.h" #include "tworker.h"
#include "dnodeInt.h" #include "dnodeVRead.h"
typedef struct {
pthread_t thread; // thread
int32_t workerId; // worker ID
} SVReadWorker;
typedef struct {
int32_t max; // max number of workers
int32_t min; // min number of workers
int32_t num; // current number of workers
SVReadWorker * worker;
pthread_mutex_t mutex;
} SVReadWorkerPool;
static void *dnodeProcessReadQueue(void *pWorker); static void *dnodeProcessReadQueue(void *pWorker);
// module global variable // module global variable
static SVReadWorkerPool tsVReadWP; static SWorkerPool tsVQueryWP;
static taos_qset tsVReadQset; static SWorkerPool tsVFetchWP;
int32_t dnodeInitVRead() { int32_t dnodeInitVRead() {
tsVReadQset = taosOpenQset(); const int32_t maxFetchThreads = 4;
tsVReadWP.min = tsNumOfCores; // calculate the available query thread
tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore; float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min;
tsVReadWP.worker = calloc(sizeof(SVReadWorker), tsVReadWP.max);
pthread_mutex_init(&tsVReadWP.mutex, NULL);
if (tsVReadWP.worker == NULL) return -1; tsVQueryWP.name = "vquery";
for (int i = 0; i < tsVReadWP.max; ++i) { tsVQueryWP.workerFp = dnodeProcessReadQueue;
SVReadWorker *pWorker = tsVReadWP.worker + i; tsVQueryWP.min = (int32_t) threadsForQuery;
pWorker->workerId = i; tsVQueryWP.max = tsVQueryWP.min;
} if (tWorkerInit(&tsVQueryWP) != 0) return -1;
tsVFetchWP.name = "vfetch";
tsVFetchWP.workerFp = dnodeProcessReadQueue;
tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores);
tsVFetchWP.max = tsVFetchWP.min;
if (tWorkerInit(&tsVFetchWP) != 0) return -1;
dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max);
return 0; return 0;
} }
void dnodeCleanupVRead() { void dnodeCleanupVRead() {
for (int i = 0; i < tsVReadWP.max; ++i) { tWorkerCleanup(&tsVFetchWP);
SVReadWorker *pWorker = tsVReadWP.worker + i; tWorkerCleanup(&tsVQueryWP);
if (pWorker->thread) {
taosQsetThreadResume(tsVReadQset);
}
}
for (int i = 0; i < tsVReadWP.max; ++i) {
SVReadWorker *pWorker = tsVReadWP.worker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
}
}
free(tsVReadWP.worker);
taosCloseQset(tsVReadQset);
pthread_mutex_destroy(&tsVReadWP.mutex);
dInfo("dnode vread is closed");
} }
void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
@ -92,6 +61,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
assert(pHead->contLen > 0);
void *pVnode = vnodeAcquire(pHead->vgId); void *pVnode = vnodeAcquire(pHead->vgId);
if (pVnode != NULL) { if (pVnode != NULL) {
int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
@ -111,43 +81,20 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
} }
void *dnodeAllocVReadQueue(void *pVnode) { void *dnodeAllocVQueryQueue(void *pVnode) {
pthread_mutex_lock(&tsVReadWP.mutex); return tWorkerAllocQueue(&tsVQueryWP, pVnode);
taos_queue queue = taosOpenQueue();
if (queue == NULL) {
pthread_mutex_unlock(&tsVReadWP.mutex);
return NULL;
}
taosAddIntoQset(tsVReadQset, queue, pVnode);
// spawn a thread to process queue
if (tsVReadWP.num < tsVReadWP.max) {
do {
SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
dError("failed to create thread to process vread vqueue since %s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
tsVReadWP.num++;
dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num);
} while (tsVReadWP.num < tsVReadWP.min);
}
pthread_mutex_unlock(&tsVReadWP.mutex);
dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue);
return queue;
} }
void dnodeFreeVReadQueue(void *pRqueue) { void *dnodeAllocVFetchQueue(void *pVnode) {
taosCloseQueue(pRqueue); return tWorkerAllocQueue(&tsVFetchWP, pVnode);
}
void dnodeFreeVQueryQueue(void *pQqueue) {
tWorkerFreeQueue(&tsVQueryWP, pQqueue);
}
void dnodeFreeVFetchQueue(void *pFqueue) {
tWorkerFreeQueue(&tsVFetchWP, pFqueue);
} }
void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
@ -164,18 +111,20 @@ void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
} }
static void *dnodeProcessReadQueue(void *pWorker) { static void *dnodeProcessReadQueue(void *wparam) {
SVReadMsg *pRead; SWorker * pWorker = wparam;
int32_t qtype; SWorkerPool *pPool = pWorker->pPool;
void * pVnode; SVReadMsg * pRead;
int32_t qtype;
void * pVnode;
while (1) { while (1) {
if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pRead, &pVnode) == 0) {
dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); dDebug("dnode vquery got no message from qset:%p, exiting", pPool->qset);
break; break;
} }
dTrace("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, dTrace("msg:%p, app:%p type:%s will be processed in vquery queue, qtype:%d", pRead, pRead->rpcAhandle,
taosMsg[pRead->msgType], qtype); taosMsg[pRead->msgType], qtype);
int32_t code = vnodeProcessRead(pVnode, pRead); int32_t code = vnodeProcessRead(pVnode, pRead);

7
src/dnode/src/dnodeVWrite.c
View File

@ -15,13 +15,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "tqueue.h" #include "tqueue.h"
#include "twal.h" #include "dnodeVWrite.h"
#include "vnode.h"
#include "dnodeInt.h"
typedef struct { typedef struct {
taos_qall qall; taos_qall qall;

278
src/dnode/src/dnodeVnodes.c Normal file
View File

@ -0,0 +1,278 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ttimer.h"
#include "dnodeEps.h"
#include "dnodeCfg.h"
#include "dnodeMInfos.h"
#include "dnodeVnodes.h"
typedef struct {
pthread_t thread;
int32_t threadIndex;
int32_t failed;
int32_t opened;
int32_t vnodeNum;
int32_t * vnodeList;
} SOpenVnodeThread;
extern void * tsDnodeTmr;
static void * tsStatusTimer = NULL;
static uint32_t tsRebootTime = 0;
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
int32_t dnodeInitStatusTimer() {
dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
tsRebootTime = taosGetTimestampSec();
taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
dInfo("dnode status timer is initialized");
return TSDB_CODE_SUCCESS;
}
void dnodeCleanupStatusTimer() {
if (tsStatusTimer != NULL) {
taosTmrStopA(&tsStatusTimer);
tsStatusTimer = NULL;
}
}
static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
DIR *dir = opendir(tsVnodeDir);
if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS;
*numOfVnodes = 0;
struct dirent *de = NULL;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
if (de->d_type & DT_DIR) {
if (strncmp("vnode", de->d_name, 5) != 0) continue;
int32_t vnode = atoi(de->d_name + 5);
if (vnode == 0) continue;
(*numOfVnodes)++;
if (*numOfVnodes >= TSDB_MAX_VNODES) {
dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
continue;
} else {
vnodeList[*numOfVnodes - 1] = vnode;
}
}
}
closedir(dir);
return TSDB_CODE_SUCCESS;
}
static void *dnodeOpenVnode(void *param) {
SOpenVnodeThread *pThread = param;
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
int32_t vgId = pThread->vnodeList[v];
if (vnodeOpen(vgId) < 0) {
dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
pThread->failed++;
} else {
dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
pThread->opened++;
}
}
dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
return NULL;
}
int32_t dnodeInitVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES] = {0};
int32_t numOfVnodes = 0;
int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
return status;
}
int32_t threadNum = tsNumOfCores;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t;
threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
}
for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum;
SOpenVnodeThread *pThread = &threads[t];
pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
}
dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum == 0) continue;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
}
pthread_attr_destroy(&thAttr);
}
int32_t openVnodes = 0;
int32_t failedVnodes = 0;
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum > 0 && pThread->thread) {
pthread_join(pThread->thread, NULL);
}
openVnodes += pThread->opened;
failedVnodes += pThread->failed;
free(pThread->vnodeList);
}
free(threads);
dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes);
if (failedVnodes != 0) {
dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
return -1;
}
return TSDB_CODE_SUCCESS;
}
void dnodeCleanupVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES]= {0};
int32_t numOfVnodes = 0;
int32_t status;
status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
return;
}
for (int32_t i = 0; i < numOfVnodes; ++i) {
vnodeClose(vnodeList[i]);
}
dInfo("total vnodes:%d are all closed", numOfVnodes);
}
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
return;
}
SStatusRsp *pStatusRsp = pMsg->pCont;
SMInfos *minfos = &pStatusRsp->mnodes;
dnodeUpdateMInfos(minfos);
SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
pCfg->moduleStatus = htonl(pCfg->moduleStatus);
pCfg->dnodeId = htonl(pCfg->dnodeId);
dnodeUpdateCfg(pCfg);
vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
dnodeUpdateEps(pEps);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
return;
}
if (tsStatusTimer == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to start status timer");
return;
}
int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
pStatus->dnodeId = htonl(dnodeGetDnodeId());
pStatus->version = htonl(tsVersion);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
// fill cluster cfg parameters
pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes);
pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance);
pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold);
pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode);
pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb);
tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
pStatus->clusterCfg.checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
.pCont = pStatus,
.contLen = contLen,
.msgType = TSDB_MSG_TYPE_DM_STATUS
};
SRpcEpSet epSet;
dnodeGetEpSetForPeer(&epSet);
dnodeSendMsgToDnode(&epSet, &rpcMsg);
}
void dnodeSendStatusMsgToMnode() {
if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
dInfo("force send status msg to mnode");
taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
}
}

23
src/inc/dnode.h
View File

@ -29,13 +29,6 @@ typedef struct {
int32_t httpReqNum; int32_t httpReqNum;
} SStatisInfo; } SStatisInfo;
typedef enum {
TSDB_RUN_STATUS_INITIALIZE,
TSDB_RUN_STATUS_RUNING,
TSDB_RUN_STATUS_STOPPED
} SRunStatus;
SRunStatus dnodeGetRunStatus();
SStatisInfo dnodeGetStatisInfo(); SStatisInfo dnodeGetStatisInfo();
bool dnodeIsFirstDeploy(); bool dnodeIsFirstDeploy();
@ -56,8 +49,10 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid);
void *dnodeAllocVWriteQueue(void *pVnode); void *dnodeAllocVWriteQueue(void *pVnode);
void dnodeFreeVWriteQueue(void *pWqueue); void dnodeFreeVWriteQueue(void *pWqueue);
void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code); void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code);
void *dnodeAllocVReadQueue(void *pVnode); void *dnodeAllocVQueryQueue(void *pVnode);
void dnodeFreeVReadQueue(void *pRqueue); void *dnodeAllocVFetchQueue(void *pVnode);
void dnodeFreeVQueryQueue(void *pQqueue);
void dnodeFreeVFetchQueue(void *pFqueue);
int32_t dnodeAllocateMPeerQueue(); int32_t dnodeAllocateMPeerQueue();
void dnodeFreeMPeerQueue(); void dnodeFreeMPeerQueue();
@ -71,6 +66,16 @@ void dnodeDelayReprocessMWriteMsg(void *pMsg);
void dnodeSendStatusMsgToMnode(); void dnodeSendStatusMsgToMnode();
typedef struct {
char *name;
int32_t (*initFp)();
void (*cleanupFp)();
} SStep;
int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize);
void dnodeReportStep(char *name, char *desc, int8_t finished);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
src/inc/taosdef.h
View File

@ -286,6 +286,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_SHOW_SQL_LEN 512 #define TSDB_SHOW_SQL_LEN 512
#define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SLOW_QUERY_SQL_LEN 512
#define TSDB_STEP_NAME_LEN 32
#define TSDB_STEP_DESC_LEN 128
#define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_HOSTNAME_LEN 64
#define TSDB_MQTT_PORT_LEN 8 #define TSDB_MQTT_PORT_LEN 8
#define TSDB_MQTT_USER_LEN 24 #define TSDB_MQTT_USER_LEN 24
@ -439,6 +442,10 @@ typedef enum {
TAOS_QTYPE_QUERY = 4 TAOS_QTYPE_QUERY = 4
} EQType; } EQType;
#define TSDB_MAX_TIERS 3
#define TSDB_MAX_DISKS_PER_TIER 16
#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
typedef enum { typedef enum {
TSDB_SUPER_TABLE = 0, // super table TSDB_SUPER_TABLE = 0, // super table
TSDB_CHILD_TABLE = 1, // table created from super table TSDB_CHILD_TABLE = 1, // table created from super table

20
src/inc/taoserror.h
View File

@ -64,7 +64,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, 0, 0x000F, "Invalid tr
TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid session id") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid session id")
TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type")
TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type")
TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Client and server's time is not synchronized")
TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready")
TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN")
@ -192,6 +192,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message no
TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, 0, 0x0404, "Action in progress")
// vnode // vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress")
@ -205,9 +206,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing da
TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode") TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Database memory is full for commit failed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, 0, 0x050C, "Database memory is full for waiting commit")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Database write operation denied")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing")
// tsdb // tsdb
@ -394,6 +396,18 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad se
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete")
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general")
// tfs
TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, 0, 0x2200, "tfs out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, 0, 0x2201, "tfs invalid mount config")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, 0, 0x2202, "tfs too many mount")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, 0, 0x2203, "tfs duplicate primary mount")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_PRIMARY_DISK, 0, 0x2204, "tfs no primary mount")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, 0, 0x2205, "tfs no mount at tier")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, 0, 0x2206, "tfs file already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, 0, 0x2207, "tfs invalid level")
TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, 0, 0x2208, "tfs no valid disk")
#ifdef TAOS_ERROR_C #ifdef TAOS_ERROR_C
}; };
#endif #endif

16
src/inc/taosmsg.h
View File

@ -105,10 +105,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
#ifndef TAOS_MESSAGE_C #ifndef TAOS_MESSAGE_C
TSDB_MSG_TYPE_MAX // 105 TSDB_MSG_TYPE_MAX // 105
@ -476,7 +473,7 @@ typedef struct {
int16_t numOfGroupCols; // num of group by columns int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx; int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx int16_t orderType; // used in group by xx order by xxx
int64_t tableLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query.
int16_t prjOrder; // global order in super table projection query. int16_t prjOrder; // global order in super table projection query.
int64_t limit; int64_t limit;
int64_t offset; int64_t offset;
@ -790,6 +787,7 @@ typedef struct {
typedef struct { typedef struct {
char sql[TSDB_SHOW_SQL_LEN]; char sql[TSDB_SHOW_SQL_LEN];
char dstTable[TSDB_TABLE_NAME_LEN];
uint32_t streamId; uint32_t streamId;
int64_t num; // number of computing/cycles int64_t num; // number of computing/cycles
int64_t useconds; int64_t useconds;
@ -840,6 +838,14 @@ typedef struct {
char ckey[TSDB_KEY_LEN]; char ckey[TSDB_KEY_LEN];
} SAuthMsg, SAuthRsp; } SAuthMsg, SAuthRsp;
typedef struct {
int8_t finished;
int8_t reserved1[7];
char name[TSDB_STEP_NAME_LEN];
char desc[TSDB_STEP_DESC_LEN];
char reserved2[64];
} SStartupStep;
#pragma pack(pop) #pragma pack(pop)
#ifdef __cplusplus #ifdef __cplusplus

2
src/inc/tcq.h
View File

@ -42,7 +42,7 @@ void cqStart(void *handle);
void cqStop(void *handle); void cqStop(void *handle);
// cqCreate is called by TSDB to start an instance of CQ // cqCreate is called by TSDB to start an instance of CQ
void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema); void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
void cqDrop(void *handle); void cqDrop(void *handle);

4
src/inc/tsdb.h
View File

@ -48,7 +48,7 @@ typedef struct {
void *cqH; void *cqH;
int (*notifyStatus)(void *, int status, int eno); int (*notifyStatus)(void *, int status, int eno);
int (*eventCallBack)(void *); int (*eventCallBack)(void *);
void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
void (*cqDropFunc)(void *handle); void (*cqDropFunc)(void *handle);
} STsdbAppH; } STsdbAppH;
@ -321,7 +321,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
*/ */
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage); void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
int tsdbInitCommitQueue(int nthreads); int tsdbInitCommitQueue();
void tsdbDestroyCommitQueue(); void tsdbDestroyCommitQueue();
int tsdbSyncCommit(TSDB_REPO_T *repo); int tsdbSyncCommit(TSDB_REPO_T *repo);
void tsdbIncCommitRef(int vgId); void tsdbIncCommitRef(int vgId);

34
src/inc/vnode.h
View File

@ -19,18 +19,9 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "trpc.h" #include "trpc.h"
#include "twal.h" #include "twal.h"
typedef enum _VN_STATUS {
TAOS_VN_STATUS_INIT = 0,
TAOS_VN_STATUS_READY = 1,
TAOS_VN_STATUS_CLOSING = 2,
TAOS_VN_STATUS_UPDATING = 3,
TAOS_VN_STATUS_RESET = 4,
} EVnodeStatus;
typedef struct { typedef struct {
int32_t len; int32_t len;
void * rsp; void * rsp;
@ -43,6 +34,7 @@ typedef struct {
void * rpcHandle; void * rpcHandle;
void * rpcAhandle; void * rpcAhandle;
void * qhandle; void * qhandle;
void * pVnode;
int8_t qtype; int8_t qtype;
int8_t msgType; int8_t msgType;
SRspRet rspRet; SRspRet rspRet;
@ -60,29 +52,35 @@ typedef struct {
SWalHead pHead[]; SWalHead pHead[];
} SVWriteMsg; } SVWriteMsg;
// vnodeStatus
extern char *vnodeStatus[]; extern char *vnodeStatus[];
// vnodeMain
int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeDrop(int32_t vgId); int32_t vnodeDrop(int32_t vgId);
int32_t vnodeOpen(int32_t vgId, char *rootDir); int32_t vnodeOpen(int32_t vgId);
int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeClose(int32_t vgId); int32_t vnodeClose(int32_t vgId);
void* vnodeAcquire(int32_t vgId); // add refcount // vnodeMgmt
void vnodeRelease(void *pVnode); // dec refCount int32_t vnodeInitMgmt();
void vnodeCleanupMgmt();
void* vnodeAcquire(int32_t vgId);
void vnodeRelease(void *pVnode);
void* vnodeGetWal(void *pVnode); void* vnodeGetWal(void *pVnode);
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
void vnodeBuildStatusMsg(void *pStatus);
void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
// vnodeWrite
int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg); int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
void vnodeBuildStatusMsg(void *pStatus); // vnodeSync
void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code);
void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
int32_t vnodeInitResources();
void vnodeCleanupResources();
// vnodeRead
int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead); void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead);
int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);

2
src/kit/shell/inc/shell.h
View File

@ -45,13 +45,13 @@ typedef struct SShellArguments {
char* timezone; char* timezone;
bool is_raw_time; bool is_raw_time;
bool is_use_passwd; bool is_use_passwd;
bool dump_config;
char file[TSDB_FILENAME_LEN]; char file[TSDB_FILENAME_LEN];
char dir[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN];
int threadNum; int threadNum;
char* commands; char* commands;
int abort; int abort;
int port; int port;
int endPort;
int pktLen; int pktLen;
char* netTestRole; char* netTestRole;
} SShellArguments; } SShellArguments;

55
src/kit/shell/src/shellEngine.c
View File

@ -32,14 +32,14 @@
/**************** Global variables ****************/ /**************** Global variables ****************/
#ifdef _TD_POWER_ #ifdef _TD_POWER_
char CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n" char CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n"
"Copyright (c) 2017 by PowerDB, Inc. All rights reserved.\n\n"; "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n";
char PROMPT_HEADER[] = "power> "; char PROMPT_HEADER[] = "power> ";
char CONTINUE_PROMPT[] = " -> "; char CONTINUE_PROMPT[] = " -> ";
int prompt_size = 7; int prompt_size = 7;
#else #else
char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n"; "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
char PROMPT_HEADER[] = "taos> "; char PROMPT_HEADER[] = "taos> ";
char CONTINUE_PROMPT[] = " -> "; char CONTINUE_PROMPT[] = " -> ";
@ -509,7 +509,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
static void shellPrintNChar(const char *str, int length, int width) { static void shellPrintNChar(const char *str, int length, int width) {
int pos = 0, cols = 0; wchar_t tail[3];
int pos = 0, cols = 0, totalCols = 0, tailLen = 0;
while (pos < length) { while (pos < length) {
wchar_t wc; wchar_t wc;
int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
@ -526,15 +528,44 @@ static void shellPrintNChar(const char *str, int length, int width) {
#else #else
int w = wcwidth(wc); int w = wcwidth(wc);
#endif #endif
if (w > 0) { if (w <= 0) {
if (width > 0 && cols + w > width) { continue;
break; }
}
if (width <= 0) {
printf("%lc", wc);
continue;
}
totalCols += w;
if (totalCols > width) {
break;
}
if (totalCols <= (width - 3)) {
printf("%lc", wc); printf("%lc", wc);
cols += w; cols += w;
} else {
tail[tailLen] = wc;
tailLen++;
} }
} }
if (totalCols > width) {
// width could be 1 or 2, so printf("...") cannot be used
for (int i = 0; i < 3; i++) {
if (cols >= width) {
break;
}
putchar('.');
++cols;
}
} else {
for (int i = 0; i < tailLen; i++) {
printf("%lc", tail[i]);
}
cols = totalCols;
}
for (; cols < width; cols++) { for (; cols < width; cols++) {
putchar(' '); putchar(' ');
} }
@ -656,13 +687,21 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
return MAX(25, width); return MAX(25, width);
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (field->bytes > tsMaxBinaryDisplayWidth) { if (field->bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width); return MAX(tsMaxBinaryDisplayWidth, width);
} else { } else {
return MAX(field->bytes, width); return MAX(field->bytes, width);
} }
case TSDB_DATA_TYPE_NCHAR: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width);
} else {
return MAX(bytes, width);
}
}
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
if (args.is_raw_time) { if (args.is_raw_time) {
return MAX(14, width); return MAX(14, width);

19
src/kit/shell/src/shellLinux.c
View File

@ -39,6 +39,7 @@ static struct argp_option options[] = {
{"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."},
{"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."}, {"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."},
{"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."},
{"dump-config", 'C', 0, 0, "Dump configuration."},
{"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."},
{"raw-time", 'r', 0, 0, "Output time as uint64_t."}, {"raw-time", 'r', 0, 0, "Output time as uint64_t."},
{"file", 'f', "FILE", 0, "Script to run without enter the shell."}, {"file", 'f', "FILE", 0, "Script to run without enter the shell."},
@ -46,8 +47,7 @@ static struct argp_option options[] = {
{"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."},
{"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."},
{"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."},
{"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, options: client|clients|server."}, {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."},
{"endport", 'e', "ENDPORT", 0, "Net test end port, default is 6042."},
{"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."},
{0}}; {0}};
@ -97,6 +97,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path); wordfree(&full_path);
break; break;
case 'C':
arguments->dump_config = true;
break;
case 's': case 's':
arguments->commands = arg; arguments->commands = arg;
break; break;
@ -130,20 +133,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 'd': case 'd':
arguments->database = arg; arguments->database = arg;
break; break;
case 'n': case 'n':
arguments->netTestRole = arg; arguments->netTestRole = arg;
break; break;
case 'e':
if (arg) {
arguments->endPort = atoi(arg);
} else {
fprintf(stderr, "Invalid end port\n");
return -1;
}
break;
case 'l': case 'l':
if (arg) { if (arg) {
arguments->pktLen = atoi(arg); arguments->pktLen = atoi(arg);
@ -152,7 +144,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
return -1; return -1;
} }
break; break;
case OPT_ABORT: case OPT_ABORT:
arguments->abort = 1; arguments->abort = 1;
break; break;

20
src/kit/shell/src/shellMain.c
View File

@ -15,6 +15,7 @@
#include "os.h" #include "os.h"
#include "shell.h" #include "shell.h"
#include "tconfig.h"
#include "tnettest.h" #include "tnettest.h"
pthread_t pid; pthread_t pid;
@ -58,11 +59,11 @@ SShellArguments args = {
.timezone = NULL, .timezone = NULL,
.is_raw_time = false, .is_raw_time = false,
.is_use_passwd = false, .is_use_passwd = false,
.dump_config = false,
.file = "\0", .file = "\0",
.dir = "\0", .dir = "\0",
.threadNum = 5, .threadNum = 5,
.commands = NULL, .commands = NULL,
.endPort = 6042,
.pktLen = 1000, .pktLen = 1000,
.netTestRole = NULL .netTestRole = NULL
}; };
@ -79,11 +80,22 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args); shellParseArgument(argc, argv, &args);
if (args.dump_config) {
taosInitGlobalCfg();
taosReadGlobalLogCfg();
if (!taosReadGlobalCfg()) {
printf("TDengine read global config failed");
exit(EXIT_FAILURE);
}
taosDumpGlobalCfg();
exit(0);
}
if (args.netTestRole && args.netTestRole[0] != 0) { if (args.netTestRole && args.netTestRole[0] != 0) {
taos_init(); taos_init();
CmdArguments cmdArgs; taosNetTest(args.netTestRole, args.host, args.port, args.pktLen);
memcpy(&cmdArgs, &args, sizeof(SShellArguments));
taosNetTest(&cmdArgs);
exit(0); exit(0);
} }

26
src/kit/shell/src/shellWindows.c
View File

@ -35,6 +35,8 @@ void printHelp() {
printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server.");
printf("%s%s\n", indent, "-c"); printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory."); printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-C");
printf("%s%s%s\n", indent, indent, "Dump configuration.");
printf("%s%s\n", indent, "-s"); printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r"); printf("%s%s\n", indent, "-r");
@ -45,6 +47,10 @@ void printHelp() {
printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
printf("%s%s\n", indent, "-t"); printf("%s%s\n", indent, "-t");
printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
printf("%s%s\n", indent, "-n");
printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup.");
printf("%s%s\n", indent, "-l");
printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
@ -100,6 +106,8 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
fprintf(stderr, "Option -c requires an argument\n"); fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} else if (strcmp(argv[i], "-C") == 0) {
arguments->dump_config = true;
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
arguments->commands = argv[++i]; arguments->commands = argv[++i];
@ -137,6 +145,24 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
// For time zone
else if (strcmp(argv[i], "-n") == 0) {
if (i < argc - 1) {
arguments->netTestRole = argv[++i];
} else {
fprintf(stderr, "option -n requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-l") == 0) {
if (i < argc - 1) {
arguments->pktLen = atoi(argv[++i]);
} else {
fprintf(stderr, "option -l requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For temperory command TODO // For temperory command TODO
else if (strcmp(argv[i], "--help") == 0) { else if (strcmp(argv[i], "--help") == 0) {
printHelp(); printHelp();

984
src/kit/taosdump/taosdump.c
View File

File diff suppressed because it is too large Load Diff

4
src/mnode/inc/mnodeMnode.h
View File

@ -43,8 +43,8 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode);
void mnodeDecMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode);
char * mnodeGetMnodeRoleStr(); char * mnodeGetMnodeRoleStr();
void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect);
void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect);
char* mnodeGetMnodeMasterEp(); char* mnodeGetMnodeMasterEp();
void mnodeGetMnodeInfos(void *mnodes); void mnodeGetMnodeInfos(void *mnodes);

7
src/mnode/src/mnodeDb.c
View File

@ -317,13 +317,6 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
return TSDB_CODE_MND_INVALID_DB_OPTION; return TSDB_CODE_MND_INVALID_DB_OPTION;
} }
#ifndef _SYNC
if (pCfg->replications != 1) {
mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
return TSDB_CODE_MND_INVALID_DB_OPTION;
}
#endif
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE); mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
return TSDB_CODE_MND_INVALID_DB_OPTION; return TSDB_CODE_MND_INVALID_DB_OPTION;

64
src/mnode/src/mnodeDnode.c
View File

@ -111,9 +111,6 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) {
static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) { static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
SDnodeObj *pDnode = pRow->pObj; SDnodeObj *pDnode = pRow->pObj;
#ifndef _SYNC
mnodeDropAllDnodeVgroups(pDnode);
#endif
mnodeDropMnodeLocal(pDnode->dnodeId); mnodeDropMnodeLocal(pDnode->dnodeId);
bnNotify(); bnNotify();
mnodeUpdateDnodeEps(); mnodeUpdateDnodeEps();
@ -306,7 +303,7 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
int32_t code = sdbUpdateRow(&row); int32_t code = sdbUpdateRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("dnodeId:%d, failed update", pDnode->dnodeId); mError("dnode:%d, failed update", pDnode->dnodeId);
} }
} }
@ -705,11 +702,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
mInfo("dnode:%d, start to drop it", pDnode->dnodeId); mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
#ifndef _SYNC
int32_t code = mnodeDropDnode(pDnode, pMsg);
#else
int32_t code = bnDropDnode(pDnode); int32_t code = bnDropDnode(pDnode);
#endif
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
return code; return code;
} }
@ -1179,58 +1172,3 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
default:return "any"; default:return "any";
} }
} }
#ifndef _SYNC
int32_t bnInit() { return TSDB_CODE_SUCCESS; }
void bnCleanUp() {}
void bnNotify() {}
void bnCheckModules() {}
void bnReset() {}
int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
char* syncRole[] = {
"offline",
"unsynced",
"syncing",
"slave",
"master"
};
int32_t bnAllocVnodes(SVgObj *pVgroup) {
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
SDnodeObj *pSelDnode = NULL;
float vnodeUsage = 1000.0;
while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
float openVnodes = pDnode->openVnodes;
if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
float usage = openVnodes / pDnode->numOfCores;
if (usage <= vnodeUsage) {
pSelDnode = pDnode;
vnodeUsage = usage;
}
}
mnodeDecDnodeRef(pDnode);
}
if (pSelDnode == NULL) {
mError("failed to alloc vnode to vgroup");
return TSDB_CODE_MND_NO_ENOUGH_DNODES;
}
pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
pVgroup->vnodeGid[0].pDnode = pSelDnode;
mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
return TSDB_CODE_SUCCESS;
}
#endif

Some files were not shown because too many files have changed in this diff Show More