xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/TD-2191

This commit is contained in:
zyyang 2021-01-05 17:12:02 +08:00
parent 3d80c54c94 7138b29f96
commit 4c18fcd557
400 changed files with 25759 additions and 9235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.appveyor.yml
View File

@ -3,6 +3,7 @@ os: Visual Studio 2015
environment:
matrix:
- ARCH: amd64
- ARCH: x86
clone_folder: c:\dev\TDengine
clone_depth: 1
@ -23,6 +24,7 @@ notifications:
- provider: Email
to:
- sangshuduo@gmail.com
on_build_success: true
on_build_failure: true
on_build_status_changed: true

8
.travis.yml
View File

@ -146,7 +146,7 @@ matrix:
branch_pattern: coverity_scan
- os: linux
dist: trusty
dist: xenial
language: c
git:
- depth: 1
@ -157,7 +157,7 @@ matrix:
- build-essential
- cmake
env:
- DESC="trusty/gcc-4.8 build"
- DESC="xenial build"
before_script:
- export TZ=Asia/Harbin
@ -227,7 +227,7 @@ matrix:
- os: linux
arch: arm64
dist: trusty
dist: xenial
language: c
git:
- depth: 1
@ -238,7 +238,7 @@ matrix:
- build-essential
- cmake
env:
- DESC="trusty/gcc-4.8 build"
- DESC="xenial build"
before_script:
- export TZ=Asia/Harbin

331
Jenkinsfile vendored
View File

@ -1,27 +1,58 @@
pipeline {
agent none
environment{
WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDinternal/community'
import hudson.model.Result
import jenkins.model.CauseOfInterruption
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine.git'
}
def abortPreviousBuilds() {
def currentJobName = env.JOB_NAME
def currentBuildNumber = env.BUILD_NUMBER.toInteger()
def jobs = Jenkins.instance.getItemByFullName(currentJobName)
def builds = jobs.getBuilds()
for (build in builds) {
if (!build.isBuilding()) {
continue;
}
stages {
stage('Parallel test stage') {
parallel {
stage('pytest') {
agent{label '184'}
steps {
if (currentBuildNumber == build.getNumber().toInteger()) {
continue;
}
build.doKill() //doTerm(),doKill(),doTerm()
}
}
//abort previous build
abortPreviousBuilds()
def abort_previous(){
def buildNumber = env.BUILD_NUMBER as int
if (buildNumber > 1) milestone(buildNumber - 1)
milestone(buildNumber)
}
def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
date
sudo rmtaos
'''
}
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git reset --hard HEAD~10
git pull
git submodule update
git fetch
git checkout ${CHANGE_BRANCH}
git reset --hard HEAD~10
git pull
git merge develop
cd ${WK}
git reset --hard
git reset --hard HEAD~10
git checkout develop
git pull
cd ${WK}
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
@ -31,62 +62,65 @@ pipeline {
make > /dev/null
make install > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh pytest
'''
return 1
}
pipeline {
agent none
environment{
WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDinternal/community'
}
stages {
stage('Parallel test stage') {
//only build pr
when {
changeRequest()
}
parallel {
stage('python_1') {
agent{label 'p1'}
steps {
pre_test()
sh '''
cd ${WKC}/tests
find pytest -name '*'sql|xargs rm -rf
./test-all.sh p1
date'''
}
}
stage('python_2') {
agent{label 'p2'}
steps {
pre_test()
sh '''
cd ${WKC}/tests
find pytest -name '*'sql|xargs rm -rf
./test-all.sh p2
date'''
}
}
stage('test_b1') {
agent{label 'master'}
agent{label 'b1'}
steps {
pre_test()
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh b1
./test-all.sh b1fq
date'''
}
}
stage('test_crash_gen') {
agent{label "185"}
agent{label "b2"}
steps {
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
'''
pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
@ -102,200 +136,35 @@ pipeline {
sh '''
date
cd ${WKC}/tests
./test-all.sh b2
./test-all.sh b2fq
date
'''
}
}
stage('test_valgrind') {
agent{label "186"}
agent{label "b3"}
steps {
pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
'''
}
sh '''
date
cd ${WKC}/tests
./test-all.sh b3
./test-all.sh b3fq
date'''
}
}
stage('connector'){
agent{label "release"}
steps{
sh'''
cd ${WORKSPACE}
git checkout develop
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
}
}
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
}
}
}
post {
success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
}

2
alert/go.mod
View File

@ -5,7 +5,7 @@ go 1.14
require (
github.com/jmoiron/sqlx v1.2.0
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/taosdata/driver-go v0.0.0-20200727182616-1a3b1941c206
github.com/taosdata/driver-go v0.0.0-20201113094317-050667e5b4d0
go.uber.org/zap v1.14.1
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c

24
cmake/define.inc
View File

@ -45,7 +45,7 @@ IF (TD_LINUX_64)
ADD_DEFINITIONS(-D_M_X64)
ADD_DEFINITIONS(-D_TD_LINUX_64)
MESSAGE(STATUS "linux64 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ADD_DEFINITIONS(-DUSE_LIBICONV)
ENDIF ()
@ -53,7 +53,7 @@ IF (TD_LINUX_32)
ADD_DEFINITIONS(-D_TD_LINUX_32)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "linux32 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
IF (TD_ARM_64)
@ -62,7 +62,7 @@ IF (TD_ARM_64)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm64 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
IF (TD_ARM_32)
@ -70,21 +70,21 @@ IF (TD_ARM_32)
ADD_DEFINITIONS(-D_TD_ARM_)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm32 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
ENDIF ()
IF (TD_MIPS_64)
ADD_DEFINITIONS(-D_TD_MIPS_64_)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "mips64 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
IF (TD_MIPS_32)
ADD_DEFINITIONS(-D_TD_MIPS_32_)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "mips32 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
ENDIF ()
IF (TD_APLHINE)
@ -105,8 +105,8 @@ IF (TD_LINUX)
MESSAGE(STATUS "set ningsi macro to true")
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable")
SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG")
SET(RELEASE_FLAGS "-O3 -Wno-error")
IF (${COVER} MATCHES "true")
MESSAGE(STATUS "Test coverage mode, add extra flags")
@ -125,9 +125,9 @@ IF (TD_DARWIN_64)
ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "darwin64 is defined")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG")
SET(RELEASE_FLAGS "-Og")
ENDIF ()
IF (TD_WINDOWS)
@ -140,7 +140,7 @@ IF (TD_WINDOWS)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /GL")
SET(RELEASE_FLAGS "/W0 /O3 /GL")
ENDIF ()
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)

2
cmake/env.inc
View File

@ -41,8 +41,10 @@ SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FL
# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
SET(CMAKE_BUILD_TYPE "Debug")
MESSAGE(STATUS "Build Debug Version")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS)

2
cmake/version.inc
View File

@ -4,7 +4,7 @@ PROJECT(TDengine)
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "2.0.9.0")
SET(TD_VER_NUMBER "2.0.12.0")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2415
deps/libcurl/include/curl/curl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

198
deps/libcurl/include/curl/curlbuild.h vendored Normal file
View File

@ -0,0 +1,198 @@
/* include/curl/curlbuild.h. Generated from curlbuild.h.in by configure. */
#ifndef __CURL_CURLBUILD_H
#define __CURL_CURLBUILD_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
/* ================================================================ */
/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */
/* ================================================================ */
/*
* NOTE 1:
* -------
*
* Nothing in this file is intended to be modified or adjusted by the
* curl library user nor by the curl library builder.
*
* If you think that something actually needs to be changed, adjusted
* or fixed in this file, then, report it on the libcurl development
* mailing list: http://cool.haxx.se/mailman/listinfo/curl-library/
*
* This header file shall only export symbols which are 'curl' or 'CURL'
* prefixed, otherwise public name space would be polluted.
*
* NOTE 2:
* -------
*
* Right now you might be staring at file include/curl/curlbuild.h.in or
* at file include/curl/curlbuild.h, this is due to the following reason:
*
* On systems capable of running the configure script, the configure process
* will overwrite the distributed include/curl/curlbuild.h file with one that
* is suitable and specific to the library being configured and built, which
* is generated from the include/curl/curlbuild.h.in template file.
*
*/
/* ================================================================ */
/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */
/* ================================================================ */
#ifdef CURL_SIZEOF_LONG
#error "CURL_SIZEOF_LONG shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_SIZEOF_LONG_already_defined
#endif
#ifdef CURL_TYPEOF_CURL_SOCKLEN_T
#error "CURL_TYPEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_already_defined
#endif
#ifdef CURL_SIZEOF_CURL_SOCKLEN_T
#error "CURL_SIZEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_already_defined
#endif
#ifdef CURL_TYPEOF_CURL_OFF_T
#error "CURL_TYPEOF_CURL_OFF_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_already_defined
#endif
#ifdef CURL_FORMAT_CURL_OFF_T
#error "CURL_FORMAT_CURL_OFF_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_already_defined
#endif
#ifdef CURL_FORMAT_CURL_OFF_TU
#error "CURL_FORMAT_CURL_OFF_TU shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_already_defined
#endif
#ifdef CURL_FORMAT_OFF_T
#error "CURL_FORMAT_OFF_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_FORMAT_OFF_T_already_defined
#endif
#ifdef CURL_SIZEOF_CURL_OFF_T
#error "CURL_SIZEOF_CURL_OFF_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_already_defined
#endif
#ifdef CURL_SUFFIX_CURL_OFF_T
#error "CURL_SUFFIX_CURL_OFF_T shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_already_defined
#endif
#ifdef CURL_SUFFIX_CURL_OFF_TU
#error "CURL_SUFFIX_CURL_OFF_TU shall not be defined except in curlbuild.h"
Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_already_defined
#endif
/* ================================================================ */
/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */
/* ================================================================ */
/* Configure process defines this to 1 when it finds out that system */
/* header file ws2tcpip.h must be included by the external interface. */
/* #undef CURL_PULL_WS2TCPIP_H */
#ifdef CURL_PULL_WS2TCPIP_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# include <winsock2.h>
# include <ws2tcpip.h>
#endif
/* Configure process defines this to 1 when it finds out that system */
/* header file sys/types.h must be included by the external interface. */
#define CURL_PULL_SYS_TYPES_H 1
#ifdef CURL_PULL_SYS_TYPES_H
# include <sys/types.h>
#endif
/* Configure process defines this to 1 when it finds out that system */
/* header file stdint.h must be included by the external interface. */
/* #undef CURL_PULL_STDINT_H */
#ifdef CURL_PULL_STDINT_H
# include <stdint.h>
#endif
/* Configure process defines this to 1 when it finds out that system */
/* header file inttypes.h must be included by the external interface. */
/* #undef CURL_PULL_INTTYPES_H */
#ifdef CURL_PULL_INTTYPES_H
# include <inttypes.h>
#endif
/* Configure process defines this to 1 when it finds out that system */
/* header file sys/socket.h must be included by the external interface. */
#define CURL_PULL_SYS_SOCKET_H 1
#ifdef CURL_PULL_SYS_SOCKET_H
# include <sys/socket.h>
#endif
/* Configure process defines this to 1 when it finds out that system */
/* header file sys/poll.h must be included by the external interface. */
/* #undef CURL_PULL_SYS_POLL_H */
#ifdef CURL_PULL_SYS_POLL_H
# include <sys/poll.h>
#endif
/* The size of `long', as computed by sizeof. */
#define CURL_SIZEOF_LONG 8
/* Integral data type used for curl_socklen_t. */
#define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t
/* The size of `curl_socklen_t', as computed by sizeof. */
#define CURL_SIZEOF_CURL_SOCKLEN_T 4
/* Data type definition of curl_socklen_t. */
typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t;
/* Signed integral data type used for curl_off_t. */
#define CURL_TYPEOF_CURL_OFF_T long
/* Data type definition of curl_off_t. */
typedef CURL_TYPEOF_CURL_OFF_T curl_off_t;
/* curl_off_t formatting string directive without "%" conversion specifier. */
#define CURL_FORMAT_CURL_OFF_T "ld"
/* unsigned curl_off_t formatting string without "%" conversion specifier. */
#define CURL_FORMAT_CURL_OFF_TU "lu"
/* curl_off_t formatting string directive with "%" conversion specifier. */
#define CURL_FORMAT_OFF_T "%ld"
/* The size of `curl_off_t', as computed by sizeof. */
#define CURL_SIZEOF_CURL_OFF_T 8
/* curl_off_t constant suffix. */
#define CURL_SUFFIX_CURL_OFF_T L
/* unsigned curl_off_t constant suffix. */
#define CURL_SUFFIX_CURL_OFF_TU UL
#endif /* __CURL_CURLBUILD_H */

262
deps/libcurl/include/curl/curlrules.h vendored Normal file
View File

@ -0,0 +1,262 @@
#ifndef __CURL_CURLRULES_H
#define __CURL_CURLRULES_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
/* ================================================================ */
/* COMPILE TIME SANITY CHECKS */
/* ================================================================ */
/*
* NOTE 1:
* -------
*
* All checks done in this file are intentionally placed in a public
* header file which is pulled by curl/curl.h when an application is
* being built using an already built libcurl library. Additionally
* this file is also included and used when building the library.
*
* If compilation fails on this file it is certainly sure that the
* problem is elsewhere. It could be a problem in the curlbuild.h
* header file, or simply that you are using different compilation
* settings than those used to build the library.
*
* Nothing in this file is intended to be modified or adjusted by the
* curl library user nor by the curl library builder.
*
* Do not deactivate any check, these are done to make sure that the
* library is properly built and used.
*
* You can find further help on the libcurl development mailing list:
* http://cool.haxx.se/mailman/listinfo/curl-library/
*
* NOTE 2
* ------
*
* Some of the following compile time checks are based on the fact
* that the dimension of a constant array can not be a negative one.
* In this way if the compile time verification fails, the compilation
* will fail issuing an error. The error description wording is compiler
* dependent but it will be quite similar to one of the following:
*
* "negative subscript or subscript is too large"
* "array must have at least one element"
* "-1 is an illegal array size"
* "size of array is negative"
*
* If you are building an application which tries to use an already
* built libcurl library and you are getting this kind of errors on
* this file, it is a clear indication that there is a mismatch between
* how the library was built and how you are trying to use it for your
* application. Your already compiled or binary library provider is the
* only one who can give you the details you need to properly use it.
*/
/*
* Verify that some macros are actually defined.
*/
#ifndef CURL_SIZEOF_LONG
# error "CURL_SIZEOF_LONG definition is missing!"
Error Compilation_aborted_CURL_SIZEOF_LONG_is_missing
#endif
#ifndef CURL_TYPEOF_CURL_SOCKLEN_T
# error "CURL_TYPEOF_CURL_SOCKLEN_T definition is missing!"
Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_is_missing
#endif
#ifndef CURL_SIZEOF_CURL_SOCKLEN_T
# error "CURL_SIZEOF_CURL_SOCKLEN_T definition is missing!"
Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_is_missing
#endif
#ifndef CURL_TYPEOF_CURL_OFF_T
# error "CURL_TYPEOF_CURL_OFF_T definition is missing!"
Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_is_missing
#endif
#ifndef CURL_FORMAT_CURL_OFF_T
# error "CURL_FORMAT_CURL_OFF_T definition is missing!"
Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_is_missing
#endif
#ifndef CURL_FORMAT_CURL_OFF_TU
# error "CURL_FORMAT_CURL_OFF_TU definition is missing!"
Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_is_missing
#endif
#ifndef CURL_FORMAT_OFF_T
# error "CURL_FORMAT_OFF_T definition is missing!"
Error Compilation_aborted_CURL_FORMAT_OFF_T_is_missing
#endif
#ifndef CURL_SIZEOF_CURL_OFF_T
# error "CURL_SIZEOF_CURL_OFF_T definition is missing!"
Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_is_missing
#endif
#ifndef CURL_SUFFIX_CURL_OFF_T
# error "CURL_SUFFIX_CURL_OFF_T definition is missing!"
Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_is_missing
#endif
#ifndef CURL_SUFFIX_CURL_OFF_TU
# error "CURL_SUFFIX_CURL_OFF_TU definition is missing!"
Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_is_missing
#endif
/*
* Macros private to this header file.
*/
#define CurlchkszEQ(t, s) sizeof(t) == s ? 1 : -1
#define CurlchkszGE(t1, t2) sizeof(t1) >= sizeof(t2) ? 1 : -1
/*
* Verify that the size previously defined and expected for long
* is the same as the one reported by sizeof() at compile time.
*/
typedef char
__curl_rule_01__
[CurlchkszEQ(long, CURL_SIZEOF_LONG)];
/*
* Verify that the size previously defined and expected for
* curl_off_t is actually the the same as the one reported
* by sizeof() at compile time.
*/
typedef char
__curl_rule_02__
[CurlchkszEQ(curl_off_t, CURL_SIZEOF_CURL_OFF_T)];
/*
* Verify at compile time that the size of curl_off_t as reported
* by sizeof() is greater or equal than the one reported for long
* for the current compilation.
*/
typedef char
__curl_rule_03__
[CurlchkszGE(curl_off_t, long)];
/*
* Verify that the size previously defined and expected for
* curl_socklen_t is actually the the same as the one reported
* by sizeof() at compile time.
*/
typedef char
__curl_rule_04__
[CurlchkszEQ(curl_socklen_t, CURL_SIZEOF_CURL_SOCKLEN_T)];
/*
* Verify at compile time that the size of curl_socklen_t as reported
* by sizeof() is greater or equal than the one reported for int for
* the current compilation.
*/
typedef char
__curl_rule_05__
[CurlchkszGE(curl_socklen_t, int)];
/* ================================================================ */
/* EXTERNALLY AND INTERNALLY VISIBLE DEFINITIONS */
/* ================================================================ */
/*
* CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow
* these to be visible and exported by the external libcurl interface API,
* while also making them visible to the library internals, simply including
* curl_setup.h, without actually needing to include curl.h internally.
* If some day this section would grow big enough, all this should be moved
* to its own header file.
*/
/*
* Figure out if we can use the ## preprocessor operator, which is supported
* by ISO/ANSI C and C++. Some compilers support it without setting __STDC__
* or __cplusplus so we need to carefully check for them too.
*/
#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \
defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \
defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \
defined(__ILEC400__)
/* This compiler is believed to have an ISO compatible preprocessor */
#define CURL_ISOCPP
#else
/* This compiler is believed NOT to have an ISO compatible preprocessor */
#undef CURL_ISOCPP
#endif
/*
* Macros for minimum-width signed and unsigned curl_off_t integer constants.
*/
#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551)
# define __CURL_OFF_T_C_HLPR2(x) x
# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x)
# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \
__CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T)
# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \
__CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU)
#else
# ifdef CURL_ISOCPP
# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix
# else
# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix
# endif
# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix)
# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T)
# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU)
#endif
/*
* Get rid of macros private to this header file.
*/
#undef CurlchkszEQ
#undef CurlchkszGE
/*
* Get rid of macros not intended to exist beyond this point.
*/
#undef CURL_PULL_WS2TCPIP_H
#undef CURL_PULL_SYS_TYPES_H
#undef CURL_PULL_SYS_SOCKET_H
#undef CURL_PULL_SYS_POLL_H
#undef CURL_PULL_STDINT_H
#undef CURL_PULL_INTTYPES_H
#undef CURL_TYPEOF_CURL_SOCKLEN_T
#undef CURL_TYPEOF_CURL_OFF_T
#ifdef CURL_NO_OLDIES
#undef CURL_FORMAT_OFF_T /* not required since 7.19.0 - obsoleted in 7.20.0 */
#endif
#endif /* __CURL_CURLRULES_H */

77
deps/libcurl/include/curl/curlver.h vendored Normal file
View File

@ -0,0 +1,77 @@
#ifndef __CURL_CURLVER_H
#define __CURL_CURLVER_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
/* This header file contains nothing but libcurl version info, generated by
a script at release-time. This was made its own header file in 7.11.2 */
/* This is the global package copyright */
#define LIBCURL_COPYRIGHT "1996 - 2015 Daniel Stenberg, <daniel@haxx.se>."
/* This is the version number of the libcurl package from which this header
file origins: */
#define LIBCURL_VERSION "7.47.0"
/* The numeric version number is also available "in parts" by using these
defines: */
#define LIBCURL_VERSION_MAJOR 7
#define LIBCURL_VERSION_MINOR 47
#define LIBCURL_VERSION_PATCH 0
/* This is the numeric version of the libcurl version number, meant for easier
parsing and comparions by programs. The LIBCURL_VERSION_NUM define will
always follow this syntax:
0xXXYYZZ
Where XX, YY and ZZ are the main version, release and patch numbers in
hexadecimal (using 8 bits each). All three numbers are always represented
using two digits. 1.2 would appear as "0x010200" while version 9.11.7
appears as "0x090b07".
This 6-digit (24 bits) hexadecimal number does not show pre-release number,
and it is always a greater number in a more recent release. It makes
comparisons with greater than and less than work.
Note: This define is the full hex number and _does not_ use the
CURL_VERSION_BITS() macro since curl's own configure script greps for it
and needs it to contain the full number.
*/
#define LIBCURL_VERSION_NUM 0x072f00
/*
* This is the date and time when the full source package was created. The
* timestamp is not stored in git, as the timestamp is properly set in the
* tarballs by the maketgz script.
*
* The format of the date should follow this template:
*
* "Mon Feb 12 11:35:33 UTC 2007"
*/
#define LIBCURL_TIMESTAMP "Wed Jan 27 07:32:44 UTC 2016"
#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z)
#define CURL_AT_LEAST_VERSION(x,y,z) \
(LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z))
#endif /* __CURL_CURLVER_H */

102
deps/libcurl/include/curl/easy.h vendored Normal file
View File

@ -0,0 +1,102 @@
#ifndef __CURL_EASY_H
#define __CURL_EASY_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#ifdef __cplusplus
extern "C" {
#endif
CURL_EXTERN CURL *curl_easy_init(void);
CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
/*
* NAME curl_easy_getinfo()
*
* DESCRIPTION
*
* Request internal information from the curl session with this function. The
* third argument MUST be a pointer to a long, a pointer to a char * or a
* pointer to a double (as the documentation describes elsewhere). The data
* pointed to will be filled in accordingly and can be relied upon only if the
* function returns CURLE_OK. This function is intended to get used *AFTER* a
* performed transfer, all results from this function are undefined until the
* transfer is completed.
*/
CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...);
/*
* NAME curl_easy_duphandle()
*
* DESCRIPTION
*
* Creates a new curl session handle with the same options set for the handle
* passed in. Duplicating a handle could only be a matter of cloning data and
* options, internal state info and things like persistent connections cannot
* be transferred. It is useful in multithreaded applications when you can run
* curl_easy_duphandle() for each new thread to avoid a series of identical
* curl_easy_setopt() invokes in every thread.
*/
CURL_EXTERN CURL* curl_easy_duphandle(CURL *curl);
/*
* NAME curl_easy_reset()
*
* DESCRIPTION
*
* Re-initializes a CURL handle to the default values. This puts back the
* handle to the same state as it was in when it was just created.
*
* It does keep: live connections, the Session ID cache, the DNS cache and the
* cookies.
*/
CURL_EXTERN void curl_easy_reset(CURL *curl);
/*
* NAME curl_easy_recv()
*
* DESCRIPTION
*
* Receives data from the connected socket. Use after successful
* curl_easy_perform() with CURLOPT_CONNECT_ONLY option.
*/
CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen,
size_t *n);
/*
* NAME curl_easy_send()
*
* DESCRIPTION
*
* Sends data over the connected socket. Use after successful
* curl_easy_perform() with CURLOPT_CONNECT_ONLY option.
*/
CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer,
size_t buflen, size_t *n);
#ifdef __cplusplus
}
#endif
#endif

74
deps/libcurl/include/curl/mprintf.h vendored Normal file
View File

@ -0,0 +1,74 @@
#ifndef __CURL_MPRINTF_H
#define __CURL_MPRINTF_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include <stdarg.h>
#include <stdio.h> /* needed for FILE */
#include "curl.h"
#ifdef __cplusplus
extern "C" {
#endif
CURL_EXTERN int curl_mprintf(const char *format, ...);
CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...);
CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...);
CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength,
const char *format, ...);
CURL_EXTERN int curl_mvprintf(const char *format, va_list args);
CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args);
CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args);
CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength,
const char *format, va_list args);
CURL_EXTERN char *curl_maprintf(const char *format, ...);
CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args);
#ifdef _MPRINTF_REPLACE
# undef printf
# undef fprintf
# undef sprintf
# undef vsprintf
# undef snprintf
# undef vprintf
# undef vfprintf
# undef vsnprintf
# undef aprintf
# undef vaprintf
# define printf curl_mprintf
# define fprintf curl_mfprintf
# define sprintf curl_msprintf
# define vsprintf curl_mvsprintf
# define snprintf curl_msnprintf
# define vprintf curl_mvprintf
# define vfprintf curl_mvfprintf
# define vsnprintf curl_mvsnprintf
# define aprintf curl_maprintf
# define vaprintf curl_mvaprintf
#endif
#ifdef __cplusplus
}
#endif
#endif /* __CURL_MPRINTF_H */

435
deps/libcurl/include/curl/multi.h vendored Normal file
View File

@ -0,0 +1,435 @@
#ifndef __CURL_MULTI_H
#define __CURL_MULTI_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
/*
This is an "external" header file. Don't give away any internals here!
GOALS
o Enable a "pull" interface. The application that uses libcurl decides where
and when to ask libcurl to get/send data.
o Enable multiple simultaneous transfers in the same thread without making it
complicated for the application.
o Enable the application to select() on its own file descriptors and curl's
file descriptors simultaneous easily.
*/
/*
* This header file should not really need to include "curl.h" since curl.h
* itself includes this file and we expect user applications to do #include
* <curl/curl.h> without the need for especially including multi.h.
*
* For some reason we added this include here at one point, and rather than to
* break existing (wrongly written) libcurl applications, we leave it as-is
* but with this warning attached.
*/
#include "curl.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef void CURLM;
typedef enum {
CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or
curl_multi_socket*() soon */
CURLM_OK,
CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */
CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */
CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */
CURLM_INTERNAL_ERROR, /* this is a libcurl bug */
CURLM_BAD_SOCKET, /* the passed in socket argument did not match */
CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */
CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was
attempted to get added - again */
CURLM_LAST
} CURLMcode;
/* just to make code nicer when using curl_multi_socket() you can now check
for CURLM_CALL_MULTI_SOCKET too in the same style it works for
curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */
#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM
/* bitmask bits for CURLMOPT_PIPELINING */
#define CURLPIPE_NOTHING 0L
#define CURLPIPE_HTTP1 1L
#define CURLPIPE_MULTIPLEX 2L
typedef enum {
CURLMSG_NONE, /* first, not used */
CURLMSG_DONE, /* This easy handle has completed. 'result' contains
the CURLcode of the transfer */
CURLMSG_LAST /* last, not used */
} CURLMSG;
struct CURLMsg {
CURLMSG msg; /* what this message means */
CURL *easy_handle; /* the handle it concerns */
union {
void *whatever; /* message-specific data */
CURLcode result; /* return code for transfer */
} data;
};
typedef struct CURLMsg CURLMsg;
/* Based on poll(2) structure and values.
* We don't use pollfd and POLL* constants explicitly
* to cover platforms without poll(). */
#define CURL_WAIT_POLLIN 0x0001
#define CURL_WAIT_POLLPRI 0x0002
#define CURL_WAIT_POLLOUT 0x0004
struct curl_waitfd {
curl_socket_t fd;
short events;
short revents; /* not supported yet */
};
/*
* Name: curl_multi_init()
*
* Desc: inititalize multi-style curl usage
*
* Returns: a new CURLM handle to use in all 'curl_multi' functions.
*/
CURL_EXTERN CURLM *curl_multi_init(void);
/*
* Name: curl_multi_add_handle()
*
* Desc: add a standard curl handle to the multi stack
*
* Returns: CURLMcode type, general multi error code.
*/
CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle,
CURL *curl_handle);
/*
* Name: curl_multi_remove_handle()
*
* Desc: removes a curl handle from the multi stack again
*
* Returns: CURLMcode type, general multi error code.
*/
CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle,
CURL *curl_handle);
/*
* Name: curl_multi_fdset()
*
* Desc: Ask curl for its fd_set sets. The app can use these to select() or
* poll() on. We want curl_multi_perform() called as soon as one of
* them are ready.
*
* Returns: CURLMcode type, general multi error code.
*/
CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle,
fd_set *read_fd_set,
fd_set *write_fd_set,
fd_set *exc_fd_set,
int *max_fd);
/*
* Name: curl_multi_wait()
*
* Desc: Poll on all fds within a CURLM set as well as any
* additional fds passed to the function.
*
* Returns: CURLMcode type, general multi error code.
*/
CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle,
struct curl_waitfd extra_fds[],
unsigned int extra_nfds,
int timeout_ms,
int *ret);
/*
* Name: curl_multi_perform()
*
* Desc: When the app thinks there's data available for curl it calls this
* function to read/write whatever there is right now. This returns
* as soon as the reads and writes are done. This function does not
* require that there actually is data available for reading or that
* data can be written, it can be called just in case. It returns
* the number of handles that still transfer data in the second
* argument's integer-pointer.
*
* Returns: CURLMcode type, general multi error code. *NOTE* that this only
* returns errors etc regarding the whole multi stack. There might
* still have occurred problems on invidual transfers even when this
* returns OK.
*/
CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle,
int *running_handles);
/*
* Name: curl_multi_cleanup()
*
* Desc: Cleans up and removes a whole multi stack. It does not free or
* touch any individual easy handles in any way. We need to define
* in what state those handles will be if this function is called
* in the middle of a transfer.
*
* Returns: CURLMcode type, general multi error code.
*/
CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle);
/*
* Name: curl_multi_info_read()
*
* Desc: Ask the multi handle if there's any messages/informationals from
* the individual transfers. Messages include informationals such as
* error code from the transfer or just the fact that a transfer is
* completed. More details on these should be written down as well.
*
* Repeated calls to this function will return a new struct each
* time, until a special "end of msgs" struct is returned as a signal
* that there is no more to get at this point.
*
* The data the returned pointer points to will not survive calling
* curl_multi_cleanup().
*
* The 'CURLMsg' struct is meant to be very simple and only contain
* very basic informations. If more involved information is wanted,
* we will provide the particular "transfer handle" in that struct
* and that should/could/would be used in subsequent
* curl_easy_getinfo() calls (or similar). The point being that we
* must never expose complex structs to applications, as then we'll
* undoubtably get backwards compatibility problems in the future.
*
* Returns: A pointer to a filled-in struct, or NULL if it failed or ran out
* of structs. It also writes the number of messages left in the
* queue (after this read) in the integer the second argument points
* to.
*/
CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle,
int *msgs_in_queue);
/*
* Name: curl_multi_strerror()
*
* Desc: The curl_multi_strerror function may be used to turn a CURLMcode
* value into the equivalent human readable error string. This is
* useful for printing meaningful error messages.
*
* Returns: A pointer to a zero-terminated error message.
*/
CURL_EXTERN const char *curl_multi_strerror(CURLMcode);
/*
* Name: curl_multi_socket() and
* curl_multi_socket_all()
*
* Desc: An alternative version of curl_multi_perform() that allows the
* application to pass in one of the file descriptors that have been
* detected to have "action" on them and let libcurl perform.
* See man page for details.
*/
#define CURL_POLL_NONE 0
#define CURL_POLL_IN 1
#define CURL_POLL_OUT 2
#define CURL_POLL_INOUT 3
#define CURL_POLL_REMOVE 4
#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD
#define CURL_CSELECT_IN 0x01
#define CURL_CSELECT_OUT 0x02
#define CURL_CSELECT_ERR 0x04
typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */
curl_socket_t s, /* socket */
int what, /* see above */
void *userp, /* private callback
pointer */
void *socketp); /* private socket
pointer */
/*
* Name: curl_multi_timer_callback
*
* Desc: Called by libcurl whenever the library detects a change in the
* maximum number of milliseconds the app is allowed to wait before
* curl_multi_socket() or curl_multi_perform() must be called
* (to allow libcurl's timed events to take place).
*
* Returns: The callback should return zero.
*/
typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */
long timeout_ms, /* see above */
void *userp); /* private callback
pointer */
CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s,
int *running_handles);
CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle,
curl_socket_t s,
int ev_bitmask,
int *running_handles);
CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle,
int *running_handles);
#ifndef CURL_ALLOW_OLD_MULTI_SOCKET
/* This macro below was added in 7.16.3 to push users who recompile to use
the new curl_multi_socket_action() instead of the old curl_multi_socket()
*/
#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z)
#endif
/*
* Name: curl_multi_timeout()
*
* Desc: Returns the maximum number of milliseconds the app is allowed to
* wait before curl_multi_socket() or curl_multi_perform() must be
* called (to allow libcurl's timed events to take place).
*
* Returns: CURLM error code.
*/
CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle,
long *milliseconds);
#undef CINIT /* re-using the same name as in curl.h */
#ifdef CURL_ISOCPP
#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num
#else
/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */
#define LONG CURLOPTTYPE_LONG
#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT
#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT
#define OFF_T CURLOPTTYPE_OFF_T
#define CINIT(name,type,number) CURLMOPT_/**/name = type + number
#endif
typedef enum {
/* This is the socket callback function pointer */
CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1),
/* This is the argument passed to the socket callback */
CINIT(SOCKETDATA, OBJECTPOINT, 2),
/* set to 1 to enable pipelining for this multi handle */
CINIT(PIPELINING, LONG, 3),
/* This is the timer callback function pointer */
CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4),
/* This is the argument passed to the timer callback */
CINIT(TIMERDATA, OBJECTPOINT, 5),
/* maximum number of entries in the connection cache */
CINIT(MAXCONNECTS, LONG, 6),
/* maximum number of (pipelining) connections to one host */
CINIT(MAX_HOST_CONNECTIONS, LONG, 7),
/* maximum number of requests in a pipeline */
CINIT(MAX_PIPELINE_LENGTH, LONG, 8),
/* a connection with a content-length longer than this
will not be considered for pipelining */
CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9),
/* a connection with a chunk length longer than this
will not be considered for pipelining */
CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10),
/* a list of site names(+port) that are blacklisted from
pipelining */
CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11),
/* a list of server types that are blacklisted from
pipelining */
CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12),
/* maximum number of open connections in total */
CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13),
/* This is the server push callback function pointer */
CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14),
/* This is the argument passed to the server push callback */
CINIT(PUSHDATA, OBJECTPOINT, 15),
CURLMOPT_LASTENTRY /* the last unused */
} CURLMoption;
/*
* Name: curl_multi_setopt()
*
* Desc: Sets options for the multi handle.
*
* Returns: CURLM error code.
*/
CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle,
CURLMoption option, ...);
/*
* Name: curl_multi_assign()
*
* Desc: This function sets an association in the multi handle between the
* given socket and a private pointer of the application. This is
* (only) useful for curl_multi_socket uses.
*
* Returns: CURLM error code.
*/
CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle,
curl_socket_t sockfd, void *sockp);
/*
* Name: curl_push_callback
*
* Desc: This callback gets called when a new stream is being pushed by the
* server. It approves or denies the new stream.
*
* Returns: CURL_PUSH_OK or CURL_PUSH_DENY.
*/
#define CURL_PUSH_OK 0
#define CURL_PUSH_DENY 1
struct curl_pushheaders; /* forward declaration only */
CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h,
size_t num);
CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h,
const char *name);
typedef int (*curl_push_callback)(CURL *parent,
CURL *easy,
size_t num_headers,
struct curl_pushheaders *headers,
void *userp);
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif

33
deps/libcurl/include/curl/stdcheaders.h vendored Normal file
View File

@ -0,0 +1,33 @@
#ifndef __STDC_HEADERS_H
#define __STDC_HEADERS_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2010, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include <sys/types.h>
size_t fread (void *, size_t, size_t, FILE *);
size_t fwrite (const void *, size_t, size_t, FILE *);
int strcasecmp(const char *, const char *);
int strncasecmp(const char *, const char *, size_t);
#endif /* __STDC_HEADERS_H */

622
deps/libcurl/include/curl/typecheck-gcc.h vendored Normal file
View File

@ -0,0 +1,622 @@
#ifndef __CURL_TYPECHECK_GCC_H
#define __CURL_TYPECHECK_GCC_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
/* wraps curl_easy_setopt() with typechecking */
/* To add a new kind of warning, add an
* if(_curl_is_sometype_option(_curl_opt))
* if(!_curl_is_sometype(value))
* _curl_easy_setopt_err_sometype();
* block and define _curl_is_sometype_option, _curl_is_sometype and
* _curl_easy_setopt_err_sometype below
*
* NOTE: We use two nested 'if' statements here instead of the && operator, in
* order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x
* when compiling with -Wlogical-op.
*
* To add an option that uses the same type as an existing option, you'll just
* need to extend the appropriate _curl_*_option macro
*/
#define curl_easy_setopt(handle, option, value) \
__extension__ ({ \
__typeof__ (option) _curl_opt = option; \
if(__builtin_constant_p(_curl_opt)) { \
if(_curl_is_long_option(_curl_opt)) \
if(!_curl_is_long(value)) \
_curl_easy_setopt_err_long(); \
if(_curl_is_off_t_option(_curl_opt)) \
if(!_curl_is_off_t(value)) \
_curl_easy_setopt_err_curl_off_t(); \
if(_curl_is_string_option(_curl_opt)) \
if(!_curl_is_string(value)) \
_curl_easy_setopt_err_string(); \
if(_curl_is_write_cb_option(_curl_opt)) \
if(!_curl_is_write_cb(value)) \
_curl_easy_setopt_err_write_callback(); \
if((_curl_opt) == CURLOPT_READFUNCTION) \
if(!_curl_is_read_cb(value)) \
_curl_easy_setopt_err_read_cb(); \
if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \
if(!_curl_is_ioctl_cb(value)) \
_curl_easy_setopt_err_ioctl_cb(); \
if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \
if(!_curl_is_sockopt_cb(value)) \
_curl_easy_setopt_err_sockopt_cb(); \
if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \
if(!_curl_is_opensocket_cb(value)) \
_curl_easy_setopt_err_opensocket_cb(); \
if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \
if(!_curl_is_progress_cb(value)) \
_curl_easy_setopt_err_progress_cb(); \
if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \
if(!_curl_is_debug_cb(value)) \
_curl_easy_setopt_err_debug_cb(); \
if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \
if(!_curl_is_ssl_ctx_cb(value)) \
_curl_easy_setopt_err_ssl_ctx_cb(); \
if(_curl_is_conv_cb_option(_curl_opt)) \
if(!_curl_is_conv_cb(value)) \
_curl_easy_setopt_err_conv_cb(); \
if((_curl_opt) == CURLOPT_SEEKFUNCTION) \
if(!_curl_is_seek_cb(value)) \
_curl_easy_setopt_err_seek_cb(); \
if(_curl_is_cb_data_option(_curl_opt)) \
if(!_curl_is_cb_data(value)) \
_curl_easy_setopt_err_cb_data(); \
if((_curl_opt) == CURLOPT_ERRORBUFFER) \
if(!_curl_is_error_buffer(value)) \
_curl_easy_setopt_err_error_buffer(); \
if((_curl_opt) == CURLOPT_STDERR) \
if(!_curl_is_FILE(value)) \
_curl_easy_setopt_err_FILE(); \
if(_curl_is_postfields_option(_curl_opt)) \
if(!_curl_is_postfields(value)) \
_curl_easy_setopt_err_postfields(); \
if((_curl_opt) == CURLOPT_HTTPPOST) \
if(!_curl_is_arr((value), struct curl_httppost)) \
_curl_easy_setopt_err_curl_httpost(); \
if(_curl_is_slist_option(_curl_opt)) \
if(!_curl_is_arr((value), struct curl_slist)) \
_curl_easy_setopt_err_curl_slist(); \
if((_curl_opt) == CURLOPT_SHARE) \
if(!_curl_is_ptr((value), CURLSH)) \
_curl_easy_setopt_err_CURLSH(); \
} \
curl_easy_setopt(handle, _curl_opt, value); \
})
/* wraps curl_easy_getinfo() with typechecking */
/* FIXME: don't allow const pointers */
#define curl_easy_getinfo(handle, info, arg) \
__extension__ ({ \
__typeof__ (info) _curl_info = info; \
if(__builtin_constant_p(_curl_info)) { \
if(_curl_is_string_info(_curl_info)) \
if(!_curl_is_arr((arg), char *)) \
_curl_easy_getinfo_err_string(); \
if(_curl_is_long_info(_curl_info)) \
if(!_curl_is_arr((arg), long)) \
_curl_easy_getinfo_err_long(); \
if(_curl_is_double_info(_curl_info)) \
if(!_curl_is_arr((arg), double)) \
_curl_easy_getinfo_err_double(); \
if(_curl_is_slist_info(_curl_info)) \
if(!_curl_is_arr((arg), struct curl_slist *)) \
_curl_easy_getinfo_err_curl_slist(); \
} \
curl_easy_getinfo(handle, _curl_info, arg); \
})
/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(),
* for now just make sure that the functions are called with three
* arguments
*/
#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param)
#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param)
/* the actual warnings, triggered by calling the _curl_easy_setopt_err*
* functions */
/* To define a new warning, use _CURL_WARNING(identifier, "message") */
#define _CURL_WARNING(id, message) \
static void __attribute__((__warning__(message))) \
__attribute__((__unused__)) __attribute__((__noinline__)) \
id(void) { __asm__(""); }
_CURL_WARNING(_curl_easy_setopt_err_long,
"curl_easy_setopt expects a long argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_curl_off_t,
"curl_easy_setopt expects a curl_off_t argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_string,
"curl_easy_setopt expects a "
"string (char* or char[]) argument for this option"
)
_CURL_WARNING(_curl_easy_setopt_err_write_callback,
"curl_easy_setopt expects a curl_write_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_read_cb,
"curl_easy_setopt expects a curl_read_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb,
"curl_easy_setopt expects a curl_ioctl_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb,
"curl_easy_setopt expects a curl_sockopt_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb,
"curl_easy_setopt expects a "
"curl_opensocket_callback argument for this option"
)
_CURL_WARNING(_curl_easy_setopt_err_progress_cb,
"curl_easy_setopt expects a curl_progress_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_debug_cb,
"curl_easy_setopt expects a curl_debug_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb,
"curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_conv_cb,
"curl_easy_setopt expects a curl_conv_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_seek_cb,
"curl_easy_setopt expects a curl_seek_callback argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_cb_data,
"curl_easy_setopt expects a "
"private data pointer as argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_error_buffer,
"curl_easy_setopt expects a "
"char buffer of CURL_ERROR_SIZE as argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_FILE,
"curl_easy_setopt expects a FILE* argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_postfields,
"curl_easy_setopt expects a void* or char* argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_curl_httpost,
"curl_easy_setopt expects a struct curl_httppost* argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_curl_slist,
"curl_easy_setopt expects a struct curl_slist* argument for this option")
_CURL_WARNING(_curl_easy_setopt_err_CURLSH,
"curl_easy_setopt expects a CURLSH* argument for this option")
_CURL_WARNING(_curl_easy_getinfo_err_string,
"curl_easy_getinfo expects a pointer to char * for this info")
_CURL_WARNING(_curl_easy_getinfo_err_long,
"curl_easy_getinfo expects a pointer to long for this info")
_CURL_WARNING(_curl_easy_getinfo_err_double,
"curl_easy_getinfo expects a pointer to double for this info")
_CURL_WARNING(_curl_easy_getinfo_err_curl_slist,
"curl_easy_getinfo expects a pointer to struct curl_slist * for this info")
/* groups of curl_easy_setops options that take the same type of argument */
/* To add a new option to one of the groups, just add
* (option) == CURLOPT_SOMETHING
* to the or-expression. If the option takes a long or curl_off_t, you don't
* have to do anything
*/
/* evaluates to true if option takes a long argument */
#define _curl_is_long_option(option) \
(0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT)
#define _curl_is_off_t_option(option) \
((option) > CURLOPTTYPE_OFF_T)
/* evaluates to true if option takes a char* argument */
#define _curl_is_string_option(option) \
((option) == CURLOPT_ACCEPT_ENCODING || \
(option) == CURLOPT_CAINFO || \
(option) == CURLOPT_CAPATH || \
(option) == CURLOPT_COOKIE || \
(option) == CURLOPT_COOKIEFILE || \
(option) == CURLOPT_COOKIEJAR || \
(option) == CURLOPT_COOKIELIST || \
(option) == CURLOPT_CRLFILE || \
(option) == CURLOPT_CUSTOMREQUEST || \
(option) == CURLOPT_DEFAULT_PROTOCOL || \
(option) == CURLOPT_DNS_INTERFACE || \
(option) == CURLOPT_DNS_LOCAL_IP4 || \
(option) == CURLOPT_DNS_LOCAL_IP6 || \
(option) == CURLOPT_DNS_SERVERS || \
(option) == CURLOPT_EGDSOCKET || \
(option) == CURLOPT_FTPPORT || \
(option) == CURLOPT_FTP_ACCOUNT || \
(option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \
(option) == CURLOPT_INTERFACE || \
(option) == CURLOPT_ISSUERCERT || \
(option) == CURLOPT_KEYPASSWD || \
(option) == CURLOPT_KRBLEVEL || \
(option) == CURLOPT_LOGIN_OPTIONS || \
(option) == CURLOPT_MAIL_AUTH || \
(option) == CURLOPT_MAIL_FROM || \
(option) == CURLOPT_NETRC_FILE || \
(option) == CURLOPT_NOPROXY || \
(option) == CURLOPT_PASSWORD || \
(option) == CURLOPT_PINNEDPUBLICKEY || \
(option) == CURLOPT_PROXY || \
(option) == CURLOPT_PROXYPASSWORD || \
(option) == CURLOPT_PROXYUSERNAME || \
(option) == CURLOPT_PROXYUSERPWD || \
(option) == CURLOPT_PROXY_SERVICE_NAME || \
(option) == CURLOPT_RANDOM_FILE || \
(option) == CURLOPT_RANGE || \
(option) == CURLOPT_REFERER || \
(option) == CURLOPT_RTSP_SESSION_ID || \
(option) == CURLOPT_RTSP_STREAM_URI || \
(option) == CURLOPT_RTSP_TRANSPORT || \
(option) == CURLOPT_SERVICE_NAME || \
(option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \
(option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \
(option) == CURLOPT_SSH_KNOWNHOSTS || \
(option) == CURLOPT_SSH_PRIVATE_KEYFILE || \
(option) == CURLOPT_SSH_PUBLIC_KEYFILE || \
(option) == CURLOPT_SSLCERT || \
(option) == CURLOPT_SSLCERTTYPE || \
(option) == CURLOPT_SSLENGINE || \
(option) == CURLOPT_SSLKEY || \
(option) == CURLOPT_SSLKEYTYPE || \
(option) == CURLOPT_SSL_CIPHER_LIST || \
(option) == CURLOPT_TLSAUTH_PASSWORD || \
(option) == CURLOPT_TLSAUTH_TYPE || \
(option) == CURLOPT_TLSAUTH_USERNAME || \
(option) == CURLOPT_UNIX_SOCKET_PATH || \
(option) == CURLOPT_URL || \
(option) == CURLOPT_USERAGENT || \
(option) == CURLOPT_USERNAME || \
(option) == CURLOPT_USERPWD || \
(option) == CURLOPT_XOAUTH2_BEARER || \
0)
/* evaluates to true if option takes a curl_write_callback argument */
#define _curl_is_write_cb_option(option) \
((option) == CURLOPT_HEADERFUNCTION || \
(option) == CURLOPT_WRITEFUNCTION)
/* evaluates to true if option takes a curl_conv_callback argument */
#define _curl_is_conv_cb_option(option) \
((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \
(option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \
(option) == CURLOPT_CONV_FROM_UTF8_FUNCTION)
/* evaluates to true if option takes a data argument to pass to a callback */
#define _curl_is_cb_data_option(option) \
((option) == CURLOPT_CHUNK_DATA || \
(option) == CURLOPT_CLOSESOCKETDATA || \
(option) == CURLOPT_DEBUGDATA || \
(option) == CURLOPT_FNMATCH_DATA || \
(option) == CURLOPT_HEADERDATA || \
(option) == CURLOPT_INTERLEAVEDATA || \
(option) == CURLOPT_IOCTLDATA || \
(option) == CURLOPT_OPENSOCKETDATA || \
(option) == CURLOPT_PRIVATE || \
(option) == CURLOPT_PROGRESSDATA || \
(option) == CURLOPT_READDATA || \
(option) == CURLOPT_SEEKDATA || \
(option) == CURLOPT_SOCKOPTDATA || \
(option) == CURLOPT_SSH_KEYDATA || \
(option) == CURLOPT_SSL_CTX_DATA || \
(option) == CURLOPT_WRITEDATA || \
0)
/* evaluates to true if option takes a POST data argument (void* or char*) */
#define _curl_is_postfields_option(option) \
((option) == CURLOPT_POSTFIELDS || \
(option) == CURLOPT_COPYPOSTFIELDS || \
0)
/* evaluates to true if option takes a struct curl_slist * argument */
#define _curl_is_slist_option(option) \
((option) == CURLOPT_HTTP200ALIASES || \
(option) == CURLOPT_HTTPHEADER || \
(option) == CURLOPT_MAIL_RCPT || \
(option) == CURLOPT_POSTQUOTE || \
(option) == CURLOPT_PREQUOTE || \
(option) == CURLOPT_PROXYHEADER || \
(option) == CURLOPT_QUOTE || \
(option) == CURLOPT_RESOLVE || \
(option) == CURLOPT_TELNETOPTIONS || \
0)
/* groups of curl_easy_getinfo infos that take the same type of argument */
/* evaluates to true if info expects a pointer to char * argument */
#define _curl_is_string_info(info) \
(CURLINFO_STRING < (info) && (info) < CURLINFO_LONG)
/* evaluates to true if info expects a pointer to long argument */
#define _curl_is_long_info(info) \
(CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE)
/* evaluates to true if info expects a pointer to double argument */
#define _curl_is_double_info(info) \
(CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST)
/* true if info expects a pointer to struct curl_slist * argument */
#define _curl_is_slist_info(info) \
(CURLINFO_SLIST < (info))
/* typecheck helpers -- check whether given expression has requested type*/
/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros,
* otherwise define a new macro. Search for __builtin_types_compatible_p
* in the GCC manual.
* NOTE: these macros MUST NOT EVALUATE their arguments! The argument is
* the actual expression passed to the curl_easy_setopt macro. This
* means that you can only apply the sizeof and __typeof__ operators, no
* == or whatsoever.
*/
/* XXX: should evaluate to true iff expr is a pointer */
#define _curl_is_any_ptr(expr) \
(sizeof(expr) == sizeof(void*))
/* evaluates to true if expr is NULL */
/* XXX: must not evaluate expr, so this check is not accurate */
#define _curl_is_NULL(expr) \
(__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL)))
/* evaluates to true if expr is type*, const type* or NULL */
#define _curl_is_ptr(expr, type) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), type *) || \
__builtin_types_compatible_p(__typeof__(expr), const type *))
/* evaluates to true if expr is one of type[], type*, NULL or const type* */
#define _curl_is_arr(expr, type) \
(_curl_is_ptr((expr), type) || \
__builtin_types_compatible_p(__typeof__(expr), type []))
/* evaluates to true if expr is a string */
#define _curl_is_string(expr) \
(_curl_is_arr((expr), char) || \
_curl_is_arr((expr), signed char) || \
_curl_is_arr((expr), unsigned char))
/* evaluates to true if expr is a long (no matter the signedness)
* XXX: for now, int is also accepted (and therefore short and char, which
* are promoted to int when passed to a variadic function) */
#define _curl_is_long(expr) \
(__builtin_types_compatible_p(__typeof__(expr), long) || \
__builtin_types_compatible_p(__typeof__(expr), signed long) || \
__builtin_types_compatible_p(__typeof__(expr), unsigned long) || \
__builtin_types_compatible_p(__typeof__(expr), int) || \
__builtin_types_compatible_p(__typeof__(expr), signed int) || \
__builtin_types_compatible_p(__typeof__(expr), unsigned int) || \
__builtin_types_compatible_p(__typeof__(expr), short) || \
__builtin_types_compatible_p(__typeof__(expr), signed short) || \
__builtin_types_compatible_p(__typeof__(expr), unsigned short) || \
__builtin_types_compatible_p(__typeof__(expr), char) || \
__builtin_types_compatible_p(__typeof__(expr), signed char) || \
__builtin_types_compatible_p(__typeof__(expr), unsigned char))
/* evaluates to true if expr is of type curl_off_t */
#define _curl_is_off_t(expr) \
(__builtin_types_compatible_p(__typeof__(expr), curl_off_t))
/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */
/* XXX: also check size of an char[] array? */
#define _curl_is_error_buffer(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), char *) || \
__builtin_types_compatible_p(__typeof__(expr), char[]))
/* evaluates to true if expr is of type (const) void* or (const) FILE* */
#if 0
#define _curl_is_cb_data(expr) \
(_curl_is_ptr((expr), void) || \
_curl_is_ptr((expr), FILE))
#else /* be less strict */
#define _curl_is_cb_data(expr) \
_curl_is_any_ptr(expr)
#endif
/* evaluates to true if expr is of type FILE* */
#define _curl_is_FILE(expr) \
(__builtin_types_compatible_p(__typeof__(expr), FILE *))
/* evaluates to true if expr can be passed as POST data (void* or char*) */
#define _curl_is_postfields(expr) \
(_curl_is_ptr((expr), void) || \
_curl_is_arr((expr), char))
/* FIXME: the whole callback checking is messy...
* The idea is to tolerate char vs. void and const vs. not const
* pointers in arguments at least
*/
/* helper: __builtin_types_compatible_p distinguishes between functions and
* function pointers, hide it */
#define _curl_callback_compatible(func, type) \
(__builtin_types_compatible_p(__typeof__(func), type) || \
__builtin_types_compatible_p(__typeof__(func), type*))
/* evaluates to true if expr is of type curl_read_callback or "similar" */
#define _curl_is_read_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), __typeof__(fread)) || \
__builtin_types_compatible_p(__typeof__(expr), curl_read_callback) || \
_curl_callback_compatible((expr), _curl_read_callback1) || \
_curl_callback_compatible((expr), _curl_read_callback2) || \
_curl_callback_compatible((expr), _curl_read_callback3) || \
_curl_callback_compatible((expr), _curl_read_callback4) || \
_curl_callback_compatible((expr), _curl_read_callback5) || \
_curl_callback_compatible((expr), _curl_read_callback6))
typedef size_t (_curl_read_callback1)(char *, size_t, size_t, void*);
typedef size_t (_curl_read_callback2)(char *, size_t, size_t, const void*);
typedef size_t (_curl_read_callback3)(char *, size_t, size_t, FILE*);
typedef size_t (_curl_read_callback4)(void *, size_t, size_t, void*);
typedef size_t (_curl_read_callback5)(void *, size_t, size_t, const void*);
typedef size_t (_curl_read_callback6)(void *, size_t, size_t, FILE*);
/* evaluates to true if expr is of type curl_write_callback or "similar" */
#define _curl_is_write_cb(expr) \
(_curl_is_read_cb(expr) || \
__builtin_types_compatible_p(__typeof__(expr), __typeof__(fwrite)) || \
__builtin_types_compatible_p(__typeof__(expr), curl_write_callback) || \
_curl_callback_compatible((expr), _curl_write_callback1) || \
_curl_callback_compatible((expr), _curl_write_callback2) || \
_curl_callback_compatible((expr), _curl_write_callback3) || \
_curl_callback_compatible((expr), _curl_write_callback4) || \
_curl_callback_compatible((expr), _curl_write_callback5) || \
_curl_callback_compatible((expr), _curl_write_callback6))
typedef size_t (_curl_write_callback1)(const char *, size_t, size_t, void*);
typedef size_t (_curl_write_callback2)(const char *, size_t, size_t,
const void*);
typedef size_t (_curl_write_callback3)(const char *, size_t, size_t, FILE*);
typedef size_t (_curl_write_callback4)(const void *, size_t, size_t, void*);
typedef size_t (_curl_write_callback5)(const void *, size_t, size_t,
const void*);
typedef size_t (_curl_write_callback6)(const void *, size_t, size_t, FILE*);
/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */
#define _curl_is_ioctl_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_ioctl_callback) || \
_curl_callback_compatible((expr), _curl_ioctl_callback1) || \
_curl_callback_compatible((expr), _curl_ioctl_callback2) || \
_curl_callback_compatible((expr), _curl_ioctl_callback3) || \
_curl_callback_compatible((expr), _curl_ioctl_callback4))
typedef curlioerr (_curl_ioctl_callback1)(CURL *, int, void*);
typedef curlioerr (_curl_ioctl_callback2)(CURL *, int, const void*);
typedef curlioerr (_curl_ioctl_callback3)(CURL *, curliocmd, void*);
typedef curlioerr (_curl_ioctl_callback4)(CURL *, curliocmd, const void*);
/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */
#define _curl_is_sockopt_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_sockopt_callback) || \
_curl_callback_compatible((expr), _curl_sockopt_callback1) || \
_curl_callback_compatible((expr), _curl_sockopt_callback2))
typedef int (_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype);
typedef int (_curl_sockopt_callback2)(const void *, curl_socket_t,
curlsocktype);
/* evaluates to true if expr is of type curl_opensocket_callback or
"similar" */
#define _curl_is_opensocket_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_opensocket_callback) ||\
_curl_callback_compatible((expr), _curl_opensocket_callback1) || \
_curl_callback_compatible((expr), _curl_opensocket_callback2) || \
_curl_callback_compatible((expr), _curl_opensocket_callback3) || \
_curl_callback_compatible((expr), _curl_opensocket_callback4))
typedef curl_socket_t (_curl_opensocket_callback1)
(void *, curlsocktype, struct curl_sockaddr *);
typedef curl_socket_t (_curl_opensocket_callback2)
(void *, curlsocktype, const struct curl_sockaddr *);
typedef curl_socket_t (_curl_opensocket_callback3)
(const void *, curlsocktype, struct curl_sockaddr *);
typedef curl_socket_t (_curl_opensocket_callback4)
(const void *, curlsocktype, const struct curl_sockaddr *);
/* evaluates to true if expr is of type curl_progress_callback or "similar" */
#define _curl_is_progress_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_progress_callback) || \
_curl_callback_compatible((expr), _curl_progress_callback1) || \
_curl_callback_compatible((expr), _curl_progress_callback2))
typedef int (_curl_progress_callback1)(void *,
double, double, double, double);
typedef int (_curl_progress_callback2)(const void *,
double, double, double, double);
/* evaluates to true if expr is of type curl_debug_callback or "similar" */
#define _curl_is_debug_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_debug_callback) || \
_curl_callback_compatible((expr), _curl_debug_callback1) || \
_curl_callback_compatible((expr), _curl_debug_callback2) || \
_curl_callback_compatible((expr), _curl_debug_callback3) || \
_curl_callback_compatible((expr), _curl_debug_callback4) || \
_curl_callback_compatible((expr), _curl_debug_callback5) || \
_curl_callback_compatible((expr), _curl_debug_callback6) || \
_curl_callback_compatible((expr), _curl_debug_callback7) || \
_curl_callback_compatible((expr), _curl_debug_callback8))
typedef int (_curl_debug_callback1) (CURL *,
curl_infotype, char *, size_t, void *);
typedef int (_curl_debug_callback2) (CURL *,
curl_infotype, char *, size_t, const void *);
typedef int (_curl_debug_callback3) (CURL *,
curl_infotype, const char *, size_t, void *);
typedef int (_curl_debug_callback4) (CURL *,
curl_infotype, const char *, size_t, const void *);
typedef int (_curl_debug_callback5) (CURL *,
curl_infotype, unsigned char *, size_t, void *);
typedef int (_curl_debug_callback6) (CURL *,
curl_infotype, unsigned char *, size_t, const void *);
typedef int (_curl_debug_callback7) (CURL *,
curl_infotype, const unsigned char *, size_t, void *);
typedef int (_curl_debug_callback8) (CURL *,
curl_infotype, const unsigned char *, size_t, const void *);
/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */
/* this is getting even messier... */
#define _curl_is_ssl_ctx_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_ssl_ctx_callback) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \
_curl_callback_compatible((expr), _curl_ssl_ctx_callback8))
typedef CURLcode (_curl_ssl_ctx_callback1)(CURL *, void *, void *);
typedef CURLcode (_curl_ssl_ctx_callback2)(CURL *, void *, const void *);
typedef CURLcode (_curl_ssl_ctx_callback3)(CURL *, const void *, void *);
typedef CURLcode (_curl_ssl_ctx_callback4)(CURL *, const void *, const void *);
#ifdef HEADER_SSL_H
/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX
* this will of course break if we're included before OpenSSL headers...
*/
typedef CURLcode (_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *);
typedef CURLcode (_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *);
typedef CURLcode (_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *);
typedef CURLcode (_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX,
const void *);
#else
typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5;
typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6;
typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7;
typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8;
#endif
/* evaluates to true if expr is of type curl_conv_callback or "similar" */
#define _curl_is_conv_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_conv_callback) || \
_curl_callback_compatible((expr), _curl_conv_callback1) || \
_curl_callback_compatible((expr), _curl_conv_callback2) || \
_curl_callback_compatible((expr), _curl_conv_callback3) || \
_curl_callback_compatible((expr), _curl_conv_callback4))
typedef CURLcode (*_curl_conv_callback1)(char *, size_t length);
typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length);
typedef CURLcode (*_curl_conv_callback3)(void *, size_t length);
typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length);
/* evaluates to true if expr is of type curl_seek_callback or "similar" */
#define _curl_is_seek_cb(expr) \
(_curl_is_NULL(expr) || \
__builtin_types_compatible_p(__typeof__(expr), curl_seek_callback) || \
_curl_callback_compatible((expr), _curl_seek_callback1) || \
_curl_callback_compatible((expr), _curl_seek_callback2))
typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int);
typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int);
#endif /* __CURL_TYPECHECK_GCC_H */

BIN
deps/libcurl/lib/libcurl.a vendored Normal file
View File

Binary file not shown.

BIN
documentation20/webdocs/assets/connector.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 538 KiB

After

Width:  |  Height:  |  Size: 85 KiB

17
documentation20/webdocs/markdowndocs/Documentation-ch.md
View File

@ -128,5 +128,18 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
## [培训和FAQ](https://www.taosdata.com/cn/faq)
- [FAQ](https://www.taosdata.com/cn/documentation20/faq)：常见问题与答案
- [应用案列](https://www.taosdata.com/cn/blog/?categories=4)：一些使用实例来解释如何使用TDengine
<ul>
<li><a l href="https://www.taosdata.com/blog/2020/12/25/2126.html">技术公开课：开源、高效的物联网大数据平台，TDengine内核技术剖析</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1941.html">TDengine视频教程-快速上手</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1945.html">TDengine视频教程-数据建模</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1961.html">TDengine视频教程-集群搭建</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1951.html">TDengine视频教程-Go Connector</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1955.html">TDengine视频教程-JDBC Connector</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1957.html">TDengine视频教程-NodeJS Connector</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1963.html">TDengine视频教程-Python Connector</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1965.html">TDengine视频教程-RESTful Connector</a></li>
<li><a l href="https://www.taosdata.com/blog/2020/11/11/1959.html">TDengine视频教程-“零”代码运维监控</a></li>
<li><a l href="https://www.taosdata.com/cn/documentation20/faq">FAQ：常见问题与答案</a></li>
<li><a l href="https://www.taosdata.com/cn/blog/?categories=4"> 应用案例：一些使用实例来解释如何使用TDengine</a></li>
</ul>

2
documentation20/webdocs/markdowndocs/Evaluation-ch.md
View File

@ -59,5 +59,3 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的
|要求运维学习成本可控| | | √ |同上。|
|要求市场有大量人才储备| √ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。|
## TDengine 性能指标介绍和验证方法

169
documentation20/webdocs/markdowndocs/Getting Started-ch.md
View File

@ -2,35 +2,7 @@
## 快捷安装
TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。
**应用驱动**
如果应用在Windows和Linux上运行，可使用C/C++/C#/JAVA/Python/Go/Node.js接口连接服务器。如果应用在Mac上运行，目前可以使用RESTful接口连接服务器。
**CPU**
CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。
**服务器**
目前TDengine服务器可以运行在以下平台上：
| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **Win64/32** | **macOS** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** |
| -------------- | --------------------- | ------------------------ | --------------- | ------------ | --------- | --------------- | ------------------------- | --------------------- |
| X64 | ● | ● | | ○/○ | ○ | ○ | ● | ● |
| 树莓派ARM32 | | ● | ● | | | | | |
| 龙芯MIPS64 | | | ● | | | | | |
| 鲲鹏 ARM64 | | ○ | ○ | | | | ● | |
| 申威 Alpha64 | | | ○ | | | ● | | |
| 飞腾ARM64 | | | ○优麒麟 | | | | | |
| 海光X64 | ● | ● | ● | | | ○ | ● | ● |
| 瑞芯微ARM64/32 | | | ○ | | | | | |
| 全志ARM64/32 | | | ○ | | | | | |
| 炬力ARM64/32 | | | ○ | | | | | |
| TI ARM32 | | | ○ | | | | | |
TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。
### 通过源码安装
@ -42,42 +14,25 @@ CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。
### 通过安装包安装
服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。
TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择：
- TDengine-server-2.0.9.0-Linux-x64.rpm (4.2M)
- TDengine-server-2.0.9.0-Linux-x64.deb (2.7M)
- TDengine-server-2.0.9.0-Linux-x64.tar.gz (4.5M)
- TDengine-server-2.0.10.0-Linux-x64.rpm (4.2M)
- TDengine-server-2.0.10.0-Linux-x64.deb (2.7M)
- TDengine-server-2.0.10.0-Linux-x64.tar.gz (4.5M)
客户端部分，Linux安装包如下：
- TDengine-client-2.0.9.0-Linux-x64.tar.gz(3.0M)
- TDengine-client-2.0.9.0-Windows-x64.exe(2.8M)
- TDengine-client-2.0.9.0-Windows-x86.exe(2.8M)
报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）：
- TDengine-alert-2.0.9.0-Linux-x64.tar.gz (8.1M)
目前，TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包:
```cmd
which systemctl
```
具体的安装过程，请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>。
具体的安装过程，请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>以及<a href="https://www.taosdata.com/blog/2020/11/11/1941.html">视频教程</a>。
## 轻松启动
安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。
```cmd
systemctl start taosd
```bash
$ systemctl start taosd
```
检查服务是否正常工作。
```cmd
systemctl status taosd
```bash
$ systemctl status taosd
```
如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。
@ -86,15 +41,24 @@ systemctl status taosd
- systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo
- 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。
- TDengine采用FQDN(一般就是hostname)作为节点的ID，为保证正常运行，需要给运行taosd的服务器配置好hostname，在客户端应用运行的机器配置好DNS服务或hosts文件，保证FQDN能够解析。
* TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包:
```bash
$ which systemctl
```
如果系统中不支持systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。
如果系统中不支持`systemd`，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。
## TDengine命令行程序
执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。
```cmd
taos
```bash
$ taos
```
如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下：
@ -134,15 +98,15 @@ Query OK, 2 row(s) in set (0.001700s)
示例：
```cmd
taos -h 192.168.0.1 -s "use db; show tables;"
```bash
$ taos -h 192.168.0.1 -s "use db; show tables;"
```
### 运行SQL命令脚本
TDengine终端可以通过`source`命令来运行SQL命令脚本.
```
```mysql
taos> source <filename>;
```
@ -157,8 +121,8 @@ taos> source <filename>;
启动TDengine的服务，在Linux终端执行taosdemo
```
> taosdemo
```bash
$ taosdemo
```
该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。
@ -169,33 +133,92 @@ taos> source <filename>;
- 查询超级表下记录总条数：
```
taos>select count(*) from test.meters;
```mysql
taos> select count(*) from test.meters;
```
- 查询10亿条记录的平均值、最大值、最小值等：
```
taos>select avg(f1), max(f2), min(f3) from test.meters;
```mysql
taos> select avg(f1), max(f2), min(f3) from test.meters;
```
- 查询loc="beijing"的记录总条数：
```
taos>select count(*) from test.meters where loc="beijing";
```mysql
taos> select count(*) from test.meters where loc="beijing";
```
- 查询areaid=10的所有记录的平均值、最大值、最小值等：
```
taos>select avg(f1), max(f2), min(f3) from test.meters where areaid=10;
```mysql
taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10;
```
- 对表t10按10s进行平均值、最大值和最小值聚合统计：
```
taos>select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
```mysql
taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
```
**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。
## 客户端和报警模块
如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下：
- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M)
- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M)
- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M)
报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）：
- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M)
## **支持平台列表**
### TDengine服务器支持的平台列表
| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** |
| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- |
| X64 | ● | ● | | ○ | ● | ● |
| 树莓派ARM32 | | ● | ● | | | |
| 龙芯MIPS64 | | | ● | | | |
| 鲲鹏 ARM64 | | ○ | ○ | | ● | |
| 申威 Alpha64 | | | ○ | ● | | |
| 飞腾ARM64 | | ○优麒麟 | | | | |
| 海光X64 | ● | ● | ● | ○ | ● | ● |
| 瑞芯微ARM64/32 | | | ○ | | | |
| 全志ARM64/32 | | | ○ | | | |
| 炬力ARM64/32 | | | ○ | | | |
| TI ARM32 | | | ○ | | | |
注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。
### TDengine客户端和连接器支持的平台列表
目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。
对照矩阵如下：
| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** |
| ----------- | --------------- | --------- | --------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ |
| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** |
| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● |
| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- |
| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- |
| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- |
| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● |
注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。
请跳转到 [连接器 ](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。

8
documentation20/webdocs/markdowndocs/Model-ch.md
View File

@ -8,14 +8,14 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个
不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如：
```cmd
```mysql
CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4 UPDATE 1;
```
上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4，允许更新数据。详细的语法及参数请见<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL</a>
创建库之后，需要使用SQL命令USE将当前库切换过来，例如：
```cmd
```mysql
USE power;
```
@ -28,7 +28,7 @@ USE power;
## 创建超级表
一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表：
```cmd
```mysql
CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);
```
与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义。第一列必须为时间戳（示例中为ts)，其他列为采集的物理量（示例中为current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 <a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a>一节。
@ -59,3 +59,5 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21
TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。
TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型会显得简单。
关于数据建模请参考<a href="https://www.taosdata.com/blog/2020/11/11/1945.html">视频教程</a>。

31
documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
View File

@ -77,7 +77,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
SHOW VARIABLES;
```
- **使用数据库**
```mysql
@ -85,7 +84,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```
使用/切换数据库
- **删除数据库**
```mysql
DROP DATABASE [IF EXISTS] db_name;
@ -120,7 +118,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
**Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。
- **显示系统所有数据库**
```mysql
SHOW DATABASES;
@ -153,7 +150,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。
- **在线修改显示字符宽度**
```mysql
@ -234,7 +230,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```mysql
ALTER TABLE stb_name ADD TAG new_tag_name tag_type;
```
为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符.
为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符。
- **删除标签**
@ -265,28 +261,24 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```
向表tb_name中插入一条记录
- **插入一条记录，数据对应到指定的列**
```mysql
INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...)
```
向表tb_name中插入一条记录，数据对应到指定的列。SQL语句中没有出现的列，数据库将自动填充为NULL。主键（时间戳）不能为NULL。
- **插入多条记录**
```mysql
INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...;
```
向表tb_name中插入多条记录
- **按指定的列插入多条记录**
```mysql
INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...)
```
向表tb_name中按指定的列插入多条记录
- **向多个表插入多条记录**
```mysql
INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)...
@ -294,7 +286,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```
同时向表tb1_name和tb2_name中分别插入多条记录
- **同时向多个表按列插入多条记录**
```mysql
INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...)
@ -382,7 +373,6 @@ taos> SELECT * FROM meters;
Query OK, 9 row(s) in set (0.002022s)
```
通配符支持表名前缀，以下两个SQL语句均为返回全部的列：
```mysql
SELECT * FROM d1001;
@ -613,7 +603,6 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
Query OK, 1 row(s) in set (0.001075s)
```
- **AVG**
```mysql
SELECT AVG(field_name) FROM tb_name [WHERE clause];
@ -757,7 +746,6 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
Query OK, 1 row(s) in set (0.000987s)
```
- **FIRST**
```mysql
SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
@ -937,7 +925,6 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
Query OK, 2 row(s) in set (0.001162s)
```
- **SPREAD**
```mysql
SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause];
@ -962,7 +949,6 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
Query OK, 1 row(s) in set (0.000836s)
```
- **四则运算**
```mysql
@ -1010,7 +996,6 @@ SELECT function_list FROM stb_name
4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。
说明：
1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。
2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
@ -1039,3 +1024,17 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER
- 标签最多允许128个，可以0个，标签总长度不超过16k个字符
- SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M
- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制
## TAOS SQL其他约定
**group by的限制**
TAOS SQL支持对标签、tbname进行group by操作，也支持普通列进行group by，前提是：仅限一列且该列的唯一值小于10万个。
**join操作的限制**
TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持两个表之间聚合后的四则运算。
**is not null与不为空的表达式适用范围**
is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。

1
documentation20/webdocs/markdowndocs/Taos Error Code-ch.md
View File

@ -1,6 +1,5 @@
# TDengine 2.0 错误码以及对应的十进制码
| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） |
|-----------------------| :---: | :---------: | :------------------------ | ---------------- |
|TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647|

54
documentation20/webdocs/markdowndocs/administrator-ch.md
View File

@ -80,6 +80,12 @@ TDengine集群的节点数必须大于等于副本数，否则创建表时将报
TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录，可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。
另外可以使用 “-C” 显示当前服务器配置参数：
```
taosd -C
```
下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节，而且这些参数的缺省配置都是工作的，一般无需设置。**注意：配置修改后，需要重启*taosd*服务才能生效。**
- firstEp: taosd启动时，主动连接的集群中首个dnode的end point, 默认值为localhost:6030。
@ -97,6 +103,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
- telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为字节。
- ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程；最大值2表示最大建立2倍CPU核数的查询线程。默认为1，表示最大和CPU核数相等的查询线程。该值可以为小数，即0.5表示最大建立CPU核数一半的查询线程。
**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。
@ -152,7 +159,13 @@ ALTER DNODE <dnode_id> <config>
## 客户端配置
TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。
TDengine系统的前台交互客户端应用程序为taos，以及应用驱动，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见<a href="https://www.taosdata.com/cn/documentation/administrator/#_TDengine_Shell命令行程序">Shell命令行程序</a>。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。
**2.0.10.0 之后版本支持命令行以下参数显示当前客户端参数的配置**
```bash
taos -C 或 taos --dump-config
```
客户端配置参数
@ -215,15 +228,15 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同
均是合法的设置东八区时区的格式。
时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词now的解析）产生影响。例如：
```
```sql
SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08';
```
在东八区，SQL语句等效于
```
```sql
SELECT count(*) FROM table_name WHERE TS<1554955268000;
```
在UTC时区，SQL语句等效于
```
```sql
SELECT count(*) FROM table_name WHERE TS<1554984068000;
```
为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
@ -234,36 +247,35 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同
Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。
## 用户管理
系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下：
```
```sql
CREATE USER <user_name> PASS <'password'>;
```
创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角
```
```sql
DROP USER <user_name>;
```
删除用户，限root用户使用
```
```sql
ALTER USER <user_name> PASS <'password'>;
```
修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角
```
```sql
ALTER USER <user_name> PRIVILEGE <super|write|read>;
```
修改用户权限为：super/write/read，不需要添加单引号
```
```mysql
SHOW USERS;
```
@ -316,12 +328,11 @@ taos> DESCRIBE d1001
```
那么可以用如下命令导入数据
```
```mysql
taos> insert into d1001 file '~/data.csv';
Query OK, 9 row(s) affected (0.004763s)
```
**taosdump工具导入**
TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：<a href='https://www.taosdata.com/blog/2020/03/09/1334.html'>TDengine DUMP工具使用指南</a>
@ -334,7 +345,7 @@ TDengine提供了方便的数据库导入导出工具taosdump。用户可以将t
如果用户需要导出一个表或一个STable中的数据，可在shell中运行
```
```mysql
select * from <tb_name> >> data.csv;
```
@ -348,37 +359,37 @@ TDengine提供了方便的数据库导出工具taosdump。用户可以根据需
系统管理员可以从CLI查询系统的连接、正在进行的查询、流式计算，并且可以关闭连接、停止正在进行的查询和流式计算。CLI里SQL语法如下：
```
```mysql
SHOW CONNECTIONS;
```
显示数据库的连接，其中一列显示ip:port, 为连接的IP地址和端口号。
```
```mysql
KILL CONNECTION <connection-id>;
```
强制关闭数据库连接，其中的connection-id是SHOW CONNECTIONS中显示的第一列的数字。
```
```mysql
SHOW QUERIES;
```
显示数据查询，其中第一列显示的以冒号隔开的两个数字为query-id，为发起该query应用连接的connection-id和查询次数。
```
```mysql
KILL QUERY <query-id>;
```
强制关闭数据查询，其中query-id是SHOW QUERIES中显示的 connection-id:query-no字串，如“105:2”，拷贝粘贴即可。
```
```mysql
SHOW STREAMS;
```
显示流式计算，其中第一列显示的以冒号隔开的两个数字为stream-id, 为启动该stream应用连接的connection-id和发起stream的次数。
```
```mysql
KILL STREAM <stream-id>;
```
@ -416,8 +427,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
## TDengine参数限制与保留关键字
- 数据库名：不能包含“.”以及特殊字符，不能超过32个字符
@ -436,8 +445,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
- 库的个数：仅受节点个数限制
- 单个库上虚拟节点个数：不能超过64个
目前TDengine有将近200个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下：
| 关键字列表 | | | | |
@ -478,4 +485,3 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
| CONCAT | GLOB | METRICS | SEMI | WAVG |
| CONFIGS | GRANTS | MIN | SET | WHERE |
| CONFLICT | GROUP | | | |

5
documentation20/webdocs/markdowndocs/advanced features-ch.md
View File

@ -60,7 +60,7 @@ create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s
会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句，
并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如：
```shell
```mysql
taos> select * from avg_vol;
ts | avg_voltage_ |
===================================================
@ -72,14 +72,13 @@ taos> select * from avg_vol;
需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。
此外，TDengine还支持用户指定连续查询的起止时间。
如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始；
如果没有输入结束时间，连续查询将永久运行；
如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。
比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。
```sql
```mysql
create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s);
```

4
documentation20/webdocs/markdowndocs/architecture-ch.md
View File

@ -1,4 +1,4 @@
#数据模型和整体架构
# 数据模型和整体架构
## 数据模型
### 物联网典型场景
@ -150,7 +150,7 @@ TDengine 分布式架构的逻辑结构图如下：
<center> 图 1 TDengine架构示意图 </center>
一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。
**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">《一篇文章说清楚TDengine的FQDN》</a>。
**数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。

12
documentation20/webdocs/markdowndocs/cluster-ch.md
View File

@ -137,6 +137,16 @@ DROP DNODE "fqdn:port";
其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号
<font color=green>**【注意】**</font>
- 一个数据节点一旦被drop之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成drop dnode操作之前，会将该dnode的数据迁移走。
- 请注意 drop dnode 和 停止taosd进程是两个不同的概念，不要混淆：因为删除dnode之前要执行迁移数据的操作，因此被删除的dnode必须保持在线状态。待删除操作结束之后，才能停止taosd进程。
- 一个数据节点被drop之后，其他节点都会感知到这个dnodeID的删除操作，任何集群中的节点都不会再接收此dnodeID的请求。
- dnodeID的是集群自动分配的，不得人工指定。它在生成时递增的，不会重复。
### 查看数据节点
执行CLI程序taos,使用root账号登录进TDengine系统，执行：
@ -216,3 +226,5 @@ SHOW MNODES;
如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。
TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。如果副本数为奇数，即使配置了arbitrator, 系统也不会去建立连接。
关于集群搭建请参考<a href="https://www.taosdata.com/blog/2020/11/11/1961.html">视频教程</a>。

386
documentation20/webdocs/markdowndocs/connector-ch.md
View File

@ -1,36 +1,162 @@
# 连接器
TDengine提供了丰富的应用程序开发接口，其中包括C/C++、C# 、Java、Python、Go、Node.js、RESTful 等，便于用户快速开发应用。
TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用。
![image-connecotr](../assets/connector.png)
目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下：
目前TDengine的连接器可支持的平台广泛，包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下：
| | **CPU** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | | |
| ---------------------------- | --------- | --------------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | --------- | --------- |
| | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** |
| **连** **接** **器** | **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | |
| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | |
| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | |
| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | |
| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | |
| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | |
| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** |
| ----------- | --------------- | --------------- | --------------- | --------------- | --------- | --------- | --------------- | ---------------- | -------------- |
| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** |
| **C/C++** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ |
| **JDBC** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ |
| **Python** | ● | ● | ● | ○ | ● | ● | ○ | -- | ○ |
| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- |
| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- |
| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- |
| **RESTful** | ● | ● | ● | ● | ● | ● | ○ | ○ | ○ |
注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
其中 ● 表示经过官方测试验证， ○ 表示非官方测试验证。
注意：
* 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。
* 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
* 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。
## 安装连接器驱动步骤
服务器应该已经安装TDengine服务端安装包。连接器驱动安装步骤如下：
**Linux**
**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载**
* X64硬件环境：TDengine-client-2.x.x.x-Linux-x64.tar.gz
* ARM64硬件环境：TDengine-client-2.x.x.x-Linux-aarch64.tar.gz
* ARM32硬件环境：TDengine-client-2.x.x.x-Linux-aarch32.tar.gz
**2. 解压缩软件包**
将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令：
`tar -xzvf TDengine-client-xxxxxxxxx.tar.gz`
其中xxxxxxx需要替换为实际版本的字符串。
**3. 执行安装脚本**
解压软件包之后，会在解压目录下看到以下文件(目录)：
​ *install_client.sh*：安装脚本，用于应用驱动程序
​ *taos.tar.gz*：应用驱动安装包
​ *driver*：TDengine应用驱动driver
​ *connector*: 各种编程语言连接器（go/grafanaplugin/nodejs/python/JDBC）
​ *examples*: 各种编程语言的示例程序(c/C#/go/JDBC/matlab/python/R)
运行install_client.sh进行安装
**4. 配置taos.cfg**
编辑taos.cfg文件(默认路径/etc/taos/taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030
**提示： 如本机没有部署TDengine服务，仅安装了应用驱动，则taos.cfg中仅需配置firstEP，无需配置FQDN。**
**Windows x64/x86**
**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载 ：**
* X64硬件环境：TDengine-client-2.X.X.X-Windows-x64.exe
* X86硬件环境：TDengine-client-2.X.X.X-Windows-x86.exe
**2. 执行安装程序，按提示选择默认值，完成安装**
**3. 安装路径**
默认安装路径为：C:\TDengine，其中包括以下文件(目录)：
​ *taos.exe*：taos shell命令行程序
​ *cfg* : 配置文件目录
​ *driver*: 应用驱动动态链接库
​ *examples*: 示例程序 bash/C/C#/go/JDBC/Python/Node.js
​ *include*: 头文件
​ *log* : 日志文件
​ *unins000.exe*: 卸载程序
**4. 配置taos.cfg**
编辑taos.cfg文件(默认路径C:\TDengine\cfg\taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030
**提示：**
**1. 如利用FQDN连接服务器，必须确认本机网络环境DNS已配置好，或在hosts文件中添加FQDN寻址记录，如编辑C:\Windows\system32\drivers\etc\hosts，添加如下的记录：** **192.168.1.99 h1.taos.com**
**2．卸载：运行unins000.exe可卸载TDengine应用驱动。**
**安装验证**
以上安装和配置完成后，并确认TDengine服务已经正常启动运行，此时可以执行taos客户端进行登录。
**Linux环境：**
在linux shell下直接执行 taos，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下：
```mysql
$ taos
Welcome to the TDengine shell from Linux, Client Version:2.0.5.0
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB)| blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
=========================================================================================================================================================================================================================
test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16| 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1| 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
Query OK, 2 row(s) in set (0.001198s)
taos>
```
**Windows（x64/x86）环境：**
在cmd下进入到c:\TDengine目录下直接执行 taos.exe，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下：
```mysql
C:\TDengine>taos
Welcome to the TDengine shell from Linux, Client Version:2.0.5.0
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
taos> show databases;
name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
===================================================================================================================================================================================================================================================================
test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
Query OK, 2 row(s) in set (0.045000s)
taos>
```
## C/C++ Connector
**C/C++连接器支持的系统有**：
| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 |
| ------------ | ------------ | -------- | -------- | -------- | ---------- |
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** |
C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）：
```C
#include <taos.h>
```
在编译时需要链接TDengine动态库 _libtaos.so_ （安装后，位于 _/usr/local/taos/driver_，gcc编译时，请加上 -ltaos）。
注意：
如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。
* 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ，安装后，位于 _/usr/local/taos/driver_。Windows为 taos.dll，安装后位于 *C:\TDengine*。
* 如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。
使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c。
### 基础API
@ -40,22 +166,18 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
初始化运行环境。如果应用没有主动调用该API，那么应用在调用`taos_connect`时将自动调用，故应用程序一般无需手动调用该API。
- `void taos_cleanup()`
清理运行环境，应用退出前应调用此API。
- `int taos_options(TSDB_OPTION option, const void * arg, ...)`
设置客户端选项，目前只支持时区设置（_TSDB_OPTION_TIMEZONE_）和编码设置（_TSDB_OPTION_LOCALE_）。时区和编码默认为操作系统当前设置。
- `char *taos_get_client_info()`
获取客户端版本信息。
- `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)`
创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含：
@ -68,23 +190,18 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。
- `char *taos_get_server_info(TAOS *taos)`
获取服务端版本信息。
- `int taos_select_db(TAOS *taos, const char *db)`
将当前的缺省数据库设置为`db`。
- `void taos_close(TAOS *taos)`
关闭连接, 其中`taos`是`taos_connect`函数返回的指针。
### 同步查询API
传统的数据库操作API，都属于同步操作。应用调用API后，一直处于阻塞状态，直到服务器返回结果。TDengine支持如下API：
@ -93,37 +210,30 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。
- `int taos_result_precision(TAOS_RES *res)`
返回结果集时间戳字段的精度，`0` 代表毫秒，`1` 代表微秒，`2` 代表纳秒。
- `TAOS_ROW taos_fetch_row(TAOS_RES *res)`
按行获取查询结果集中的数据。
- `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)`
批量获取查询结果集中的数据，返回值为获取到的数据的行数。
- `int taos_num_fields(TAOS_RES *res)` 和 `int taos_field_count(TAOS_RES *res)`
这两个API等价，用于获取查询结果集中的列数。
- `int* taos_fetch_lengths(TAOS_RES *res)`
获取结果集中每个字段的长度。 返回值是一个数组，其长度为结果集的列数。
- `int taos_affected_rows(TAOS_RES *res)`
获取被所执行的 SQL 语句影响的行数。
- `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)`
获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下：
@ -136,30 +246,24 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
} TAOS_FIELD;
```
- `void taos_stop_query(TAOS_RES *res)`
停止一个查询的执行。
- `void taos_free_result(TAOS_RES *res)`
释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。
- `char *taos_errstr(TAOS_RES *res)`
获取最近一次API调用失败的原因,返回值为字符串。
- `char *taos_errno(TAOS_RES *res)`
获取最近一次API调用失败的原因，返回值为错误代码。
**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
### 异步查询API
同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。
@ -184,7 +288,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
* res：`taos_query_a`回调时返回的结果集
* fp：回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体；`numOfRows`是获取到的数据的行数（不是整个查询结果集的函数）。 在回调函数中，应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后，应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理，直到返回的记录数（numOfRows）为零（结果返回完成）或记录数为负值（查询出错）。
- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);`
异步获取一条记录。其中：
@ -194,7 +297,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。
### 参数绑定API
除了直接调用 `taos_query` 进行查询，TDengine也提供了支持参数绑定的Prepare API，与 MySQL 一样，这些API目前也仅支持用问号`?`来代表待绑定的参数，具体如下：
@ -239,7 +341,6 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线
执行完毕，释放所有资源。
### 连续查询接口
TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时间段，对一张或多张数据库的表(数据流)进行各种实时聚合计算操作。操作简单，仅有打开、关闭流的API。具体如下：
@ -256,12 +357,10 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
返回值为NULL，表示创建成功，返回值不为空，表示成功。
- `void taos_close_stream (TAOS_STREAM *tstr)`
关闭数据流，其中提供的参数是taos_open_stream的返回值。用户停止流式计算的时候，务必关闭该数据流。
### 数据订阅接口
订阅API目前支持订阅一张或多张表，并通过定期轮询的方式不断获取写入表中的最新数据。
@ -285,7 +384,6 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
* param：调用 `taos_subscribe`时客户程序提供的附加参数
* code：错误码
* `TAOS_RES *taos_consume(TAOS_SUB *tsub)`
同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。
@ -294,13 +392,14 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。
## Python Connector
Python连接器的使用参见<a href="https://www.taosdata.com/blog/2020/11/11/1963.html">视频教程</a>
### 安装准备
* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）][4]
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
* 已安装python 2.7 or >= 3.4
* 已安装pip
* 已安装pip 或 pip3
### Python客户端安装
@ -312,18 +411,18 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时
或
​ `pip install src/connector/python/linux/python3/`
​ `pip3 install src/connector/python/linux/python3/`
#### Windows
在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos <em>cmd</em> 命令行界面
```cmd
cd C:\TDengine\connector\python\windows
pip install python2\
python -m pip install python2\
```
或
```cmd
cd C:\TDengine\connector\python\windows
pip install python3\
python -m pip install python3\
```
*如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。
@ -402,7 +501,6 @@ for d in data:
sub.close()
```
* 关闭连接
```python
c1.close()
@ -426,7 +524,6 @@ conn.close()
用于生成taos.TDengineConnection的实例。
### Python客户端使用示例代码
在tests/examples/python中，我们提供了一个示例Python程序read_example.py，可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后，通过import taos引入taos类。主要步骤如下
@ -439,7 +536,7 @@ conn.close()
## RESTful Connector
为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。
为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。RESTful连接器的使用参见<a href=https://www.taosdata.com/blog/2020/11/11/1965.html>视频教程</a>。
### HTTP请求格式
@ -472,13 +569,13 @@ HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应
使用curl通过自定义身份认证方式来发起一个HTTP Request，语法如下：
```
```bash
curl -H 'Authorization: Basic <TOKEN>' -d '<SQL>' <ip>:<PORT>/rest/sql
```
或者
```
```bash
curl -u username:password -d '<SQL>' <ip>:<PORT>/rest/sql
```
@ -488,7 +585,7 @@ curl -u username:password -d '<SQL>' <ip>:<PORT>/rest/sql
返回值为JSON格式，如下:
```
```json
{
"status": "succ",
"head": ["Time Stamp","current", …],
@ -511,7 +608,7 @@ curl -u username:password -d '<SQL>' <ip>:<PORT>/rest/sql
HTTP请求中需要带有授权码`<TOKEN>`，用于身份识别。授权码通常由管理员提供，可简单的通过发送`HTTP GET`请求来获取授权码，操作如下：
```
```bash
curl http://<ip>:6041/rest/login/<username>/<password>
```
@ -525,13 +622,13 @@ curl http://<ip>:6041/rest/login/<username>/<password>
获取授权码示例：
```
```bash
curl http://192.168.0.1:6041/rest/login/root/taosdata
```
返回值：
```
```json
{
"status": "succ",
"code": 0,
@ -543,12 +640,12 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata
- 在demo库里查询表d1001的所有记录：
```
```bash
curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sql
```
返回值：
```
```json
{
"status": "succ",
"head": ["Time Stamp","current","voltage","phase"],
@ -562,12 +659,12 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001
- 创建库demo：
```
```bash
curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql
```
返回值：
```
```json
{
"status": "succ",
"head": ["affected_rows"],
@ -582,13 +679,13 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19
HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如
```
```bash
curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sqlt
```
返回值：
```
```json
{
"status": "succ",
"head": ["column1","column2","column3"],
@ -603,13 +700,13 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001
#### 结果集采用UTC时间字符串
HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如
```
```bash
curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6041/rest/sqlutc
```
返回值：
```
```json
{
"status": "succ",
"head": ["column1","column2","column3"],
@ -633,31 +730,39 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间
## CSharp Connector
在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
C#连接器支持的系统有：Linux 64/Windows x64/Windows x86
#### 安装TDengine客户端
### 安装准备
C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
* .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。
* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件
### 安装验证
- TDengineDriver.cs 调用taos.dll文件的Native C方法
- TDengineTest.cs 参考程序示例
运行install_directory/examples/C#/C#Checker/C#Checker.exe
在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件
```cmd
cd {install_directory}/examples/C#/C#Checker
csc /optimize *.cs
C#Checker.exe -h <fqdn>
```
#### 使用方法
### C#连接器的使用
- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法
- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中
在Windows系统上，.NET应用程序可以使用TDengine的.NET接口来执行所有数据库的操作。使用.NET接口的步骤如下所示：
#### 注意事项
1. 将.NET接口文件TDengineDrivercs.cs加入到应用程序所在.NET项目中。
2. 用户可以参考TDengineTest.cs来定义数据库连接参数，以及如何执行数据插入、查询等操作；
- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。
- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。
此.NET接口需要用到taos.dll文件，所以在执行应用程序前，拷贝Windows客户端install_directory/driver目录中的taos.dll文件到.NET项目最后生成.exe可执行文件所在文件夹。之后运行exe文件，即可访问TDengine数据库并做插入、查询等操作。
#### 第三方驱动
**注意：**
1. TDengine V2.0.3.0之后同时支持32位和64位Windows系统，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请选择对应的“X86” 或“x64”。
2. 此.NET接口目前已经在Visual Studio 2015/2017中验证过，其它VS版本尚待验证。
### 第三方驱动
Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考
@ -668,10 +773,15 @@ https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos
https://www.taosdata.com/blog/2020/11/02/1901.html
```
## Go Connector
TDengine提供了GO驱动程序`taosSql`. `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`
### 安装准备
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。
使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及<a href="https://www.taosdata.com/blog/2020/11/11/1951.html">视频教程</a>。
```Go
import (
@ -679,16 +789,16 @@ import (
_ "github.com/taosdata/driver-go/taosSql"
)
```
**建议Go版本是1.13或以上，并开启模块支持：**
**建议使用Go版本1.13或以上，并开启模块支持：**
```
```bash
go env -w GO111MODULE=on
go env -w GOPROXY=https://goproxy.io,direct
```
### 常用API
- sql.Open(DRIVER_NAME string, dataSourceName string) *DB`
- `sql.Open(DRIVER_NAME string, dataSourceName string) *DB`
该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine
@ -720,50 +830,90 @@ go env -w GOPROXY=https://goproxy.io,direct
## Node.js Connector
TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)：
Node.js连接器支持的系统有：
首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器.
| **CPU类型** | x64（64bit） | | | aarch64 | aarch32 |
| ------------ | ------------ | -------- | -------- | -------- | -------- |
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** |
```cmd
Node.js连接器的使用参见<a href="https://www.taosdata.com/blog/2020/11/11/1957.html">视频教程</a>
### 安装准备
* 应用驱动安装请参考<a href="https://www.taosdata.com/cn/documentation/connector/#安装连接器驱动步骤">安装连接器驱动步骤</a>。
### 安装Node.js连接器
用户可以通过<a href="https://www.npmjs.com/">npm</a>来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下：
首先，通过<a href="https://www.npmjs.com/">npm</a>安装node.js 连接器.
```bash
npm install td2.0-connector
```
我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下
我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
我们使用<a href="https://github.com/nodejs/node-gyp">node-gyp</a>和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
### Linux
- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
- `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。
- `make`
- c语言编译器比如[GCC](https://gcc.gnu.org)
- c语言编译器比如<a href="https://gcc.gnu.org">GCC</a>
### Windows
#### 安装方法1
使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
使用微软的<a href="https://github.com/felixrieseberg/windows-build-tools">windows-build-tools</a>在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
#### 安装方法2
手动安装以下工具:
- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community)
- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7`
- 安装Visual Studio相关：<a href="https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools>Visual Studio Build 工具</a> 或者 <a href="https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community">Visual Studio 2017 Community</a>
- 安装 <a href="https://www.python.org/downloads/">Python</a> 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7`
- 进入`cmd`命令行界面, `npm config set msvs_version 2017`
如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules)
如果以上步骤不能成功执行, 可以参考微软的node.js用户手册<a href="https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules">Microsoft's Node.js Guidelines for Windows</a>
如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64".
### 使用方法
### 示例程序
示例程序源码位于install_directory/examples/nodejs，有：
Node-example.js node.js示例源程序
Node-example-raw.js
### 安装验证
在安装好TDengine客户端后，使用nodejsChecker.js程序能够验证当前环境是否支持nodejs方式访问Tdengine。
验证方法：
1. 新建安装验证目录，例如：~/tdengine-test，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。
2. 在命令中执行以下命令：
```bash
npm init -y
npm install td2.0-connector
node nodejsChecker.js host=localhost
```
3. 执行以上步骤后，在命令行会输出nodejs连接Tdengine实例，并执行简答插入和查询的结果。
### Node.js连接器的使用
(http://docs.taosdata.com/node)
以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node)
以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考<a href="http://docs.taosdata.com/node">该文档</a>
#### 连接
#### 建立连接
使用node.js连接器时，必须先<em>require</em> ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信
使用node.js连接器时，必须先<em>require</em> `td2.0-connector`，然后使用 `taos.connect` 函数。`taos.connect` 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化`cursor` 来和TDengine服务端通信
```javascript
const taos = require('td2.0-connector');
@ -777,15 +927,35 @@ var cursor = conn.cursor(); // Initializing a new cursor
conn.close();
```
#### 执行SQL和插入数据
对于DDL语句（例如create database、create table、use等），可以使用cursor的execute方法。代码如下：
```js
cursor.execute('create database if not exists test;')
```
以上代码创建了一个名称为test的数据库。对于DDL语句，一般没有返回值，cursor的execute返回值为0。
对于Insert语句，代码如下：
```js
var affectRows = cursor.execute('insert into test.weather values(now, 22.3, 34);')
```
execute方法的返回值为该语句影响的行数，上面的sql向test库的weather表中，插入了一条数据，则返回值affectRows为1。
TDengine目前还不支持update和delete语句。
#### 查询
可通过 ```cursor.query``` 函数来查询数据库。
可通过 `cursor.query` 函数来查询数据库。
```javascript
var query = cursor.query('show databases;')
```
查询的结果可以通过 ```query.execute()``` 函数获取并打印出来
查询的结果可以通过 `query.execute()` 函数获取并打印出来
```javascript
var promise = query.execute();
@ -793,7 +963,7 @@ promise.then(function(result) {
result.pretty();
});
```
格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。
格式化查询语句还可以使用`query`的`bind`方法。如下面的示例：`query`会自动将提供的数值填入查询语句的`?`里。
```javascript
var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5);
@ -801,8 +971,7 @@ query.execute().then(function(result) {
result.pretty();
})
```
如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下：
如果在`query`语句里提供第二个参数并设为`true`也可以立即获取查询结果。如下：
```javascript
var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true)
@ -823,12 +992,7 @@ promise2.then(function(result) {
})
```
### 示例
[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.
[4]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client
<a href="https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js">这里</a>提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
<a href="https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js">这里</a>同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.

14
documentation20/webdocs/markdowndocs/connector-java-ch.md
View File

@ -1,6 +1,12 @@
# Java Connector
Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。
Java连接器支持的系统有：
| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 |
| ------------ | ------------ | -------- | -------- | -------- | -------- |
| **OS类型** | Linux | Win64 | Win32 | Linux | Linux |
| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** |
Java连接器的使用请参见<a href=https://www.taosdata.com/blog/2020/11/11/1955.html>视频教程</a>。
TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
@ -21,7 +27,6 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一
* 目前不支持表间的 union 操作。
* 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。
## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
@ -70,7 +75,6 @@ maven 项目中使用如下 pom.xml 配置即可：
下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。
## 使用说明
### 获取连接
@ -212,7 +216,6 @@ while(resultSet.next()){
```
> 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
### 订阅
#### 创建
@ -227,7 +230,7 @@ TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from met
* sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据
* restart：如果订阅已经存在，是重新开始，还是继续之前的订阅
如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic' 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
#### 消费数据
@ -255,7 +258,6 @@ sub.close(true);
`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。
### 关闭资源
```java

21
documentation20/webdocs/markdowndocs/faq-ch.md
View File

@ -32,7 +32,7 @@
3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd*
4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。
4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">一篇文章说清楚TDengine的FQDN</a>。
5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
@ -51,19 +51,16 @@
* Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。
10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：<a href="https://www.taosdata.com/blog/2020/09/08/1816.html">TDengine 内嵌网络检测工具使用指南</a>。
## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？
产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查：
1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。
1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：<a href="https://www.taosdata.com/blog/2020/09/11/1824.html">一篇文章说清楚TDengine的FQDN</a>。
2. 如果网络配置有DNS server, 请检查是否正常工作
3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。
4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法连接服务器的
## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误
如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。
@ -86,7 +83,7 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统
## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？
windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下：
Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下：
```JAVA
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
@ -94,7 +91,7 @@ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8");
Connection = DriverManager.getConnection(url, properties);
```
## 12.TDengine GO windows驱动的如何编译？
请看为此问题撰写的<a href='blog/2020/01/06/tdengine-go-windows驱动的编译/'>技术博客
请看为此问题撰写的<a href='blog/2020/01/06/tdengine-go-windows驱动的编译/'>技术博客</a>
## 13.JDBC报错： the excuted SQL is not a DML or a DDL？
请更新至最新的JDBC驱动
@ -109,14 +106,10 @@ Connection = DriverManager.getConnection(url, properties);
常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。
## 15. 表名显示不全
由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
## 16. 如何进行数据迁移？
TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A移动机器B时，注意如下两件事：
@ -125,11 +118,9 @@ TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A
- 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。
- 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。
## 17. 怎么报告问题？
如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包:
如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包：
1. /var/log/taos
2. /etc/taos

33
documentation20/webdocs/markdowndocs/insert-ch.md
View File

@ -28,16 +28,16 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6,
- 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。
## Prometheus直接写入
[Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
<a href="https://www.prometheus.io/">Prometheus</a>作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具<a href="https://github.com/taosdata/Bailongma">Bailongma</a>，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文<a href="https://www.taosdata.com/blog/2020/02/03/1189.html">用Docker容器快速搭建一个Devops监控Demo</a>即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
### 从源代码编译blm_prometheus
用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
用户需要从github下载<a href="https://github.com/taosdata/Bailongma">Bailongma</a>的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
- Linux操作系统的服务器
- 安装好Golang, 1.10版本以上
- 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器）
Bailongma项目中有一个文件夹blm_prometheus，存放了prometheus的写入API程序。编译过程如下：
```
```bash
cd blm_prometheus
go build
```
@ -45,10 +45,10 @@ go build
一切正常的情况下，就会在对应的目录下生成一个blm_prometheus的可执行程序。
### 安装Prometheus
通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/)
通过Prometheus的官网下载安装。<a href="https://prometheus.io/download/">下载地址</a>
### 配置Prometheus
参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的<remote_write>部分，增加以下配置
参考Prometheus的<a href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/">配置文档</a>，在Prometheus的配置文件中的<remote_write>部分，增加以下配置
- url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节
@ -79,7 +79,7 @@ blm_prometheus对prometheus提供服务的端口号。
### 启动示例
通过以下命令启动一个blm_prometheus的API服务
```
```bash
./blm_prometheus -port 8088
```
假设blm_prometheus所在服务器的IP地址为"10.1.2.3"，则在prometheus的配置文件中<remote_write>部分增加url为
@ -107,16 +107,16 @@ prometheus产生的数据格式如下：
}
```
其中，apiserver_request_latencies_bucket为prometheus采集的时序数据的名称，后面{}中的为该时序数据的标签。blm_prometheus会以时序数据的名称在TDengine中自动创建一个超级表，并将{}中的标签转换成TDengine的tag值，Timestamp作为时间戳，value作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。
```
```mysql
use prometheus;
select * from apiserver_request_latencies_bucket;
```
## Telegraf直接写入
[Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具，TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
<a href="https://www.influxdata.com/time-series-platform/telegraf/"Telegraf</a>是一流行的IT运维数据采集开源工具，TDengine提供一个小工具<a href="https://github.com/taosdata/Bailongma">Bailongma</a>，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文<a href="https://www.taosdata.com/blog/2020/02/03/1189.html">用Docker容器快速搭建一个Devops监控Demo</a>即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。
### 从源代码编译blm_telegraf
用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
用户需要从github下载<a href="https://github.com/taosdata/Bailongma">Bailongma</a>的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件：
- Linux操作系统的服务器
- 安装好Golang, 1.10版本以上
@ -124,7 +124,7 @@ select * from apiserver_request_latencies_bucket;
Bailongma项目中有一个文件夹blm_telegraf，存放了Telegraf的写入API程序。编译过程如下：
```
```bash
cd blm_telegraf
go build
```
@ -148,7 +148,7 @@ go build
- hostname: 区分不同采集设备的机器名称，需确保其唯一性
- metric_batch_size: 100，允许Telegraf每批次写入记录最大数量，增大其数量可以降低Telegraf的请求发送频率。
关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的[文档](https://docs.influxdata.com/telegraf/v1.11/)。
关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的<a href="https://docs.influxdata.com/telegraf/v1.11/">文档</a>。
### 启动blm_telegraf程序
blm_telegraf程序有以下选项，在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。
@ -175,7 +175,7 @@ blm_telegraf对telegraf提供服务的端口号。
### 启动示例
通过以下命令启动一个blm_telegraf的API服务
```
```bash
./blm_telegraf -host 127.0.0.1 -port 8089
```
@ -213,20 +213,17 @@ telegraf产生的数据格式如下：
其中，name字段为telegraf采集的时序数据的名称，tags字段为该时序数据的标签。blm_telegraf会以时序数据的名称在TDengine中自动创建一个超级表，并将tags字段中的标签转换成TDengine的tag值，Timestamp作为时间戳，fields字段中的值作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。
```
```mysql
use telegraf;
select * from cpu;
```
MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
## EMQ Broker 直接写入
[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
<a href="https://github.com/emqx/emqx">EMQ</a>是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考<a href="https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine">EMQ 官方文档</a>。
## HiveMQ Broker 直接写入
[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。
<a href="https://www.hivemq.com/">HiveMQ</a> 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 <a href="https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md">HiveMQ extension - TDengine 说明文档</a>。

16
documentation20/webdocs/markdowndocs/replica-ch.md
View File

@ -30,7 +30,7 @@ TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储
- master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master.
- slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。
- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务
- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务。
- offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。
**Quorum:**
@ -83,10 +83,10 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性
如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的连接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的连接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下：
1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced.
2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程
2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程。
3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。
4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。
5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version).
5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version)。
具体的流程图如下：
@ -124,7 +124,7 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性
如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。
1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mnode里的文件，数量是固定的，对应于acct, user, db, table等文件。
1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mode里的文件，数量是固定的，对应于acct, user, db, table等文件。
2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。
3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。
4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则
@ -212,10 +212,10 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系
相同之处：
- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程
- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的
- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程。
- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的。
- 数据转发流程完全一样，Master(leader)需要等待回复确认。
- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步
- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步。
不同之处：
@ -226,7 +226,7 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系
## Meta Data的数据复制
TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析
TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析。
TDengine里Meta Data包括以下：

36
packaging/cfg/taos.cfg
View File

@ -29,16 +29,20 @@
# number of threads per CPU core
# numOfThreadsPerCore 1.0
# the proportion of total threads responsible for query
# ratioOfQueryThreads 0.5
# the proportion of total CPU cores available for query processing
# 2.0: the query threads will be set to double of the CPU cores.
# 1.0: all CPU cores are available for query processing [default].
# 0.5: only half of the CPU cores are available for query.
# 0.0: only one core available.
# tsRatioOfQueryCores 1.0
# number of management nodes in the system
# numOfMnodes 3
# enable/disable backuping vnode directory when removing dnode
# enable/disable backuping vnode directory when removing vnode
# vnodeBak 1
# if report installation / use information
# enable/disable installation / usage report
# telemetryReporting 1
# enable/disable load balancing
@ -77,7 +81,7 @@
# minimum time window, milli-second
# minIntervalTime 10
# maximum delay before launching a stream compution, milli-second
# maximum delay before launching a stream computation, milli-second
# maxStreamCompDelay 20000
# maximum delay before launching a stream computation for the first time, milli-second
@ -152,7 +156,7 @@
# max number of connections allowed in dnode
# maxShellConns 5000
# max numerber of connections allowed in client
# max number of connections allowed in client
# maxConnections 5000
# stop writing logs when the disk size of the log folder is less than this value
@ -183,7 +187,7 @@
# restfulRowLimit 10240
# The following parameter is used to limit the maximum number of lines in log files.
# max number of rows per log filters
# max number of lines per log filters
# numOfLogLines 10000000
# enable/disable async log
@ -195,7 +199,9 @@
# The following parameters are used for debug purpose only.
# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log.
# 131: output warning and error
# 135: output debug, warning and error
# 143: output trace, debug, warning and error to log
# 199: output debug, warning and error to both screen and file
# 207: output trace, debug, warning and error to both screen and file
@ -227,10 +233,10 @@
# cDebugFlag 131
# debug flag for JNI
# jniDebugflag 131
# jniDebugFlag 131
# debug flag for storage
# uDebugflag 131
# uDebugFlag 131
# debug flag for http server
# httpDebugFlag 131
@ -239,12 +245,12 @@
# monDebugFlag 131
# debug flag for query
# qDebugflag 131
# qDebugFlag 131
# debug flag for vnode
# vDebugflag 131
# vDebugFlag 131
# debug flag for http server
# debug flag for TSDB
# tsdbDebugFlag 131
# debug flag for continue query
@ -265,5 +271,5 @@
# enable/disable stream (continuous query)
# stream 1
# only 50% CPU resources will be used in query processing
# halfCoresForQuery 0
# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode
# retrieveBlockingModel 0

1
packaging/deb/DEBIAN/prerm
View File

@ -26,6 +26,7 @@ else
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :

1
packaging/deb/makedeb.sh
View File

@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosdemox ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin

4
packaging/rpm/tdengine.spec
View File

@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemox %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
@ -135,7 +136,8 @@ if [ $1 -eq 0 ];then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
#${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :

197
packaging/tools/install.sh
View File

@ -9,6 +9,9 @@ set -e
verMode=edge
pagMode=full
iplist=""
serverFqdn=""
# -----------------------Variables definition---------------------
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
@ -172,6 +175,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
@ -183,6 +187,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
@ -227,6 +232,157 @@ function install_header() {
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
}
function add_newHostname_to_hosts() {
localIp="127.0.0.1"
OLD_IFS="$IFS"
IFS=" "
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
arr=($iphost)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$localIp" ]]; then
return
fi
done
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||:
}
function set_hostname() {
echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:"
read newHostname
while true; do
if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then
break
else
read -p "Please enter one hostname(must not be 'localhost'):" newHostname
fi
done
${csudo} hostname $newHostname ||:
retval=`echo $?`
if [[ $retval != 0 ]]; then
echo
echo "set hostname fail!"
return
fi
#echo -e -n "$(hostnamectl status --static)"
#echo -e -n "$(hostnamectl status --transient)"
#echo -e -n "$(hostnamectl status --pretty)"
#ubuntu/centos /etc/hostname
if [[ -e /etc/hostname ]]; then
${csudo} echo $newHostname > /etc/hostname ||:
fi
#debian: #HOSTNAME=yourname
if [[ -e /etc/sysconfig/network ]]; then
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||:
fi
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
serverFqdn=$newHostname
if [[ -e /etc/hosts ]]; then
add_newHostname_to_hosts $newHostname
fi
}
function is_correct_ipaddr() {
newIp=$1
OLD_IFS="$IFS"
IFS=" "
arr=($iplist)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$newIp" ]]; then
return 0
fi
done
return 1
}
function set_ipAsFqdn() {
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||:
if [ -z "$iplist" ]; then
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||:
fi
if [ -z "$iplist" ]; then
echo
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
localFqdn="127.0.0.1"
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
echo
return
fi
echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:"
echo
echo -e -n "${GREEN}$iplist${NC}"
echo
echo
echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:"
read localFqdn
while true; do
if [ ! -z "$localFqdn" ]; then
# Check if correct ip address
is_correct_ipaddr $localFqdn
retval=`echo $?`
if [[ $retval != 0 ]]; then
read -p "Please choose an IP from local IP list:" localFqdn
else
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
break
fi
else
read -p "Please choose an IP from local IP list:" localFqdn
fi
done
}
function local_fqdn_check() {
#serverFqdn=$(hostname -f)
echo
echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}"
echo
if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
echo
while true
do
read -r -p "Set hostname now? [Y/n] " input
if [ ! -n "$input" ]; then
set_hostname
break
else
case $input in
[yY][eE][sS]|[yY])
set_hostname
break
;;
[nN][oO]|[nN])
set_ipAsFqdn
break
;;
*)
echo "Invalid input..."
;;
esac
fi
done
fi
}
function install_config() {
#${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || :
@ -249,6 +405,8 @@ function install_config() {
return 0
fi
local_fqdn_check
#FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
#FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)"
#PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)"
@ -446,6 +604,9 @@ function install_service_on_systemd() {
${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}"
${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
#${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/setDelay.sh' >> ${taosd_service_config}"
#${csudo} bash -c "echo 'ExecStartPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}"
#${csudo} bash -c "echo 'ExecStopPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
@ -454,6 +615,7 @@ function install_service_on_systemd() {
${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
#${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}"
${csudo} bash -c "echo >> ${taosd_service_config}"
${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
@ -634,9 +796,9 @@ function update_TDengine() {
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell${NC}"
fi
echo
@ -709,34 +871,47 @@ function install_TDengine() {
echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}"
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
fi
#if [ ${openresty_work} = 'true' ]; then
# echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
#else
# echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
#fi
if [ ! -z "$firstEp" ]; then
echo
echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}"
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
fi
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo
else # Only install client
install_bin
install_config
echo
echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}"
fi
touch ~/.taos_history
rm -rf $(tar -tf taos.tar.gz)
}
## ==============================Main program starts from here============================
serverFqdn=$(hostname -f)
if [ "$verType" == "server" ]; then
# Install server and client
if [ -x ${bin_dir}/taosd ]; then

2
packaging/tools/install_client.sh
View File

@ -86,6 +86,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
fi
${csudo} rm -f ${bin_link_dir}/rmtaos || :
@ -97,6 +98,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
fi
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || :

2
packaging/tools/install_client_power.sh
View File

@ -86,6 +86,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/power || :
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdemox || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
fi
${csudo} rm -f ${bin_link_dir}/rmpower || :
@ -97,6 +98,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || :
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
[ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || :
[ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
fi
[ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || :

191
packaging/tools/install_power.sh
View File

@ -9,6 +9,8 @@ set -e
verMode=edge
pagMode=full
iplist=""
serverFqdn=""
# -----------------------Variables definition---------------------
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
@ -172,7 +174,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerd || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/powerdemox || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -183,7 +185,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || :
[ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || :
[ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
[ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
[ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || :
[ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
@ -227,6 +229,157 @@ function install_header() {
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
}
function add_newHostname_to_hosts() {
localIp="127.0.0.1"
OLD_IFS="$IFS"
IFS=" "
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
arr=($iphost)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$localIp" ]]; then
return
fi
done
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||:
}
function set_hostname() {
echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:"
read newHostname
while true; do
if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then
break
else
read -p "Please enter one hostname(must not be 'localhost'):" newHostname
fi
done
${csudo} hostname $newHostname ||:
retval=`echo $?`
if [[ $retval != 0 ]]; then
echo
echo "set hostname fail!"
return
fi
#echo -e -n "$(hostnamectl status --static)"
#echo -e -n "$(hostnamectl status --transient)"
#echo -e -n "$(hostnamectl status --pretty)"
#ubuntu/centos /etc/hostname
if [[ -e /etc/hostname ]]; then
${csudo} echo $newHostname > /etc/hostname ||:
fi
#debian: #HOSTNAME=yourname
if [[ -e /etc/sysconfig/network ]]; then
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||:
fi
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
serverFqdn=$newHostname
if [[ -e /etc/hosts ]]; then
add_newHostname_to_hosts $newHostname
fi
}
function is_correct_ipaddr() {
newIp=$1
OLD_IFS="$IFS"
IFS=" "
arr=($iplist)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$newIp" ]]; then
return 0
fi
done
return 1
}
function set_ipAsFqdn() {
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||:
if [ -z "$iplist" ]; then
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||:
fi
if [ -z "$iplist" ]; then
echo
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
localFqdn="127.0.0.1"
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
echo
return
fi
echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:"
echo
echo -e -n "${GREEN}$iplist${NC}"
echo
echo
echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:"
read localFqdn
while true; do
if [ ! -z "$localFqdn" ]; then
# Check if correct ip address
is_correct_ipaddr $localFqdn
retval=`echo $?`
if [[ $retval != 0 ]]; then
read -p "Please choose an IP from local IP list:" localFqdn
else
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
break
fi
else
read -p "Please choose an IP from local IP list:" localFqdn
fi
done
}
function local_fqdn_check() {
#serverFqdn=$(hostname -f)
echo
echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}"
echo
if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
echo
while true
do
read -r -p "Set hostname now? [Y/n] " input
if [ ! -n "$input" ]; then
set_hostname
break
else
case $input in
[yY][eE][sS]|[yY])
set_hostname
break
;;
[nN][oO]|[nN])
set_ipAsFqdn
break
;;
*)
echo "Invalid input..."
;;
esac
fi
done
fi
}
function install_config() {
#${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || :
@ -249,6 +402,8 @@ function install_config() {
return 0
fi
local_fqdn_check
#FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
#FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)"
#PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)"
@ -611,9 +766,9 @@ function update_PowerDB() {
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
else
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}"
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell${NC}"
fi
echo
@ -686,17 +841,30 @@ function install_PowerDB() {
echo -e "${GREEN_DARK}To start PowerDB ${NC}: powerd${NC}"
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
else
echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}"
fi
#if [ ${openresty_work} = 'true' ]; then
# echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
#else
# echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}"
#fi
if [ ! -z "$firstEp" ]; then
echo
echo -e "${GREEN_DARK}Please run${NC}: power -h $firstEp${GREEN_DARK} to login into cluster, then${NC}"
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}"
echo
fi
echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}"
echo
@ -713,6 +881,7 @@ function install_PowerDB() {
## ==============================Main program starts from here============================
serverFqdn=$(hostname -f)
if [ "$verType" == "server" ]; then
# Install server and client
if [ -x ${bin_dir}/powerd ]; then

2
packaging/tools/makeclient.sh
View File

@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
else

1
packaging/tools/makeclient_power.sh
View File

@ -77,6 +77,7 @@ if [ "$osType" != "Darwin" ]; then
cp ${build_dir}/bin/taos ${install_dir}/bin/power
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${script_dir}/set_core.sh ${install_dir}/bin
cp ${script_dir}/get_client.sh ${install_dir}/bin

2
packaging/tools/makepkg.sh
View File

@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"

1
packaging/tools/makepkg_power.sh
View File

@ -77,6 +77,7 @@ else
cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${build_dir}/bin/tarbitrator ${install_dir}/bin
cp ${script_dir}/set_core.sh ${install_dir}/bin

187
packaging/tools/post.sh
View File

@ -3,6 +3,10 @@
# This file is used to install tdengine rpm package on centos systems. The operating system
# is required to use systemd to manage services at boot
#set -x
iplist=""
serverFqdn=""
# -----------------------Variables definition---------------------
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
@ -92,6 +96,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -102,10 +107,162 @@ function install_bin() {
[ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || :
[ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || :
[ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${bin_dir}/taosdemox ] && ${csudo} ln -s ${bin_dir}/taosdemox ${bin_link_dir}/taosdemox || :
[ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || :
[ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || :
}
function add_newHostname_to_hosts() {
localIp="127.0.0.1"
OLD_IFS="$IFS"
IFS=" "
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
arr=($iphost)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$localIp" ]]; then
return
fi
done
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||:
}
function set_hostname() {
echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:"
read newHostname
while true; do
if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then
break
else
read -p "Please enter one hostname(must not be 'localhost'):" newHostname
fi
done
${csudo} hostname $newHostname ||:
retval=`echo $?`
if [[ $retval != 0 ]]; then
echo
echo "set hostname fail!"
return
fi
#echo -e -n "$(hostnamectl status --static)"
#echo -e -n "$(hostnamectl status --transient)"
#echo -e -n "$(hostnamectl status --pretty)"
#ubuntu/centos /etc/hostname
if [[ -e /etc/hostname ]]; then
${csudo} echo $newHostname > /etc/hostname ||:
fi
#debian: #HOSTNAME=yourname
if [[ -e /etc/sysconfig/network ]]; then
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||:
fi
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
serverFqdn=$newHostname
if [[ -e /etc/hosts ]]; then
add_newHostname_to_hosts $newHostname
fi
}
function is_correct_ipaddr() {
newIp=$1
OLD_IFS="$IFS"
IFS=" "
arr=($iplist)
IFS="$OLD_IFS"
for s in ${arr[@]}
do
if [[ "$s" == "$newIp" ]]; then
return 0
fi
done
return 1
}
function set_ipAsFqdn() {
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||:
if [ -z "$iplist" ]; then
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||:
fi
if [ -z "$iplist" ]; then
echo
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
localFqdn="127.0.0.1"
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
echo
return
fi
echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:"
echo
echo -e -n "${GREEN}$iplist${NC}"
echo
echo
echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:"
read localFqdn
while true; do
if [ ! -z "$localFqdn" ]; then
# Check if correct ip address
is_correct_ipaddr $localFqdn
retval=`echo $?`
if [[ $retval != 0 ]]; then
read -p "Please choose an IP from local IP list:" localFqdn
else
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
break
fi
else
read -p "Please choose an IP from local IP list:" localFqdn
fi
done
}
function local_fqdn_check() {
#serverFqdn=$(hostname -f)
echo
echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}"
echo
if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
echo
while true
do
read -r -p "Set hostname now? [Y/n] " input
if [ ! -n "$input" ]; then
set_hostname
break
else
case $input in
[yY][eE][sS]|[yY])
set_hostname
break
;;
[nN][oO]|[nN])
set_ipAsFqdn
break
;;
*)
echo "Invalid input..."
;;
esac
fi
done
fi
}
function install_config() {
if [ ! -f ${cfg_install_dir}/taos.cfg ]; then
${csudo} ${csudo} mkdir -p ${cfg_install_dir}
@ -113,6 +270,14 @@ function install_config() {
${csudo} chmod 644 ${cfg_install_dir}/*
fi
# Save standard input to 6 and open / dev / TTY on standard input
exec 6<&0 0</dev/tty
local_fqdn_check
# restore the backup standard input, and turn off 6
exec 0<&6 6<&-
${csudo} mv ${cfg_dir}/taos.cfg ${cfg_dir}/taos.cfg.org
${csudo} ln -s ${cfg_install_dir}/taos.cfg ${cfg_dir}
#FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
@ -267,7 +432,7 @@ function install_service() {
}
function install_TDengine() {
echo -e "${GREEN}Start to install TDEngine...${NC}"
echo -e "${GREEN}Start to install TDengine...${NC}"
#install log and data dir , then ln to /usr/local/taos
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
@ -300,13 +465,26 @@ function install_TDengine() {
echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}"
fi
echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
if [ ! -z "$firstEp" ]; then
echo
echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}"
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
fi
echo
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
@ -314,4 +492,5 @@ function install_TDengine() {
## ==============================Main program starts from here============================
serverFqdn=$(hostname -f)
install_TDengine

2
packaging/tools/preun.sh
View File

@ -119,4 +119,4 @@ if ((${service_mod}==2)); then
kill_taosd
fi
echo -e "${GREEN}TDEngine is removed successfully!${NC}"
echo -e "${GREEN}TDengine is removed successfully!${NC}"

1
packaging/tools/remove.sh
View File

@ -72,6 +72,7 @@ function clean_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :

1
packaging/tools/remove_client.sh
View File

@ -38,6 +38,7 @@ function clean_bin() {
# Remove link
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdemox || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/set_core || :

1
packaging/tools/remove_client_power.sh
View File

@ -38,6 +38,7 @@ function clean_bin() {
# Remove link
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdemox || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/set_core || :

1
packaging/tools/remove_power.sh
View File

@ -72,6 +72,7 @@ function clean_bin() {
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerd || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdemox || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :

30
packaging/tools/set_core.sh
View File

@ -2,19 +2,39 @@
#
# This file is used to set config for core when taosd crash
set -e
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
# set -e
# set -x
corePath=$1
csudo=""
if command -v sudo > /dev/null; then
csudo="sudo"
fi
#ulimit -c unlimited
if [[ ! -n ${corePath} ]]; then
echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:"
read corePath
while true; do
if [[ ! -z "$corePath" ]]; then
break
else
read -p "Please enter a file directory to save the coredump file:" corePath
fi
done
fi
ulimit -c unlimited
${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||:
${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||:
source /etc/profile
${csudo} mkdir -p /coredump ||:
${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||:
${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||:
${csudo} mkdir -p ${corePath} ||:
${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||:
${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||:

4
snap/snapcraft.yaml
View File

@ -1,6 +1,6 @@
name: tdengine
base: core18
version: '2.0.9.0'
version: '2.0.12.0'
icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT.
description: |
@ -72,7 +72,7 @@ parts:
- usr/bin/taosd
- usr/bin/taos
- usr/bin/taosdemo
- usr/lib/libtaos.so.2.0.9.0
- usr/lib/libtaos.so.2.0.12.0
- usr/lib/libtaos.so.1
- usr/lib/libtaos.so

251
src/balance/src/bnMain.c
View File

@ -44,7 +44,7 @@ static void bnUnLock() {
static bool bnCheckFree(SDnodeObj *pDnode) {
if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]);
return false;
}
@ -66,6 +66,77 @@ static bool bnCheckFree(SDnodeObj *pDnode) {
return true;
}
static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
SVnodeGid tmp = *pVnodeGid1;
*pVnodeGid1 = *pVnodeGid2;
*pVnodeGid2 = tmp;
}
static void bnAdjustVnodeIndex(SVgObj *pInVg) {
int32_t d0Id = pInVg->vnodeGid[0].dnodeId;
int32_t d1Id = pInVg->vnodeGid[1].dnodeId;
int32_t d2Id = pInVg->vnodeGid[2].dnodeId;
int32_t vgId = pInVg->vgId;
int32_t d0Num = 0;
int32_t d1Num = 0;
int32_t d2Num = 0;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = mnodeGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->vgId != vgId) {
if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++;
if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++;
if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++;
}
mnodeDecVgroupRef(pVgroup);
}
if (pInVg->numOfVnodes == 1) {
}
if (pInVg->numOfVnodes == 2) {
mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num);
if (d0Num > d1Num) {
mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
}
}
if (pInVg->numOfVnodes >= 3) {
mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num);
if (d0Num <= d1Num && d0Num <= d2Num) {
if (d1Num > d2Num) {
mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
}
} else if (d1Num <= d2Num && d1Num <= d0Num) {
mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
if (d0Num > d2Num) {
mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
}
} else {
mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]);
if (d1Num > d0Num) {
mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
}
}
}
for (int i = 0; i < pInVg->numOfVnodes; ++i) {
mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId);
}
}
static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
@ -88,17 +159,11 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
pVgroup->numOfVnodes = numOfVnodes;
bnAdjustVnodeIndex(pVgroup);
mnodeUpdateVgroup(pVgroup);
}
static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
// SVnodeGid tmp = *pVnodeGid1;
// *pVnodeGid1 = *pVnodeGid2;
// *pVnodeGid2 = tmp;
}
int32_t bnAllocVnodes(SVgObj *pVgroup) {
static int32_t randIndex = 0;
int32_t dnode = 0;
int32_t vnodes = 0;
@ -120,8 +185,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
break;
} else {
mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
pDnode->alternativeRole);
dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
}
}
}
@ -137,7 +201,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status],
pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
mnodeDecDnodeRef(pDnode);
}
@ -149,36 +213,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
}
}
/*
* make the choice more random.
* replica 1: no choice
* replica 2: there are 2 combinations
* replica 3 or larger: there are 6 combinations
*/
if (pVgroup->numOfVnodes == 1) {
} else if (pVgroup->numOfVnodes == 2) {
if (randIndex++ % 2 == 0) {
bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
}
} else {
int32_t randVal = randIndex++ % 6;
if (randVal == 1) { // 1, 0, 2
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
} else if (randVal == 2) { // 1, 2, 0
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
} else if (randVal == 3) { // 2, 1, 0
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
} else if (randVal == 4) { // 2, 0, 1
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
}
if (randVal == 5) { // 0, 2, 1
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
} else {
} // 0, 1, 2
}
bnAdjustVnodeIndex(pVgroup);
bnReleaseDnodes();
bnUnLock();
return TSDB_CODE_SUCCESS;
@ -189,17 +224,32 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
return false;
}
int32_t rmVnodeVer = 0;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pVnode = pVgroup->vnodeGid + i;
if (pVnode == pRmVnode) {
rmVnodeVer = mnodeGetVgidVer(pVnode->vver);
mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i,
pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer);
}
}
bool isReady = false;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pVnode = pVgroup->vnodeGid + i;
SDnodeObj *pDnode = pVnode->pDnode;
if (pVnode == pRmVnode) continue;
int32_t vver = mnodeGetVgidVer(pVnode->vver);
mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId,
pVnode->pDnode->status, syncRole[pVnode->role]);
if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i,
pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue;
if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) {
if (rmVnodeVer == 0 || vver >= rmVnodeVer) {
mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d",
pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
isReady = true;
}
}
@ -214,50 +264,14 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
static int32_t bnRemoveVnode(SVgObj *pVgroup) {
if (pVgroup->numOfVnodes <= 1) return -1;
SVnodeGid *pRmVnode = NULL;
SVnodeGid *pSelVnode = NULL;
int32_t maxScore = 0;
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]);
SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId);
if (pDnode == NULL) {
mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId);
pRmVnode = pVnode;
break;
}
if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId);
pRmVnode = pVnode;
} else if (pVnode->dnodeId == pVgroup->lbDnodeId) {
mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId);
pRmVnode = pVnode;
} else {
if (pSelVnode == NULL) {
pSelVnode = pVnode;
maxScore = pDnode->score;
} else {
if (maxScore < pDnode->score) {
pSelVnode = pVnode;
maxScore = pDnode->score;
}
}
}
mnodeDecDnodeRef(pDnode);
}
if (pRmVnode != NULL) {
pSelVnode = pRmVnode;
}
SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1];
mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId);
if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
mDebug("vgId:%d, is not ready", pVgroup->vgId);
return -1;
} else {
mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
bnDiscardVnode(pVgroup, pSelVnode);
return TSDB_CODE_SUCCESS;
}
@ -275,36 +289,48 @@ static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
return false;
}
/**
* desc: add vnode to vgroup, find a new one if dest dnode is null
**/
static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
if (pDestDnode == NULL) {
static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) {
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
SDnodeObj *pDnode = tsBnDnodes.list[i];
if (pDnode == pSrcDnode) continue;
if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
if (!bnCheckFree(pDnode)) continue;
pDestDnode = pDnode;
mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
return pDnode;
}
return NULL;
}
static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
if (pDestDnode == NULL || pSrcDnode == pDestDnode) {
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
SVnodeGid vnodeGids[TSDB_MAX_REPLICA];
memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
int32_t numOfVnodes = pVgroup->numOfVnodes;
vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId;
vnodeGids[numOfVnodes].pDnode = pDestDnode;
numOfVnodes++;
// move the src vnode to the end
for (int32_t v = 0; v < numOfVnodes; ++v) {
if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) {
bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]);
pVgroup->lbDnodeId = pSrcDnode->dnodeId;
break;
}
}
if (pDestDnode == NULL) {
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes;
pVnodeGid->dnodeId = pDestDnode->dnodeId;
pVnodeGid->pDnode = pDestDnode;
pVgroup->numOfVnodes++;
if (pSrcDnode != NULL) {
pVgroup->lbDnodeId = pSrcDnode->dnodeId;
// adjust the vgroup postion
if (pSrcDnode == NULL) {
bnAdjustVnodeIndex(pVgroup);
}
memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
pVgroup->numOfVnodes = numOfVnodes;
atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
mnodeUpdateVgroup(pVgroup);
@ -315,16 +341,16 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes
static bool bnMonitorBalance() {
if (tsBnDnodes.size < 2) return false;
mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size);
for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
SDnodeObj *pDnode = tsBnDnodes.list[src];
mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
pDnode->openVnodes);
mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId,
dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes);
}
float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
if (scoresDiff < 0.01) {
mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff);
return false;
}
@ -392,7 +418,7 @@ void bnReset() {
tsAccessSquence = 0;
}
static int32_t bnMonitorVgroups() {
static bool bnMonitorVgroups() {
void * pIter = NULL;
SVgObj *pVgroup = NULL;
bool hasUpdatingVgroup = false;
@ -412,7 +438,13 @@ static int32_t bnMonitorVgroups() {
} else if (vgReplica < dbReplica) {
mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
hasUpdatingVgroup = true;
code = bnAddVnode(pVgroup, NULL, NULL);
SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup);
if (pAvailDnode == NULL) {
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
} else {
code = bnAddVnode(pVgroup, NULL, pAvailDnode);
}
}
mnodeDecVgroupRef(pVgroup);
@ -545,6 +577,7 @@ void bnCheckStatus() {
mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
pDnode->lastAccess, pDnode->status);
bnSetVgroupOffline(pDnode);
bnStartTimer(3000);
}
}
mnodeDecDnodeRef(pDnode);

2
src/balance/src/bnScore.c
View File

@ -299,7 +299,7 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]);
cols++;
numOfRows++;

6
src/balance/src/bnThread.c
View File

@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) {
}
pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
mDebug("balance thread wakes up to work");
bool updateSoon = bnStart();
mDebug("balance thread finished this poll, updateSoon:%d", updateSoon);
bnStartTimer(updateSoon ? 1000 : -1);
pthread_mutex_unlock(&(tsBnThread.mutex));
}
@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) {
tsBnThread.timer = NULL;
tsAccessSquence++;
bnCheckStatus();
bnStartTimer(-1);
bnCheckStatus();
if (handle == NULL) {
if (tsAccessSquence % tsBalanceInterval == 0) {
@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) {
bool updateSoon = (mseconds != -1);
if (updateSoon) {
mTrace("balance function will be called after %" PRId64 " ms", mseconds);
taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
} else {
taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);

7
src/client/inc/tscLocalMerge.h
View File

@ -38,12 +38,6 @@ typedef struct SLocalDataSource {
tFilePage filePage;
} SLocalDataSource;
enum {
TSC_LOCALREDUCE_READY = 0x0,
TSC_LOCALREDUCE_IN_PROGRESS = 0x1,
TSC_LOCALREDUCE_TOBE_FREED = 0x2,
};
typedef struct SLocalReducer {
SLocalDataSource ** pLocalDataSrc;
int32_t numOfBuffer;
@ -56,7 +50,6 @@ typedef struct SLocalReducer {
tFilePage * pTempBuffer;
struct SQLFunctionCtx *pCtx;
int32_t rowSize; // size of each intermediate result.
int32_t status; // denote it is in reduce process, in reduce process, it
bool hasPrevRow; // cannot be released
bool hasUnprocessedRow;
tOrderDescriptor * pDesc;

16
src/client/inc/tscLog.h
View File

@ -23,15 +23,15 @@ extern "C" {
#include "tlog.h"
extern int32_t cDebugFlag;
extern int32_t tscEmbedded;
extern int8_t tscEmbedded;
#define tscFatal(...) { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscError(...) { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscWarn(...) { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscInfo(...) { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}
#define tscDebug(...) { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }}
#define tscTrace(...) { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }}
#define tscDebugL(...){ if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }}
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}

20
src/client/inc/tscUtil.h
View File

@ -20,9 +20,6 @@
extern "C" {
#endif
/*
* @date 2018/09/30
*/
#include "exception.h"
#include "os.h"
#include "qExtbuffer.h"
@ -216,7 +213,7 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex);
SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex);
SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex);
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache);
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
@ -234,7 +231,7 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
void tscResetForNextRetrieve(SSqlRes* pRes);
void tscDoQuery(SSqlObj* pSql);
@ -276,7 +273,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
bool hasMoreVnodesToTry(SSqlObj *pSql);
bool hasMoreClauseToTry(SSqlObj* pSql);
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache);
void tscFreeQueryInfo(SSqlCmd* pCmd);
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
@ -287,6 +284,17 @@ bool tscSetSqlOwner(SSqlObj* pSql);
void tscClearSqlOwner(SSqlObj* pSql);
int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize);
char* serializeTagData(STagData* pTagData, char* pMsg);
int32_t copyTagData(STagData* dst, const STagData* src);
STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
uint32_t tscGetTableMetaMaxSize();
int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name);
STableMeta* tscTableMetaClone(STableMeta* pTableMeta);
void* malloc_throw(size_t size);
void* calloc_throw(size_t nmemb, size_t size);
char* strdup_throw(const char* str);

5
src/client/inc/tschemautil.h
View File

@ -105,7 +105,10 @@ SSchema tscGetTbnameColumnSchema();
* @param size size of the table meta
* @return
*/
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg);
bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
#ifdef __cplusplus
}

62
src/client/inc/tsclient.h
View File

@ -56,22 +56,28 @@ typedef struct STableComInfo {
int32_t rowSize;
} STableComInfo;
typedef struct SCorVgroupInfo {
int32_t version;
typedef struct SNewVgroupInfo {
int32_t vgId;
int8_t inUse;
int8_t numOfEps;
SEpAddr1 epAddr[TSDB_MAX_REPLICA];
} SCorVgroupInfo;
SEpAddrMsg ep[TSDB_MAX_REPLICA];
} SNewVgroupInfo;
typedef struct CChildTableMeta {
int32_t vgId;
STableId id;
uint8_t tableType;
char sTableName[TSDB_TABLE_FNAME_LEN];
} CChildTableMeta;
typedef struct STableMeta {
STableComInfo tableInfo;
int32_t vgId;
STableId id;
uint8_t tableType;
char sTableName[TSDB_TABLE_FNAME_LEN];
int16_t sversion;
int16_t tversion;
char sTableId[TSDB_TABLE_FNAME_LEN];
SVgroupInfo vgroupInfo;
SCorVgroupInfo corVgroupInfo;
STableId id;
STableComInfo tableInfo;
SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
} STableMeta;
@ -170,7 +176,7 @@ typedef struct SParamInfo {
} SParamInfo;
typedef struct STableDataBlocks {
char tableId[TSDB_TABLE_FNAME_LEN];
char tableName[TSDB_TABLE_FNAME_LEN];
int8_t tsSource; // where does the UNIX timestamp come from, server or client
bool ordered; // if current rows are ordered or not
int64_t vgId; // virtual group id
@ -229,7 +235,7 @@ typedef struct {
int32_t numOfTablesInSubmit;
};
int32_t insertType;
uint32_t insertType;
int32_t clauseIndex; // index of multiple subclause query
char * curSql; // current sql, resume position of sql after parsing paused
@ -246,9 +252,9 @@ typedef struct {
int8_t dataSourceType; // load data from file or not
int8_t submitSchema; // submit block is built with table schema
STagData *pTagData; // NOTE: pTagData->data is used as a variant length array
STagData tagData; // NOTE: pTagData->data is used as a variant length array
STableMeta **pTableMetaList; // all involved tableMeta list of current insert sql statement.
char **pTableNameList; // all involved tableMeta list of current insert sql statement.
int32_t numOfTables;
SHashObj *pTableBlockHashList; // data block for each table
@ -285,8 +291,8 @@ typedef struct {
char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t)
SColumnIndex* pColumnIndex;
SArithmeticSupport* pArithSup; // support the arithmetic expression calculation on agg functions
struct SLocalReducer* pLocalReducer;
SArithmeticSupport *pArithSup; // support the arithmetic expression calculation on agg functions
struct SLocalReducer *pLocalReducer;
} SSqlRes;
typedef struct STscObj {
@ -307,7 +313,7 @@ typedef struct STscObj {
SRpcCorEpSet *tscCorMgmtEpSet;
void* pDnodeConn;
pthread_mutex_t mutex;
T_REF_DECLARE()
int32_t numOfObj; // number of sqlObj from this tscObj
} STscObj;
typedef struct SSubqueryState {
@ -385,7 +391,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet);
int tscProcessSql(SSqlObj *pSql);
int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex);
void tscQueueAsyncRes(SSqlObj *pSql);
void tscAsyncResultOnError(SSqlObj *pSql);
void tscQueueAsyncError(void(*fp), void *param, int32_t code);
@ -399,7 +405,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo);
void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
void tscResetSqlCmdObj(SSqlCmd *pCmd);
/**
* free query result of the sql object
@ -413,14 +419,13 @@ void tscFreeSqlResult(SSqlObj *pSql);
*/
void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeRegisteredSqlObj(void *pSql);
void tscFreeTableMetaHelper(void *pTableMeta);
void tscCloseTscObj(void *pObj);
// todo move to taos? or create a new file: taos_internal.h
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, TAOS **taos);
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res);
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res);
void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
@ -478,15 +483,16 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
}
}
extern SCacheObj* tscMetaCache;
extern int tscObjRef;
extern void * tscTmr;
extern void * tscQhandle;
extern int tscKeepConn[];
extern int tsInsertHeadSize;
extern int tscNumOfThreads;
extern int tscRefId;
extern int32_t sentinel;
extern SHashObj *tscVgroupMap;
extern SHashObj *tscTableMetaInfo;
extern int tscObjRef;
extern void *tscTmr;
extern void *tscQhandle;
extern int tscKeepConn[];
extern int tscRefId;
extern int tscNumOfObj; // number of existed sqlObj in current process.
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

27
src/client/src/tscAsync.c
View File

@ -18,7 +18,6 @@
#include "tnote.h"
#include "trpc.h"
#include "tcache.h"
#include "tscLog.h"
#include "tscSubquery.h"
#include "tscLocalMerge.h"
@ -57,7 +56,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
if (pSql->sqlstr == NULL) {
tscError("%p failed to malloc sql string buffer", pSql);
pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -71,7 +70,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
if (code != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -166,7 +165,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
pRes->code = numOfRows;
}
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -217,7 +216,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) {
pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
pSql->param = param;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -280,7 +279,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
pSql->param = param;
pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -381,16 +380,20 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) {
taosScheduleTask(tscQhandle, &schedMsg);
}
void tscQueueAsyncRes(SSqlObj *pSql) {
void tscAsyncResultOnError(SSqlObj *pSql) {
if (pSql == NULL || pSql->signature != pSql) {
tscDebug("%p SqlObj is freed, not add into queue async res", pSql);
return;
}
assert(pSql->res.code != TSDB_CODE_SUCCESS);
tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
SSqlRes *pRes = &pSql->res;
assert(pSql->fp != NULL && pSql->fetchFp != NULL);
if (pSql->fp == NULL || pSql->fetchFp == NULL){
return;
}
pSql->fp = pSql->fetchFp;
(*pSql->fp)(pSql->param, pSql, pRes->code);
@ -419,7 +422,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
// check if it is a sub-query of super table query first, if true, enter another routine
if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) {
tscDebug("%p update table meta in local cache, continue to process sql and send the corresponding query", pSql);
tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
@ -436,7 +439,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
return;
} else { // continue to process normal async query
if (pCmd->parseFinished) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql);
tscDebug("%p update local table meta, continue to process sql and send corresponding query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
@ -451,7 +454,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (pCmd->command == TSDB_SQL_SELECT) {
tscDebug("%p redo parse sql string and proceed", pSql);
pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd, false);
tscResetSqlCmdObj(pCmd);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
@ -528,6 +531,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
_error:
if (code != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
}

44
src/client/src/tscLocal.c
View File

@ -17,7 +17,6 @@
#include "taosmsg.h"
#include "taosdef.h"
#include "tcache.h"
#include "tname.h"
#include "tscLog.h"
#include "tscUtil.h"
@ -47,6 +46,7 @@ typedef struct SCreateBuilder {
int32_t (*fp)(void *para, char* result);
Stage callStage;
} SCreateBuilder;
static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
@ -207,10 +207,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
const int32_t TYPE_COLUMN_LENGTH = 16;
const int32_t NOTE_COLUMN_MIN_LENGTH = 8;
int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql);
// if (noteFieldLen == 0) {
// noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
// }
int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
tscFieldInfoUpdateOffset(pQueryInfo);
@ -275,7 +272,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) {
if (pRes->code != TSDB_CODE_SUCCESS) {
taos_free_result(pSql);
free(builder);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -293,7 +290,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) {
if (pRes->code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, code);
} else {
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
}
}
}
@ -571,9 +568,9 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
char fullName[TSDB_TABLE_FNAME_LEN] = {0};
char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0};
extractDBName(pTableMetaInfo->name, fullName);
extractTableName(pMeta->sTableId, param->sTableName);
extractTableName(pMeta->sTableName, param->sTableName);
snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName);
extractTableName(pTableMetaInfo->name, param->buf);
@ -822,26 +819,39 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) {
}
// TODO add test cases.
static int32_t checkForOnlineNode(SSqlObj* pSql) {
int32_t* data = pSql->res.length;
if (data == NULL) {
return TSDB_CODE_SUCCESS;
}
int32_t total = data[0];
int32_t online = data[1];
return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS;
}
static int32_t tscProcessServStatus(SSqlObj *pSql) {
STscObj* pObj = pSql->pTscObj;
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
if (pHb != NULL) {
int32_t code = pHb->res.code;
pSql->res.code = pHb->res.code;
taosReleaseRef(tscObjRef, pObj->hbrid);
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
return pSql->res.code;
}
} else {
if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
return pSql->res.code;
}
pSql->res.code = checkForOnlineNode(pHb);
if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
return pSql->res.code;
}
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
int32_t val = 1;
tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t));
return TSDB_CODE_SUCCESS;
@ -890,7 +900,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
} else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) {
pRes->code = tscProcessShowCreateDatabase(pSql);
} else if (pCmd->command == TSDB_SQL_RESET_CACHE) {
taosCacheEmpty(tscMetaCache);
taosHashEmpty(tscTableMetaInfo);
pRes->code = TSDB_CODE_SUCCESS;
} else if (pCmd->command == TSDB_SQL_SERV_VERSION) {
pRes->code = tscProcessServerVer(pSql);
@ -914,7 +924,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
(*pSql->fp)(pSql->param, pSql, code);
} else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){
} else {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
return code;
}

46
src/client/src/tscLocalMerge.c
View File

@ -89,11 +89,11 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
pCtx->startOffset = 0;
pCtx->size = 1;
pCtx->hasNull = true;
pCtx->currentStage = SECONDARY_STAGE_MERGE;
pCtx->currentStage = MERGE_STAGE;
// for top/bottom function, the output of timestamp is the first column
int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf;
pCtx->param[2].i64Key = pQueryInfo->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
@ -493,13 +493,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
// there is no more result, so we release all allocated resource
SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL);
if (pLocalReducer != NULL) {
int32_t status = 0;
while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY,
TSC_LOCALREDUCE_TOBE_FREED)) == TSC_LOCALREDUCE_IN_PROGRESS) {
taosMsleep(100);
tscDebug("%p waiting for delete procedure, status: %d", pSql, status);
}
pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) {
@ -911,6 +904,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
}
}
if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) {
pRes->numOfRows = 0;
pBeforeFillData->num = 0;
pLocalReducer->discard = true;
return;
}
pRes->numOfRowsGroup += pRes->numOfRows;
// impose the limitation of output rows on the final result
@ -1067,7 +1067,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer,
pCtx->param[0].i64Key = pExpr->param[0].i64Key;
}
pCtx->currentStage = SECONDARY_STAGE_MERGE;
pCtx->currentStage = MERGE_STAGE;
if (needInit) {
aAggs[pCtx->functionId].init(pCtx);
@ -1080,7 +1080,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer,
continue;
}
aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]);
aAggs[functionId].mergeFunc(&pLocalReducer->pCtx[j]);
}
}
@ -1296,6 +1296,10 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// re
for (int32_t i = 0; i < t; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity;
if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) {
pLocalReducer->pCtx[i].ptsOutputBuf = pLocalReducer->pCtx[0].aOutputBuf;
}
}
memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage));
@ -1430,24 +1434,13 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// set the data merge in progress
int32_t prevStatus =
atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS);
if (prevStatus != TSC_LOCALREDUCE_READY) {
assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already
return TSDB_CODE_SUCCESS;
}
tFilePage *tmpBuffer = pLocalReducer->pTempBuffer;
if (doHandleLastRemainData(pSql)) {
pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result.
return TSDB_CODE_SUCCESS;
}
if (doBuildFilledResultForGroup(pSql)) {
pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result.
return TSDB_CODE_SUCCESS;
}
@ -1503,7 +1496,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
pLocalReducer->discardData->num = 0;
if (saveGroupResultInfo(pSql)) {
pLocalReducer->status = TSC_LOCALREDUCE_READY;
return TSDB_CODE_SUCCESS;
}
@ -1549,7 +1541,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
// here we do not check the return value
adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree);
assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS);
if (pRes->numOfRows == 0) {
handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer);
@ -1560,7 +1551,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
* If previous group is not skipped, keep it in pRes->numOfGroups
*/
if (notSkipped && saveGroupResultInfo(pSql)) {
pLocalReducer->status = TSC_LOCALREDUCE_READY;
return TSDB_CODE_SUCCESS;
}
@ -1580,7 +1570,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
if (pRes->numOfRows == 0) {
continue;
} else {
pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result.
return TSDB_CODE_SUCCESS;
}
} else { // result buffer is not full
@ -1605,9 +1594,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
genFinalResults(pSql, pLocalReducer, true);
}
assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS && pRes->row == 0);
pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result.
return TSDB_CODE_SUCCESS;
}
@ -1661,7 +1647,7 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
// calculate the result from several other columns
if (pSup->pArithExprInfo != NULL) {
arithSup.pArithExpr = pSup->pArithExprInfo;
tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc);
arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc);
} else {
SSqlExpr* pExpr = pSup->pSqlExpr;
memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num));

68
src/client/src/tscParseInsert.c
View File

@ -731,7 +731,7 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColI
return code;
}
dataBuf->vgId = pTableMeta->vgroupInfo.vgId;
dataBuf->vgId = pTableMeta->vgId;
dataBuf->numOfTables = 1;
*totalNum += numOfRows;
@ -796,8 +796,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sToken = tStrGetToken(sql, &index, false, 0, NULL);
sql += index;
tscAllocPayload(pCmd, sizeof(STagData));
//the source super table is moved to the secondary position of the pTableMetaInfo list
if (pQueryInfo->numOfTables < 2) {
tscAddEmptyMetaInfo(pQueryInfo);
@ -809,13 +807,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return code;
}
STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data));
if (pTag == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memset(pTag, 0, offsetof(STagData, data));
tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
pCmd->pTagData = pTag;
tstrncpy(pCmd->tagData.name, pSTableMeterMetaInfo->name, sizeof(pCmd->tagData.name));
pCmd->tagData.dataLen = 0;
code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) {
@ -946,14 +939,15 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
}
tdSortKVRowByColIdx(row);
pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row));
pCmd->tagData.dataLen = kvRowLen(row);
char* pTag = realloc(pCmd->tagData.data, pCmd->tagData.dataLen);
if (pTag == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pCmd->pTagData = pTag;
pTag->dataLen = htonl(kvRowLen(row));
kvRowCpy(pTag->data, row);
kvRowCpy(pTag, row);
free(row);
pCmd->tagData.data = pTag;
index = 0;
sToken = tStrGetToken(sql, &index, false, 0, NULL);
@ -972,7 +966,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
}
createTable = true;
code = tscGetMeterMetaEx(pSql, pTableMetaInfo, true);
code = tscGetTableMetaEx(pSql, pTableMetaInfo, true);
if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
return code;
}
@ -983,7 +977,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
} else {
sql = sToken.z;
}
code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false);
code = tscGetTableMetaEx(pSql, pTableMetaInfo, false);
if (pCmd->curSql == NULL) {
assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS);
@ -1298,7 +1292,6 @@ int tsInsertInitialCheck(SSqlObj *pSql) {
pCmd->count = 0;
pCmd->command = TSDB_SQL_INSERT;
pSql->res.numOfRows = 0;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex);
@ -1346,7 +1339,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) {
free(sqlstr);
} else {
tscResetSqlCmdObj(pCmd, true);
tscResetSqlCmdObj(pCmd);
free(pSql->sqlstr);
pSql->sqlstr = sqlstr;
pSql->parseRetry++;
@ -1358,12 +1351,12 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr);
ret = tscToSQLCmd(pSql, &SQLInfo);
if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) {
tscResetSqlCmdObj(pCmd, true);
tscResetSqlCmdObj(pCmd);
pSql->parseRetry++;
ret = tscToSQLCmd(pSql, &SQLInfo);
}
SQLInfoDestroy(&SQLInfo);
SqlInfoDestroy(&SQLInfo);
}
/*
@ -1420,13 +1413,25 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error
assert(taos_errno(pSql) == code);
do {
if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
assert(pSql->res.numOfRows == 0);
int32_t errc = fseek(fp, 0, SEEK_SET);
if (errc < 0) {
tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno));
} else {
break;
}
}
taos_free_result(pSql);
tfree(pSupporter);
fclose(fp);
pParentSql->res.code = code;
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
} while (0);
}
// accumulate the total submit records
@ -1446,18 +1451,21 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
int32_t count = 0;
int32_t maxRows = 0;
tscDestroyBlockArrayList(pSql->cmd.pDataBlocks);
pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
tfree(pCmd->pTableNameList);
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
if (pCmd->pTableBlockHashList == NULL) {
pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
}
STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE,
sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL);
if (ret != TSDB_CODE_SUCCESS) {
// return ret;
}
taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock);
tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows);
char *tokenBuf = calloc(1, 4096);
while ((readLen = tgetline(&line, &n, fp)) != -1) {
@ -1492,7 +1500,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
code = doPackSendDataBlock(pSql, count, pTableDataBlock);
if (code != TSDB_CODE_SUCCESS) {
pParentSql->res.code = code;
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -1519,8 +1527,6 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport));
SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES);
pCmd->count = 1;
FILE *fp = fopen(pCmd->payload, "r");
@ -1528,8 +1534,8 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
tfree(pSupporter)
tscQueueAsyncRes(pSql);
tfree(pSupporter);
tscAsyncResultOnError(pSql);
return;
}

288
src/client/src/tscSQLParser.c
View File

@ -66,9 +66,9 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength);
static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult);
static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
int8_t type, char* fieldName, SSqlExpr* pSqlExpr);
@ -87,7 +87,7 @@ static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuery
static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem);
static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem);
static int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql);
static int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL);
@ -666,6 +666,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ
const char* msg1 = "invalid query expression";
const char* msg2 = "interval cannot be less than 10 ms";
const char* msg3 = "sliding cannot be used without interval";
const char* msg4 = "top/bottom query does not support order by value in interval query";
SSqlCmd* pCmd = &pSql->cmd;
@ -712,6 +713,11 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_TSC_INVALID_SQL;
}
int32_t colId = pQueryInfo->order.orderColId;
if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
return TSDB_CODE_SUCCESS;
}
@ -904,7 +910,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
* that are corresponding to the old name for the new table name.
*/
if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) {
tscClearTableMetaInfo(pTableMetaInfo, false);
tscClearTableMetaInfo(pTableMetaInfo);
}
return TSDB_CODE_SUCCESS;
@ -996,33 +1002,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
return false;
}
int32_t nLen = 0;
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (p->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
nLen += p->bytes;
}
// max tag row length must be less than TSDB_MAX_TAGS_LEN
if (nLen > TSDB_MAX_TAGS_LEN) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
}
// field name must be unique
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (has(pFieldList, 0, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
}
/* timestamp in tag is not allowed */
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
@ -1054,6 +1033,33 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
}
}
int32_t nLen = 0;
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (p->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
nLen += p->bytes;
}
// max tag row length must be less than TSDB_MAX_TAGS_LEN
if (nLen > TSDB_MAX_TAGS_LEN) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
}
// field name must be unique
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (has(pFieldList, 0, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
}
return true;
}
@ -1279,7 +1285,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex*
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSqlExprItem* pItem) {
const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables";
const char* msg2 = "invalid arithmetic expression in select clause";
const char* msg3 = "tag columns can not be used in arithmetic expression";
@ -1420,7 +1426,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
return TSDB_CODE_SUCCESS;
}
static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem) {
SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex);
@ -1484,7 +1490,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
for (int32_t i = 0; i < pSelection->nExpr; ++i) {
int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
tSQLExprItem* pItem = &pSelection->a[i];
tSqlExprItem* pItem = &pSelection->a[i];
// project on all fields
int32_t optr = pItem->pNode->nSQLOptr;
@ -1643,7 +1649,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
return numOfTotalColumns;
}
int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) {
int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem) {
const char* msg0 = "invalid column name";
const char* msg1 = "tag for normal table query is not allowed";
@ -1767,7 +1773,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
return TSDB_CODE_SUCCESS;
}
void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) {
void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) {
if (pItem->aliasName != NULL) {
tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
} else if (multiCols) {
@ -1775,19 +1781,22 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT
int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN);
tstrncpy(uname, pToken->z, len);
if (tsKeepOriginalColumnName) { // keep the original column name
tstrncpy(name, uname, TSDB_COL_NAME_LEN);
} else {
int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1;
char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0};
snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
}
} else { // use the user-input result column name
int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN);
tstrncpy(name, pItem->pNode->token.z, len);
}
}
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) {
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) {
STableMetaInfo* pTableMetaInfo = NULL;
int32_t optr = pItem->pNode->nSQLOptr;
@ -1817,7 +1826,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (pItem->pNode->pParam != NULL) {
tSQLExprItem* pParamElem = &pItem->pNode->pParam->a[0];
tSqlExprItem* pParamElem = &pItem->pNode->pParam->a[0];
SStrToken* pToken = &pParamElem->pNode->colInfo;
int16_t sqlOptr = pParamElem->pNode->nSQLOptr;
if ((pToken->z == NULL || pToken->n == 0)
@ -1918,7 +1927,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]);
tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]);
if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -2037,7 +2046,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
/* in first/last function, multiple columns can be add to resultset */
for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) {
tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]);
tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[i]);
if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -2150,7 +2159,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]);
tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]);
if (pParamElem->pNode->nSQLOptr != TK_ID) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -2350,7 +2359,7 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column
return columnList;
}
void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength) {
void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength) {
if (pItem->aliasName != NULL) {
strncpy(resultFieldName, pItem->aliasName, nameLength);
} else {
@ -3517,7 +3526,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL};
tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL};
// sql function list in selection clause.
// Append the sqlExpr into exprList of pQueryInfo structure sequentially
@ -3734,7 +3743,7 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg
return invalidSqlErrMsg(msgBuf, msg);
}
*parent = tSQLExprCreate((*parent), pExpr, parentOptr);
*parent = tSqlExprCreate((*parent), pExpr, parentOptr);
} else {
*parent = pExpr;
}
@ -3782,7 +3791,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
* to release expression, e.g., m1.ts = m2.ts,
* since this expression is used to set the join query type
*/
tSQLExprDestroy(*pExpr);
tSqlExprDestroy(*pExpr);
} else {
ret = setExprToCond(&pCondExpr->pTimewindow, *pExpr, msg3, parentOptr, pQueryInfo->msg);
}
@ -3928,17 +3937,17 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) {
if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL &&
((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) {
tSQLExprNodeDestroy(*pExpr);
tSqlExprNodeDestroy(*pExpr);
*pExpr = NULL;
} else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) {
tSQLExpr* tmpPtr = (*pExpr)->pRight;
tSQLExprNodeDestroy(*pExpr);
tSqlExprNodeDestroy(*pExpr);
(*pExpr) = tmpPtr;
} else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) {
tSQLExpr* tmpPtr = (*pExpr)->pLeft;
tSQLExprNodeDestroy(*pExpr);
tSqlExprNodeDestroy(*pExpr);
(*pExpr) = tmpPtr;
}
@ -3961,7 +3970,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo*
(*pExpr) = NULL;
} else {
*pOut = tSQLExprCreate(NULL, NULL, (*pExpr)->nSQLOptr);
*pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->nSQLOptr);
doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex);
doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex);
@ -4010,6 +4019,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
if (pExpr->nSQLOptr == TK_LIKE) {
char* str = taosStringBuilderGetResult(sb, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
return TSDB_CODE_SUCCESS;
}
@ -4059,6 +4069,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
char* str = taosStringBuilderGetResult(&sb1, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
taosStringBuilderDestroy(&sb1);
tfree(segments);
@ -4168,23 +4179,23 @@ static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr
static void cleanQueryExpr(SCondExpr* pCondExpr) {
if (pCondExpr->pTableCond) {
tSQLExprDestroy(pCondExpr->pTableCond);
tSqlExprDestroy(pCondExpr->pTableCond);
}
if (pCondExpr->pTagCond) {
tSQLExprDestroy(pCondExpr->pTagCond);
tSqlExprDestroy(pCondExpr->pTagCond);
}
if (pCondExpr->pColumnCond) {
tSQLExprDestroy(pCondExpr->pColumnCond);
tSqlExprDestroy(pCondExpr->pColumnCond);
}
if (pCondExpr->pTimewindow) {
tSQLExprDestroy(pCondExpr->pTimewindow);
tSqlExprDestroy(pCondExpr->pTimewindow);
}
if (pCondExpr->pJoinExpr) {
tSQLExprDestroy(pCondExpr->pJoinExpr);
tSqlExprDestroy(pCondExpr->pJoinExpr);
}
}
@ -4253,7 +4264,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw);
doCompactQueryExpr(pExpr);
tSQLExprDestroy(p1);
tSqlExprDestroy(p1);
tExprTreeDestroy(&p, NULL);
taosArrayDestroy(colList);
@ -4469,6 +4480,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
const char* msg = "illegal value or data overflow";
const char* msg1 = "value is expected";
const char* msg2 = "invalid fill option";
const char* msg3 = "top/bottom not support fill";
if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
@ -4551,6 +4563,14 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
for(int32_t i = 0; i < numOfExprs; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
return TSDB_CODE_SUCCESS;
}
@ -4643,7 +4663,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
} else {
} else { // order by top/bottom result value column is not supported in case of interval query.
assert(!(orderByTags && orderByTS));
}
@ -4910,6 +4930,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
tVariantListItem* pItem = taosArrayGet(pVarList, 1);
SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex);
pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13);
}
@ -4931,7 +4953,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
pUpdateMsg->head.vgId = htonl(pTableMeta->vgId);
pUpdateMsg->tid = htonl(pTableMeta->id.tid);
pUpdateMsg->uid = htobe64(pTableMeta->id.uid);
pUpdateMsg->colId = htons(pTagsSchema->colId);
@ -5437,6 +5459,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update;
pMsg->cacheLastRow = pCreateDb->cachelast;
}
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
@ -5913,12 +5936,16 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
if (pExprList->nExpr != 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
bool server_status = false;
tSQLExpr* pExpr = pExprList->a[0].pNode;
if (pExpr->operand.z == NULL) {
//handle 'select 1'
if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) {
server_status = true;
} else {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
// TODO redefine the function
SDNodeDynConfOption functionsInfo[5] = {{"database()", 10},
{"server_version()", 16},
@ -5927,6 +5954,9 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
{"current_user()", 14}};
int32_t index = -1;
if (server_status == true) {
index = 2;
} else {
for (int32_t i = 0; i < tListLen(functionsInfo); ++i) {
if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 &&
functionsInfo[i].len == pExpr->operand.n) {
@ -5934,6 +5964,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
break;
}
}
}
switch (index) {
case 0:
@ -6141,38 +6172,41 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
const int32_t TABLE_INDEX = 0;
const int32_t STABLE_INDEX = 1;
STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
// super table name, create table by using dst
SStrToken* pToken = &(pCreateTable->usingInfo.stableName);
int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo);
for(int32_t j = 0; j < numOfTables; ++j) {
SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j);
SStrToken* pToken = &pCreateTableInfo->stableName;
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
int32_t code = tscSetTableFullName(pStableMeterMetaInfo, pToken, pSql);
int32_t code = tscSetTableFullName(pStableMetaInfo, pToken, pSql);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// get meter meta from mnode
tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name));
SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
// get table meta from mnode
tstrncpy(pCreateTableInfo->tagdata.name, pStableMetaInfo->name, tListLen(pCreateTableInfo->tagdata.name));
SArray* pList = pCreateTableInfo->pTagVals;
code = tscGetTableMeta(pSql, pStableMeterMetaInfo);
code = tscGetTableMeta(pSql, pStableMetaInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
size_t size = taosArrayGetSize(pList);
if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) {
if (tscGetNumOfTags(pStableMetaInfo->pTableMeta) != size) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
// too long tag values will return invalid sql, not be truncated automatically
SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta);
SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta);
STagData *pTag = &pCreateTableInfo->tagdata;
STagData* pTag = &pCreateTable->usingInfo.tagdata;
SKVRowBuilder kvRowBuilder = {0};
if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -6207,8 +6241,6 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal);
}
@ -6219,20 +6251,28 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
tdSortKVRowByColIdx(row);
pTag->dataLen = kvRowLen(row);
if (pTag->data == NULL) {
pTag->data = malloc(pTag->dataLen);
}
kvRowCpy(pTag->data, row);
free(row);
// table name
if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) {
if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX);
ret = tscSetTableFullName(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX);
ret = tscSetTableFullName(pTableMetaInfo, &pCreateTableInfo->name, pSql);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
pCreateTableInfo->fullname = strndup(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN);
}
return TSDB_CODE_SUCCESS;
}
@ -6343,18 +6383,52 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
return TSDB_CODE_SUCCESS;
}
static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
const char* msg3 = "start(end) time of query range required or time range too large";
if (pQueryInfo->interval.interval == 0) {
return TSDB_CODE_SUCCESS;
}
bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER);
if (initialWindows) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
int64_t intervalRange = 0;
if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') {
int64_t f = 1;
if (pQueryInfo->interval.intervalUnit == 'n') {
f = 30L * MILLISECOND_PER_DAY;
} else if (pQueryInfo->interval.intervalUnit == 'y') {
f = 365L * MILLISECOND_PER_DAY;
}
intervalRange = pQueryInfo->interval.interval * f;
} else {
intervalRange = pQueryInfo->interval.interval;
}
// number of result is not greater than 10,000,000
if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
return TSDB_CODE_SUCCESS;
}
int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0));
const char* msg0 = "invalid table name";
const char* msg2 = "point interpolation query needs timestamp";
const char* msg5 = "fill only available for interval query";
const char* msg6 = "start(end) time of query range required or time range too large";
const char* msg7 = "illegal number of tables in from clause";
const char* msg8 = "too many columns in selection clause";
const char* msg9 = "TWA query requires both the start and end time";
const char* msg10 = "too many tables in from clause";
const char* msg11 = "invalid table alias name";
const char* msg1 = "point interpolation query needs timestamp";
const char* msg2 = "fill only available for interval query";
const char* msg3 = "start(end) time of query range required or time range too large";
const char* msg4 = "illegal number of tables in from clause";
const char* msg5 = "too many columns in selection clause";
const char* msg6 = "too many tables in from clause";
const char* msg7 = "invalid table alias name";
int32_t code = TSDB_CODE_SUCCESS;
@ -6370,7 +6444,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
// too many result columns not support order by in query
if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
/*
@ -6388,13 +6462,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
size_t fromSize = taosArrayGetSize(pQuerySql->from);
if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
pQueryInfo->command = TSDB_SQL_SELECT;
if (fromSize > 4) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
// set all query tables, which are maybe more than one.
@ -6427,12 +6501,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i + 1);
if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
// has no table alias name
@ -6510,12 +6584,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
}
}
// user does not specified the query time window, twa is not allowed in such case.
if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
(pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
// no result due to invalid query time range
if (pQueryInfo->window.skey > pQueryInfo->window.ekey) {
pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
@ -6523,7 +6591,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
}
if (!hasTimestampForPointInterpQuery(pQueryInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
// in case of join query, time range is required.
@ -6531,7 +6599,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
if (timeRange == 0 && pQueryInfo->window.skey == 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
@ -6545,31 +6613,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
tscFieldInfoUpdateOffset(pQueryInfo);
if (pQuerySql->fillType != NULL) {
if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
/*
* fill options are set at the end position, when all columns are set properly
* the columns may be increased due to group by operation
*/
if (pQuerySql->fillType != NULL) {
if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
if ((code = checkQueryRangeForFill(pCmd, pQueryInfo)) != TSDB_CODE_SUCCESS) {
return code;
}
if (pQueryInfo->interval.interval > 0 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') {
bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER);
if (initialWindows) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
// number of result is not greater than 10,000,000
if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
}
int32_t ret = parseFillClause(pCmd, pQueryInfo, pQuerySql);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) {
return code;
}
}

68
src/client/src/tscSchemaUtil.c
View File

@ -130,22 +130,14 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
return NULL;
}
static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
corVgroupInfo->version = 0;
corVgroupInfo->inUse = 0;
corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) {
corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn);
corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port;
}
}
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
assert(pTableMetaMsg != NULL);
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0);
int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
pTableMeta->tableType = pTableMetaMsg->tableType;
pTableMeta->vgId = pTableMetaMsg->vgroup.vgId;
pTableMeta->tableInfo = (STableComInfo) {
.numOfTags = pTableMetaMsg->numOfTags,
@ -156,22 +148,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pTableMeta->id.tid = pTableMetaMsg->tid;
pTableMeta->id.uid = pTableMetaMsg->uid;
SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;
for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i];
pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn));
pVgroupInfo->epAddr[i].port = pEpMsg->port;
}
tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo);
pTableMeta->sversion = pTableMetaMsg->sversion;
pTableMeta->tversion = pTableMetaMsg->tversion;
tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN);
tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN);
memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
@ -180,11 +159,42 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
if (size != NULL) {
*size = sizeof(STableMeta) + schemaSize;
return pTableMeta;
}
bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) {
assert(pExisted != NULL && src != NULL);
if (pExisted->numOfEps != src->numOfEps) {
return false;
}
return pTableMeta;
for(int32_t i = 0; i < pExisted->numOfEps; ++i) {
if (pExisted->ep[i].port != src->epAddr[i].port) {
return false;
}
if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) {
return false;
}
}
return true;
}
SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) {
assert(pVgroupMsg != NULL);
SNewVgroupInfo info = {0};
info.numOfEps = pVgroupMsg->numOfEps;
info.vgId = pVgroupMsg->vgId;
info.inUse = 0;
for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) {
tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN);
info.ep[i].port = pVgroupMsg->epAddr[i].port;
}
return info;
}
// todo refactor

500
src/client/src/tscServer.c
View File

@ -14,7 +14,6 @@
*/
#include "os.h"
#include "tcache.h"
#include "tcmdtype.h"
#include "trpc.h"
#include "tscLocalMerge.h"
@ -45,32 +44,30 @@ static int32_t getWaitingTimeInterval(int32_t count) {
return 0;
}
return initial * (2<<(count - 2));
return initial * ((2u)<<(count - 2));
}
static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);
SRpcEpSet* pEpSet = &pSql->epSet;
static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupInfo* pVgroupInfo) {
assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);
// Issue the query to one of the vnode among a vgroup randomly.
// change the inUse property would not affect the isUse attribute of STableMeta
pEpSet->inUse = rand() % pVgroupInfo->numOfEps;
// apply the FQDN string length check here
bool hasFqdn = false;
bool existed = false;
pEpSet->numOfEps = pVgroupInfo->numOfEps;
for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
if (!hasFqdn) {
hasFqdn = (strlen(pEpSet->fqdn[i]) > 0);
int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN);
if (len > 0) {
tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
existed = true;
}
}
assert(hasFqdn);
assert(existed);
}
static void tscDumpMgmtEpSet(SSqlObj *pSql) {
@ -88,6 +85,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) {
if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) {
return false;
}
for (int32_t i = 0; i < s1->numOfEps; i++) {
if (s1->port[i] != s2->port[i]
|| strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0)
@ -95,6 +93,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) {
}
return true;
}
void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) {
// no need to update if equal
SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet;
@ -102,37 +101,45 @@ void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) {
pCorEpSet->epSet = *pEpSet;
taosCorEndWrite(&pCorEpSet->version);
}
static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) {
if (pVgroupInfo == NULL) { return;}
taosCorBeginRead(&pVgroupInfo->version);
int8_t inUse = pVgroupInfo->inUse;
pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0;
pEpSet->numOfEps = pVgroupInfo->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i]));
pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i]));
pEpSet->port[i] = pVgroupInfo->ep[i].port;
}
taosCorEndRead(&pVgroupInfo->version);
}
static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pObj->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
taosCorBeginWrite(&pVgroupInfo->version);
tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
tfree(pVgroupInfo->epAddr[i].fqdn);
pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i];
if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) {
return;
}
tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse);
taosCorEndWrite(&pVgroupInfo->version);
int32_t vgId = pTableMetaInfo->pTableMeta->vgId;
if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) {
assert(vgId == 0);
return;
}
SNewVgroupInfo vgroupInfo = {.vgId = -1};
taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo));
assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0);
tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps);
vgroupInfo.inUse = pEpSet->inUse;
vgroupInfo.numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) {
strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN);
vgroupInfo.ep[i].port = pEpSet->port[i];
}
tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps);
taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo));
}
void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
@ -147,13 +154,19 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
SSqlObj *pSql = tres;
SSqlRes *pRes = &pSql->res;
if (code == 0) {
if (code == TSDB_CODE_SUCCESS) {
SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp;
SRpcEpSet * epSet = &pRsp->epSet;
SRpcEpSet *epSet = &pRsp->epSet;
if (epSet->numOfEps > 0) {
tscEpSetHtons(epSet);
if (!tscEpSetIsEqual(&pSql->pTscObj->tscCorMgmtEpSet->epSet, epSet)) {
tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse);
for (int8_t i = 0; i < epSet->numOfEps; i++) {
tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]);
}
tscUpdateMgmtEpSet(pSql, epSet);
}
}
pSql->pTscObj->connId = htonl(pRsp->connId);
@ -161,11 +174,40 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
tscKillConnection(pObj);
return;
} else {
if (pRsp->queryId) tscKillQuery(pObj, htonl(pRsp->queryId));
if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
if (pRsp->queryId) {
tscKillQuery(pObj, htonl(pRsp->queryId));
}
if (pRsp->streamId) {
tscKillStream(pObj, htonl(pRsp->streamId));
}
}
int32_t total = htonl(pRsp->totalDnodes);
int32_t online = htonl(pRsp->onlineDnodes);
assert(online <= total);
if (online < total) {
tscError("HB:%p, total dnode:%d, online dnode:%d", pSql, total, online);
pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
}
if (pRes->length == NULL) {
pRes->length = calloc(2, sizeof(int32_t));
}
pRes->length[0] = total;
pRes->length[1] = online;
} else {
tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code));
if (pRes->length == NULL) {
pRes->length = calloc(2, sizeof(int32_t));
}
pRes->length[1] = 0;
if (pRes->length[0] == 0) {
pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node
}
}
if (pObj->hbrid != 0) {
@ -269,7 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
return;
}
if (pEpSet) {
if (pEpSet) { // todo update this
if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
if (pCmd->command < TSDB_SQL_MGMT) {
tscUpdateVgroupInfo(pSql, pEpSet);
@ -291,7 +333,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID ||
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL ||
rpcMsg->code == TSDB_CODE_APP_NOT_READY)) {
tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
pSql->retry++;
tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), pSql->retry);
pSql->res.code = rpcMsg->code; // keep the previous error code
if (pSql->retry > pSql->maxRetry) {
@ -401,7 +445,7 @@ int doProcessSql(SSqlObj *pSql) {
}
if (pRes->code != TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return pRes->code;
}
@ -410,7 +454,7 @@ int doProcessSql(SSqlObj *pSql) {
// NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads.
if (code != TSDB_CODE_SUCCESS) {
pRes->code = code;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return code;
}
@ -478,8 +522,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
} else {
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId);
pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId);
tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgId);
}
pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg);
@ -498,7 +542,6 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// NOTE: shell message size should not include SMsgDesc
int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
int32_t vgId = pTableMeta->vgroupInfo.vgId;
SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
pMsgDesc->numOfVnodes = htonl(1); // always one vnode
@ -506,7 +549,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += sizeof(SMsgDesc);
SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
pShellMsg->header.vgId = htonl(vgId);
pShellMsg->header.vgId = htonl(pTableMeta->vgId);
pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc
pShellMsg->length = pShellMsg->header.contLen;
@ -514,9 +557,12 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// pSql->cmd.payloadLen is set during copying data into payload
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet);
tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit,
SNewVgroupInfo vgroupInfo = {0};
taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo));
tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo);
tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit,
pSql->epSet.numOfEps);
return TSDB_CODE_SUCCESS;
}
@ -524,9 +570,11 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
/*
* for table query, simply return the size <= 1k
*/
static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) {
const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5;
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
SSqlCmd* pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
@ -534,6 +582,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2);
int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0;
int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1;
int32_t tableSerialize = 0;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -550,7 +600,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
}
return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize +
tableSerialize + 4096;
tableSerialize + sqlLen + 4096;
}
static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
@ -560,24 +610,31 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
SVgroupInfo* pVgroupInfo = NULL;
int32_t vgId = -1;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
int32_t index = pTableMetaInfo->vgroupIndex;
assert(index >= 0);
SVgroupInfo* pVgroupInfo = NULL;
if (pTableMetaInfo->vgroupList->numOfVgroups > 0) {
assert(index < pTableMetaInfo->vgroupList->numOfVgroups);
pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
}
vgId = pVgroupInfo->vgId;
tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo);
tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups);
} else {
pVgroupInfo = &pTableMeta->vgroupInfo;
vgId = pTableMeta->vgId;
SNewVgroupInfo vgroupInfo = {0};
taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo));
tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo);
}
assert(pVgroupInfo != NULL);
pSql->epSet.inUse = rand()%pSql->epSet.numOfEps;
tscSetDnodeEpSet(pSql, pVgroupInfo);
pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId);
pQueryMsg->head.vgId = htonl(vgId);
STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
pTableIdInfo->tid = htonl(pTableMeta->id.tid);
@ -596,7 +653,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index);
// set the vgroup info
tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo);
tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo);
pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId);
int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList);
@ -623,7 +680,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
int32_t size = tscEstimateQueryMsgSize(pCmd, pCmd->clauseIndex);
int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
tscError("%p failed to malloc for query msg", pSql);
@ -656,6 +713,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version));
int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList);
int32_t sqlLen = (int32_t) strlen(pSql->sqlstr);
if (pQueryInfo->order.order == TSDB_ORDER_ASC) {
pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey);
@ -679,9 +737,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit;
pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len);
pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->queryType = htonl(pQueryInfo->type);
pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit);
pQueryMsg->sqlstrLen = htonl(sqlLen);
size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number
@ -851,13 +911,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) {
SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j);
*((int16_t *)pMsg) = pCol->colId;
*((int16_t *)pMsg) = htons(pCol->colId);
pMsg += sizeof(pCol->colId);
*((int16_t *)pMsg) += pCol->colIndex;
*((int16_t *)pMsg) += htons(pCol->colIndex);
pMsg += sizeof(pCol->colIndex);
*((int16_t *)pMsg) += pCol->flag;
*((int16_t *)pMsg) += htons(pCol->flag);
pMsg += sizeof(pCol->flag);
memcpy(pMsg, pCol->name, tListLen(pCol->name));
@ -916,12 +976,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
}
if (pQueryInfo->tagCond.tbnameCond.cond == NULL) {
*pMsg = 0;
pMsg++;
} else {
strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond);
pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1;
SCond* pCond = &pQueryInfo->tagCond.tbnameCond;
if (pCond->len > 0) {
strncpy(pMsg, pCond->cond, pCond->len);
pMsg += pCond->len;
}
// compressed ts block
@ -942,6 +1000,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
}
memcpy(pMsg, pSql->sqlstr, sqlLen);
pMsg += sqlLen;
int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
tscDebug("%p msg built success, len:%d bytes", pSql, msgLen);
@ -1210,7 +1271,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pShowMsg->payloadLen = htons(pEpAddr->n);
}
pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen;
pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen);
return TSDB_CODE_SUCCESS;
}
@ -1234,12 +1295,12 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &(pSql->cmd);
int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg);
int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg);
SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo;
if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) {
size += sizeof(STagData);
int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo);
size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN);
} else {
size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count);
}
@ -1267,33 +1328,55 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload;
strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name);
// use dbinfo from table id without modifying current db info
tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateTableMsg->db);
SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo;
pCreateTableMsg->igExists = pCreateTable->existCheck ? 1 : 0;
pCreateTableMsg->numOfColumns = htons(pCmd->numOfCols);
pCreateTableMsg->numOfTags = htons(pCmd->count);
pCreateTableMsg->sqlLen = 0;
char *pMsg = (char *)pCreateTableMsg->schema;
SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg));
char* pMsg = NULL;
int8_t type = pInfo->pCreateTableInfo->type;
if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value
STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata;
*(int32_t*)pMsg = htonl(pTag->dataLen);
pMsg += sizeof(int32_t);
memcpy(pMsg, pTag->name, sizeof(pTag->name));
pMsg += sizeof(pTag->name);
memcpy(pMsg, pTag->data, pTag->dataLen);
pMsg += pTag->dataLen;
SArray* list = pInfo->pCreateTableInfo->childTableInfo;
int32_t numOfTables = (int32_t) taosArrayGetSize(list);
pCreateTableMsg->numOfTables = htonl(numOfTables);
pMsg = (char*) pCreateMsg;
for(int32_t i = 0; i < numOfTables; ++i) {
SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg;
pCreate->numOfColumns = htons(pCmd->numOfCols);
pCreate->numOfTags = htons(pCmd->count);
pMsg += sizeof(SCreateTableMsg);
SCreatedTableInfo* p = taosArrayGet(list, i);
strcpy(pCreate->tableId, p->fullname);
pCreate->igExists = (p->igExist)? 1 : 0;
// use dbinfo from table id without modifying current db info
tscGetDBInfoFromTableFullName(p->fullname, pCreate->db);
pMsg = serializeTagData(&p->tagdata, pMsg);
int32_t len = (int32_t)(pMsg - (char*) pCreate);
pCreate->len = htonl(len);
}
} else { // create (super) table
pSchema = (SSchema *)pCreateTableMsg->schema;
pCreateTableMsg->numOfTables = htonl(1); // only one table will be created
strcpy(pCreateMsg->tableId, pTableMetaInfo->name);
// use dbinfo from table id without modifying current db info
tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateMsg->db);
SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo;
pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0;
pCreateMsg->numOfColumns = htons(pCmd->numOfCols);
pCreateMsg->numOfTags = htons(pCmd->count);
pCreateMsg->sqlLen = 0;
pMsg = (char *)pCreateMsg->schema;
pSchema = (SSchema *)pCreateMsg->schema;
for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@ -1310,7 +1393,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SQuerySQL *pQuerySql = pInfo->pCreateTableInfo->pSelect;
strncpy(pMsg, pQuerySql->selectToken.z, pQuerySql->selectToken.n + 1);
pCreateTableMsg->sqlLen = htons(pQuerySql->selectToken.n + 1);
pCreateMsg->sqlLen = htons(pQuerySql->selectToken.n + 1);
pMsg += pQuerySql->selectToken.n + 1;
}
}
@ -1386,51 +1469,18 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
pCmd->payloadLen = htonl(pUpdateMsg->head.contLen);
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet);
SNewVgroupInfo vgroupInfo = {.vgId = -1};
taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo));
assert(vgroupInfo.vgId > 0);
tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo);
return TSDB_CODE_SUCCESS;
}
//int tscBuildCancelQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// SCancelQueryMsg *pCancelMsg = (SCancelQueryMsg*) pSql->cmd.payload;
// pCancelMsg->qhandle = htobe64(pSql->res.qhandle);
//
// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
//
// if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
// int32_t vgIndex = pTableMetaInfo->vgroupIndex;
// if (pTableMetaInfo->pVgroupTables == NULL) {
// SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList;
// assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
//
// pCancelMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex);
// } else {
// int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
// assert(vgIndex >= 0 && vgIndex < numOfVgroups);
//
// SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex);
//
// pCancelMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId);
// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex);
// }
// } else {
// STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
// pCancelMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
// tscDebug("%p build cancel query msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId);
// }
//
// pSql->cmd.payloadLen = sizeof(SCancelQueryMsg);
// pSql->cmd.msgType = TSDB_MSG_TYPE_CANCEL_QUERY;
//
// pCancelMsg->header.contLen = htonl(sizeof(SCancelQueryMsg));
// return TSDB_CODE_SUCCESS;
//}
int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SAlterDbMsg);
@ -1492,7 +1542,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) {
if (code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows);
} else {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
}
@ -1521,7 +1571,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
int32_t code = pRes->code;
if (pRes->code != TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return code;
}
@ -1540,7 +1590,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
if (pRes->code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
} else {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
return code;
@ -1587,13 +1637,8 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
if (pCmd->autoCreated && pCmd->pTagData != NULL) {
int len = htonl(pCmd->pTagData->dataLen);
if (len > 0) {
len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen);
memcpy(pInfoMsg->tags, pCmd->pTagData, len);
pMsg += len;
}
if (pCmd->autoCreated && pCmd->tagData.dataLen != 0) {
pMsg = serializeTagData(&pCmd->tagData, pMsg);
}
pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg);
@ -1789,19 +1834,46 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pSchema++;
}
size_t size = 0;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg);
// todo add one more function: taosAddDataIfNotExists();
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
assert(pTableMetaInfo->pTableMeta == NULL);
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
// check if super table hashmap or not
int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN);
if (pTableMetaInfo->pTableMeta == NULL) {
free(pTableMeta);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
// super tableMeta data alreay exists, create it according to tableMeta and add it to hash map
STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg);
uint32_t size = tscGetTableMetaSize(pSupTableMeta);
int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size);
assert(code == TSDB_CODE_SUCCESS);
tfree(pSupTableMeta);
CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta);
taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), cMeta, sizeof(CChildTableMeta));
tfree(cMeta);
} else {
uint32_t s = tscGetTableMetaSize(pTableMeta);
taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, s);
}
// update the vgroupInfo if needed
if (pTableMeta->vgId > 0) {
int32_t vgId = pTableMeta->vgId;
assert(pTableMeta->tableType != TSDB_SUPER_TABLE);
SNewVgroupInfo vgroupInfo = {.inUse = -1};
taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo));
if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) ||
(vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it
vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup);
taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo));
tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap));
}
}
tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name);
@ -1812,8 +1884,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
/**
* multi table meta rsp pkg format:
* | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
* |...... 1B 1B 4B
* | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
* |...... 1B 4B
**/
int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
#if 0
@ -1967,14 +2039,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
return pSql->res.code;
}
/*
* current process do not use the cache at all
*/
int tscProcessShowRsp(SSqlObj *pSql) {
STableMetaMsg *pMetaMsg;
SShowRsp * pShow;
SSchema * pSchema;
char key[20];
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
@ -1999,20 +2067,10 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pSchema++;
}
key[0] = pCmd->msgType + 'a';
strcpy(key + 1, "showlist");
tfree(pTableMetaInfo->pTableMeta);
pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg);
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false);
}
size_t size = 0;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size,
tsTableMetaKeepTimer * 1000);
SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
if (pQueryInfo->colList == NULL) {
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
}
@ -2035,12 +2093,9 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
tscFieldInfoUpdateOffset(pQueryInfo);
tfree(pTableMeta);
return 0;
}
// TODO multithread problem
static void createHBObj(STscObj* pObj) {
if (pObj->hbrid != 0) {
return;
@ -2093,6 +2148,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
if (pConnect->epSet.numOfEps > 0) {
tscEpSetHtons(&pConnect->epSet);
tscUpdateMgmtEpSet(pSql, &pConnect->epSet);
for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
tscDebug("%p epSet.fqdn[%d]: %s, pObj:%p", pSql, i, pConnect->epSet.fqdn[i], pObj);
}
}
strcpy(pObj->sversion, pConnect->serverVersion);
@ -2118,54 +2177,34 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
int tscProcessDropDbRsp(SSqlObj *pSql) {
pSql->pTscObj->db[0] = 0;
taosCacheEmpty(tscMetaCache);
taosHashEmpty(tscTableMetaInfo);
return 0;
}
int tscProcessDropTableRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMeta == NULL) { /* not in cache, abort */
return 0;
}
/*
* 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed.
* 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema,
* here the table will reside in a new vnode.
* The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache
* instead.
*/
tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
if (pTableMetaInfo->pTableMeta) {
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
}
//The cached tableMeta is expired in this case, so clean it in hash table
taosHashRemove(tscTableMetaInfo, pTableMetaInfo->name, strnlen(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN));
tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, pTableMetaInfo->name,
(int32_t) taosHashGetSize(tscTableMetaInfo));
assert(pTableMetaInfo->pTableMeta == NULL);
return 0;
}
int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMeta == NULL) { /* not in cache, abort */
return 0;
}
char* name = pTableMetaInfo->name;
tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name);
tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name);
taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
if (pTableMetaInfo->pTableMeta) {
bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
tfree(pTableMetaInfo->pTableMeta);
if (isSuperTable) { // if it is a super table, reset whole query cache
tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
taosCacheEmpty(tscMetaCache);
}
if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta
taosHashEmpty(tscTableMetaInfo);
}
return 0;
@ -2175,6 +2214,7 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) {
UNUSED(pSql);
return 0;
}
int tscProcessShowCreateRsp(SSqlObj *pSql) {
return tscLocalResultCommonBuilder(pSql, 1);
}
@ -2250,7 +2290,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
void tscTableMetaCallBack(void *param, TAOS_RES *res, int code);
static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("%p malloc failed for new sqlobj to get table meta", pSql);
@ -2277,15 +2317,13 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
if (pSql->cmd.pTagData != NULL) {
int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
pNew->cmd.pTagData = calloc(1, size);
if (pNew->cmd.pTagData == NULL) {
if (pSql->cmd.autoCreated) {
int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData);
if (code != TSDB_CODE_SUCCESS) {
tscError("%p malloc failed for new tag data to get table meta", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
}
tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
@ -2297,7 +2335,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
int32_t code = tscProcessSql(pNew);
if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated
}
return code;
@ -2305,31 +2343,39 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
assert(strlen(pTableMetaInfo->name) != 0);
tfree(pTableMetaInfo->pTableMeta);
// If this STableMetaInfo owns a table meta, release it first
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false);
uint32_t size = tscGetTableMetaMaxSize();
pTableMetaInfo->pTableMeta = calloc(1, size);
pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1;
int32_t len = (int32_t) strlen(pTableMetaInfo->name);
taosHashGetClone(tscTableMetaInfo, pTableMetaInfo->name, len, NULL, pTableMetaInfo->pTableMeta, -1);
// TODO resize the tableMeta
STableMeta* pMeta = pTableMetaInfo->pTableMeta;
if (pMeta->id.uid > 0) {
if (pMeta->tableType == TSDB_CHILD_TABLE) {
int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, pTableMetaInfo->name);
if (code != TSDB_CODE_SUCCESS) {
return getTableMetaFromMnode(pSql, pTableMetaInfo);
}
}
pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMetaInfo->pTableMeta != NULL) {
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns,
tinfo.numOfTags, pTableMetaInfo->pTableMeta);
return TSDB_CODE_SUCCESS;
}
return getTableMetaFromMgmt(pSql, pTableMetaInfo);
return getTableMetaFromMnode(pSql, pTableMetaInfo);
}
int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) {
int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) {
pSql->cmd.autoCreated = createIfNotExists;
return tscGetTableMeta(pSql, pTableMetaInfo);
}
/**
* retrieve table meta from mnode, and update the local table meta cache.
* retrieve table meta from mnode, and update the local table meta hashmap.
* @param pSql sql object
* @param tableIndex table index
* @return status code
@ -2337,17 +2383,19 @@ int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create
int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) {
SSqlCmd *pCmd = &pSql->cmd;
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
const char* name = pTableMetaInfo->name;
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
if (pTableMetaInfo->pTableMeta) {
tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta);
if (pTableMeta) {
tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name,
tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid);
}
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
return getTableMetaFromMgmt(pSql, pTableMetaInfo);
// remove stored tableMeta info in hash table
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
return getTableMetaFromMnode(pSql, pTableMetaInfo);
}
static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) {
@ -2387,7 +2435,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta);
STableMeta* pTableMeta = tscTableMetaClone(pMInfo->pTableMeta);
tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables);
}

39
src/client/src/tscSql.c
View File

@ -115,8 +115,6 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
pObj->signature = pObj;
pObj->pDnodeConn = pDnodeConn;
T_REF_INIT_VAL(pObj, 1);
tstrncpy(pObj->user, user, sizeof(pObj->user));
secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@ -172,11 +170,9 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
if (taos != NULL) {
*taos = pObj;
}
registerSqlObj(pSql);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
pObj->rid = taosAddRef(tscRefId, pObj);
registerSqlObj(pSql);
return pSql;
}
@ -288,34 +284,21 @@ void taos_close(TAOS *taos) {
return;
}
// make sure that the close connection can only be executed once.
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
if (pObj->hbrid > 0) {
if (RID_VALID(pObj->hbrid)) {
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
if (pHb != NULL) {
if (pHb->rpcRid > 0) { // wait for rsp from dnode
if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode
rpcCancelRequest(pHb->rpcRid);
pHb->rpcRid = -1;
}
tscDebug("%p HB is freed", pHb);
taos_free_result(pHb);
taosReleaseRef(tscObjRef, pHb->self);
taos_free_result(pHb);
}
}
int32_t ref = T_REF_DEC(pObj);
assert(ref >= 0);
if (ref > 0) {
tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
return;
}
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
taosRemoveRef(tscRefId, pObj->rid);
}
@ -331,7 +314,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
tsem_post(&pSql->rspSem);
}
TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) {
TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) {
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
@ -357,7 +340,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES
doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
if (res != NULL) {
*res = pSql;
atomic_store_64(res, pSql->self);
}
tsem_wait(&pSql->rspSem);
@ -368,7 +351,7 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL);
}
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) {
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) {
return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res);
}
@ -726,7 +709,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
pSubObj->rpcRid = -1;
}
tscQueueAsyncRes(pSubObj);
tscAsyncResultOnError(pSubObj);
taosReleaseRef(tscObjRef, pSubObj->self);
}
@ -762,7 +745,7 @@ void taos_stop_query(TAOS_RES *res) {
pSql->rpcRid = -1;
}
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
}
@ -926,7 +909,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) {
// must before clean the sqlcmd object
tscResetSqlCmdObj(&pSql->cmd, false);
tscResetSqlCmdObj(&pSql->cmd);
SSqlCmd *pCmd = &pSql->cmd;

7
src/client/src/tscStream.c
View File

@ -167,7 +167,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
retryDelay);
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true);
char* name = pTableMetaInfo->name;
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
@ -269,9 +271,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name,
pStream->numOfRes);
// release the metric/meter meta information reference, so data in cache can be updated
tfree(pTableMetaInfo->pTableMeta);
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;

6
src/client/src/tscSub.c
View File

@ -69,14 +69,17 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
}
void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
if( sub == NULL)
if( sub == NULL) {
return;
}
SSub* pSub = (SSub*)sub;
SSubscriptionProgress target = {.uid = uid, .key = ts};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
if (p != NULL) {
p->key = ts;
tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
}
}
@ -502,6 +505,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription
pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key;
tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey);
}
if (pSub->pTimer == NULL) {

83
src/client/src/tscSubquery.c
View File

@ -384,7 +384,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
}
SQueryInfo *pSubQueryInfo = tscGetQueryInfoDetail(&pPrevSub->cmd, 0);
STSBuf *pTSBuf = pSubQueryInfo->tsBuf;
STSBuf *pTsBuf = pSubQueryInfo->tsBuf;
pSubQueryInfo->tsBuf = NULL;
// free result for async object will also free sqlObj
@ -402,7 +402,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
pSql->pSubs[i] = pNew;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object
pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object
// set the second stage sub query for join process
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE);
@ -779,7 +779,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
pParentSql->res.code = numOfRows;
quitAllSubquery(pParentSql, pSupporter);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -796,7 +796,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
quitAllSubquery(pParentSql, pSupporter);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -845,7 +845,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
if (code != TSDB_CODE_SUCCESS) {
freeJoinSubqueryObj(pParentSql);
pParentSql->res.code = code;
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
taosArrayDestroy(s1);
taosArrayDestroy(s2);
@ -916,7 +916,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
pParentSql->res.code = numOfRows;
quitAllSubquery(pParentSql, pSupporter);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows);
pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -1028,7 +1028,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
pParentSql->res.code = numOfRows;
tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows));
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -1155,7 +1155,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) {
if (pSql->res.code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, 0);
} else {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
return;
@ -1233,7 +1233,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) {
if (pSql->res.code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, 0);
} else {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
return;
@ -1344,7 +1344,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code);
quitAllSubquery(pParentSql, pSupporter);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -1357,7 +1357,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
pParentSql->res.code = code;
quitAllSubquery(pParentSql, pSupporter);
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
return;
}
@ -1403,7 +1403,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
}
}
}
@ -1612,7 +1612,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
_error:
pRes->code = code;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
}
static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
@ -1648,7 +1648,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
pRes->qhandle = 0x1; // hack the qhandle check
const uint32_t nBufferSize = (1u << 16); // 64KB
const uint32_t nBufferSize = (1u << 16u); // 64KB
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize);
if (ret != 0) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
tfree(pMemoryBuf);
return ret;
}
@ -1680,7 +1680,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub);
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return ret;
}
@ -1890,7 +1890,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
(*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code);
} else { // regular super table query
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
}
}
}
@ -1968,7 +1968,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
tscQueueAsyncRes(pParentSql);
tscAsyncResultOnError(pParentSql);
}
}
@ -2151,7 +2151,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) {
if (pParentObj->retry > pParentObj->maxRetry) {
tscError("%p max retry reached, abort the retry effort", pParentObj)
tscError("%p max retry reached, abort the retry effort", pParentObj);
return false;
}
@ -2209,10 +2209,10 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
// restore user defined fp
pParentObj->fp = pParentObj->fetchFp;
int32_t numOfSub = pParentObj->subState.numOfSub;
doFreeInsertSupporter(pParentObj);
if (pParentObj->res.code == TSDB_CODE_SUCCESS) {
tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows);
doFreeInsertSupporter(pParentObj);
// todo remove this parameter in async callback function definition.
// all data has been sent to vnode, call user function
@ -2220,8 +2220,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
(*pParentObj->fp)(pParentObj->param, pParentObj, v);
} else {
if (!needRetryInsert(pParentObj, numOfSub)) {
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj);
tscAsyncResultOnError(pParentObj);
return;
}
@ -2232,7 +2231,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
numOfFailed += 1;
// clean up tableMeta in cache
tscFreeQueryInfo(&pSql->cmd, true);
tscFreeQueryInfo(&pSql->cmd);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0);
STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0);
tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL);
@ -2244,27 +2243,29 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj,
pParentObj->res.numOfRows, numOfFailed, numOfSub);
tscDebug("%p cleanup %d tableMeta in cache", pParentObj, pParentObj->cmd.numOfTables);
tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables);
for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) {
taosCacheRelease(tscMetaCache, (void**)&(pParentObj->cmd.pTableMetaList[i]), true);
char* name = pParentObj->cmd.pTableNameList[i];
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
}
pParentObj->cmd.parseFinished = false;
pParentObj->subState.numOfRemain = numOfFailed;
tscResetSqlCmdObj(&pParentObj->cmd, false);
tscResetSqlCmdObj(&pParentObj->cmd);
// in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema.
tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry++);
tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry);
pParentObj->retry++;
int32_t code = tsParseSql(pParentObj, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
if (code != TSDB_CODE_SUCCESS) {
pParentObj->res.code = code;
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj);
tscAsyncResultOnError(pParentObj);
return;
}
@ -2279,7 +2280,6 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
*/
int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) {
assert(pSql != NULL && pSql->param != NULL);
// SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param;
@ -2288,11 +2288,8 @@ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) {
STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.pDataBlocks, pSupporter->index);
int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
// free the data block created from insert sql string
// pCmd->pDataBlocks = tscDestroyBlockArrayList(pParent->cmd.pDataBlocks);
if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return code; // here the pSql may have been released already.
}
@ -2307,7 +2304,11 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
if (pSql->pSubs != NULL) {
for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
SSqlObj* pSub = pSql->pSubs[i];
SInsertSupporter* pSup = calloc(1, sizeof(SInsertSupporter));
pSup->index = i;
pSup->pSql = pSql;
pSub->param = pSup;
tscDebug("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, i);
if (pSub->res.code != TSDB_CODE_SUCCESS) {
tscHandleInsertRetry(pSql, pSub);
@ -2481,7 +2482,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
SSqlRes* pRes = &pSql->res;
if (pRes->code != TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
@ -2496,19 +2497,19 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
tscAsyncResultOnError(pSql);
return;
}
tscRestoreSQLFuncForSTableQuery(pQueryInfo);
}
while (1) {
assert (pRes->row >= pRes->numOfRows);
doBuildResFromSubqueries(pSql);
tsem_post(&pSql->rspSem);
return;
if (pRes->code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
} else {
tscAsyncResultOnError(pSql);
}
}

70
src/client/src/tscSystem.c
View File

@ -31,18 +31,21 @@
#include "tlocale.h"
// global, not configurable
SCacheObj* tscMetaCache;
int tscObjRef = -1;
void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
int tsInsertHeadSize;
int tscRefId = -1;
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
int tscNumOfThreads;
int32_t sentinel = TSC_VAR_NOT_RELEASE;
SHashObj *tscVgroupMap; // hash map to keep the global vgroup info
SHashObj *tscTableMetaInfo; // table meta info
int32_t tscObjRef = -1;
void *tscTmr;
void *tscQhandle;
int32_t tscRefId = -1;
int32_t tscNumOfObj = 0; // number of sqlObj in current process.
static void *tscCheckDiskUsageTmr;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
taosGetDisk();
@ -115,7 +118,7 @@ void taos_init_imp(void) {
int queueSize = tsMaxConnections*2;
double factor = (tscEmbedded == 0)? 2.0:4.0;
tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor);
int32_t tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor);
if (tscNumOfThreads < 2) {
tscNumOfThreads = 2;
}
@ -131,10 +134,11 @@ void taos_init_imp(void) {
taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr);
}
int64_t refreshTime = 10; // 10 seconds by default
if (tscMetaCache == NULL) {
tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
if (tscTableMetaInfo == NULL) {
tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tscDebug("TableMeta:%p", tscTableMetaInfo);
}
tscRefId = taosOpenRef(200, tscCloseTscObj);
@ -152,30 +156,38 @@ void taos_init() { pthread_once(&tscinit, taos_init_imp); }
void taos_cleanup(void) {
tscDebug("start to cleanup client environment");
void* m = tscMetaCache;
if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) {
taosCacheCleanup(m);
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
int refId = atomic_exchange_32(&tscObjRef, -1);
if (refId != -1) {
taosCloseRef(refId);
}
taosHashCleanup(tscTableMetaInfo);
tscTableMetaInfo = NULL;
m = tscQhandle;
if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) {
taosCleanUpScheduler(m);
}
taosHashCleanup(tscVgroupMap);
tscVgroupMap = NULL;
int32_t id = tscObjRef;
tscObjRef = -1;
taosCloseRef(id);
void* p = tscQhandle;
tscQhandle = NULL;
taosCleanUpScheduler(p);
id = tscRefId;
tscRefId = -1;
taosCloseRef(id);
taosCloseRef(tscRefId);
taosCleanupKeywordsTable();
taosCloseLog();
if (tscEmbedded == 0) rpcCleanup();
m = tscTmr;
if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {
taosTmrCleanUp(m);
if (tscEmbedded == 0) {
rpcCleanup();
}
p = tscTmr;
tscTmr = NULL;
taosTmrCleanUp(p);
}
static int taos_options_imp(TSDB_OPTION option, const char *pStr) {

281
src/client/src/tscUtil.c
View File

@ -18,7 +18,6 @@
#include "os.h"
#include "qAst.h"
#include "taosmsg.h"
#include "tcache.h"
#include "tkey.h"
#include "tmd5.h"
#include "tscLocalMerge.h"
@ -31,7 +30,7 @@
#include "ttokendef.h"
static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo);
SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
if (pTagCond->pCond == NULL) {
@ -379,17 +378,16 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) {
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
}
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
void tscFreeQueryInfo(SSqlCmd* pCmd) {
if (pCmd == NULL || pCmd->numOfClause == 0) {
return;
}
for (int32_t i = 0; i < pCmd->numOfClause; ++i) {
char* addr = (char*)pCmd - offsetof(SSqlObj, cmd);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i);
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache);
clearAllTableMetaInfo(pQueryInfo);
tfree(pQueryInfo);
}
@ -397,7 +395,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
tfree(pCmd->pQueryInfo);
}
void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
void tscResetSqlCmdObj(SSqlCmd* pCmd) {
pCmd->command = 0;
pCmd->numOfCols = 0;
pCmd->count = 0;
@ -405,13 +403,19 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
pCmd->msgType = 0;
pCmd->parseFinished = 0;
pCmd->autoCreated = 0;
pCmd->numOfTables = 0;
tfree(pCmd->pTableMetaList);
for(int32_t i = 0; i < pCmd->numOfTables; ++i) {
if (pCmd->pTableNameList && pCmd->pTableNameList[i]) {
tfree(pCmd->pTableNameList[i]);
}
}
pCmd->numOfTables = 0;
tfree(pCmd->pTableNameList);
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList);
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
tscFreeQueryInfo(pCmd, removeFromCache);
tscFreeQueryInfo(pCmd);
}
void tscFreeSqlResult(SSqlObj* pSql) {
@ -452,35 +456,14 @@ void tscFreeRegisteredSqlObj(void *pSql) {
SSqlObj* p = *(SSqlObj**)pSql;
STscObj* pTscObj = p->pTscObj;
assert(p->self != 0);
assert(RID_VALID(p->self));
tscFreeSqlObj(p);
taosReleaseRef(tscRefId, pTscObj->rid);
int32_t ref = T_REF_DEC(pTscObj);
assert(ref >= 0);
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref);
if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj);
taosRemoveRef(tscRefId, pTscObj->rid);
}
}
void tscFreeTableMetaHelper(void *pTableMeta) {
STableMeta* p = (STableMeta*) pTableMeta;
int32_t numOfEps = p->vgroupInfo.numOfEps;
assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps; ++i) {
tfree(p->vgroupInfo.epAddr[i].fqdn);
}
int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps1; ++i) {
tfree(p->corVgroupInfo.epAddr[i].fqdn);
}
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1);
int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1);
tscDebug("%p free SqlObj, total in tscObj:%d, total:%d", pSql, num, total);
}
void tscFreeSqlObj(SSqlObj* pSql) {
@ -510,9 +493,10 @@ void tscFreeSqlObj(SSqlObj* pSql) {
pSql->self = 0;
tscFreeSqlResult(pSql);
tscResetSqlCmdObj(pCmd, false);
tscResetSqlCmdObj(pCmd);
tfree(pCmd->pTagData);
tfree(pCmd->tagData.data);
pCmd->tagData.dataLen = 0;
memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
tfree(pCmd->payload);
@ -532,7 +516,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
// free the refcount for metermeta
if (pDataBlock->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
tfree(pDataBlock->pTableMeta);
}
tfree(pDataBlock);
@ -603,15 +587,15 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
// set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache
if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) {
tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
tstrncpy(pTableMetaInfo->name, pDataBlock->tableName, sizeof(pTableMetaInfo->name));
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tfree(pTableMetaInfo->pTableMeta);
}
pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta);
pTableMetaInfo->pTableMeta = tscTableMetaClone(pDataBlock->pTableMeta);
} else {
assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0);
assert(strncmp(pTableMetaInfo->name, pDataBlock->tableName, tListLen(pDataBlock->tableName)) == 0);
}
/*
@ -674,14 +658,10 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
dataBuf->size = startOffset;
dataBuf->tsSource = -1;
tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId));
tstrncpy(dataBuf->tableName, name, sizeof(dataBuf->tableName));
/*
* The table meta may be released since the table meta cache are completed clean by other thread
* due to operation such as drop database. So here we add the reference count directly instead of invoke
* taosGetDataFromCache, which may return NULL value.
*/
dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta);
//Here we keep the tableMeta to avoid it to be remove by other threads.
dataBuf->pTableMeta = tscTableMetaClone(pTableMeta);
assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL);
*dataBlocks = dataBuf;
@ -787,15 +767,15 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) {
return result;
}
static void extractTableMeta(SSqlCmd* pCmd) {
static void extractTableNameList(SSqlCmd* pCmd) {
pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList);
pCmd->pTableMetaList = calloc(pCmd->numOfTables, POINTER_BYTES);
pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES);
STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL);
int32_t i = 0;
while(p1) {
STableDataBlocks* pBlocks = *p1;
pCmd->pTableMetaList[i++] = taosCacheTransfer(tscMetaCache, (void**) &pBlocks->pTableMeta);
pCmd->pTableNameList[i++] = strndup(pBlocks->tableName, TSDB_TABLE_FNAME_LEN);
p1 = taosHashIterate(pCmd->pTableBlockHashList, p1);
}
@ -803,6 +783,7 @@ static void extractTableMeta(SSqlCmd* pCmd) {
}
int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
SSqlCmd* pCmd = &pSql->cmd;
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
@ -817,7 +798,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
STableDataBlocks* dataBuf = NULL;
int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
INSERT_HEAD_SIZE, 0, pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
if (ret != TSDB_CODE_SUCCESS) {
tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret);
taosHashCleanup(pVnodeDataBlockHashList);
@ -851,7 +832,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
tscSortRemoveDataBlockDupRows(pOneTableBlock);
char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
tscDebug("%p name:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableName,
pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
@ -881,7 +862,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) {
pOneTableBlock = *p;
}
extractTableMeta(pCmd);
extractTableNameList(pCmd);
// free the table data blocks;
pCmd->pDataBlocks = pVnodeDataBlockList;
@ -902,6 +883,7 @@ void tscCloseTscObj(void *param) {
rpcClose(pObj->pDnodeConn);
pObj->pDnodeConn = NULL;
}
tfree(pObj->tscCorMgmtEpSet);
pthread_mutex_destroy(&pObj->mutex);
@ -1530,6 +1512,7 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) {
}
dest->tbnameCond.uid = src->tbnameCond.uid;
dest->tbnameCond.len = src->tbnameCond.len;
memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo));
dest->relType = src->relType;
@ -1825,14 +1808,12 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) {
return pa;
}
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables);
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) {
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
tscClearTableMetaInfo(pTableMetaInfo, removeFromCache);
tscClearTableMetaInfo(pTableMetaInfo);
free(pTableMetaInfo);
}
@ -1886,14 +1867,12 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) {
return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL);
}
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) {
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) {
if (pTableMetaInfo == NULL) {
return;
}
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
}
tfree(pTableMetaInfo->pTableMeta);
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscColumnListDestroy(pTableMetaInfo->tagColList);
@ -1910,10 +1889,12 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
}
void registerSqlObj(SSqlObj* pSql) {
int32_t ref = T_REF_INC(pSql->pTscObj);
tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref);
taosAcquireRef(tscRefId, pSql->pTscObj->rid);
pSql->self = taosAddRef(tscObjRef, pSql);
int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1);
int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1);
tscDebug("%p new SqlObj from %p, total in tscObj:%d, total:%d", pSql, pSql->pTscObj, num, total);
}
SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {
@ -1931,16 +1912,12 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
pCmd->parseFinished = 1;
pCmd->autoCreated = pSql->cmd.autoCreated;
if (pSql->cmd.pTagData != NULL) {
int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
pNew->cmd.pTagData = calloc(1, size);
if (pNew->cmd.pTagData == NULL) {
int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData);
if (code != TSDB_CODE_SUCCESS) {
tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0);
free(pNew);
return NULL;
}
memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
}
if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);
@ -1950,27 +1927,21 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
pNew->fp = fp;
pNew->fetchFp = fp;
pNew->param = param;
pNew->sqlstr = NULL;
pNew->maxRetry = TSDB_MAX_REPLICA;
pNew->sqlstr = strdup(pSql->sqlstr);
if (pNew->sqlstr == NULL) {
tscError("%p new subquery failed", pSql);
tscFreeSqlObj(pNew);
return NULL;
}
SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0);
assert(pSql->cmd.clauseIndex == 0);
STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL);
registerSqlObj(pNew);
return pNew;
}
static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) {
static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) {
int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo);
if (numOfOutput == 0) {
return;
@ -2025,13 +1996,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
pNew->sqlstr = strdup(pSql->sqlstr);
if (pNew->sqlstr == NULL) {
tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
SSqlCmd* pnCmd = &pNew->cmd;
memcpy(pnCmd, pCmd, sizeof(SSqlCmd));
@ -2044,7 +2009,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pnCmd->numOfClause = 0;
pnCmd->clauseIndex = 0;
pnCmd->pDataBlocks = NULL;
pnCmd->numOfTables = 0;
pnCmd->parseFinished = 1;
pnCmd->pTableNameList = NULL;
pnCmd->pTableBlockHashList = NULL;
if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -2115,11 +2084,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
goto _error;
}
doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid);
doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid);
pNew->fp = fp;
pNew->fetchFp = fp;
pNew->param = param;
pNew->maxRetry = TSDB_MAX_REPLICA;
@ -2127,7 +2095,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
STableMetaInfo* pFinalInfo = NULL;
if (pPrevSql == NULL) {
STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup
STableMeta* pTableMeta = tscTableMetaClone(pTableMetaInfo->pTableMeta);
assert(pTableMeta != NULL);
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList,
@ -2135,15 +2103,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
} else { // transfer the ownership of pTableMeta to the newly create sql object.
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta);
STableMeta* pPrevTableMeta = tscTableMetaClone(pPrevInfo->pTableMeta);
SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList;
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList,
pTableMetaInfo->pVgroupTables);
}
// this case cannot be happened
if (pFinalInfo->pTableMeta == NULL) {
tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name);
tscError("%p new subquery failed since no tableMeta, name:%s", pSql, name);
if (pPrevSql != NULL) { // pass the previous error to client
assert(pPrevSql->res.code != TSDB_CODE_SUCCESS);
@ -2516,7 +2484,7 @@ bool tscSetSqlOwner(SSqlObj* pSql) {
SSqlRes* pRes = &pSql->res;
// set the sql object owner
uint64_t threadId = taosGetPthreadId();
uint64_t threadId = taosGetSelfPthreadId();
if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) {
pRes->code = TSDB_CODE_QRY_IN_EXEC;
return false;
@ -2570,6 +2538,10 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
tfree(pVgroupInfo->epAddr[j].fqdn);
}
for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) {
assert( pVgroupInfo->epAddr[j].fqdn == NULL );
}
}
tfree(vgroupList);
@ -2582,6 +2554,125 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
for(int32_t i = 0; i < dst->numOfEps; ++i) {
tfree(dst->epAddr[i].fqdn);
dst->epAddr[i].port = src->epAddr[i].port;
assert(dst->epAddr[i].fqdn == NULL);
dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
}
}
char* serializeTagData(STagData* pTagData, char* pMsg) {
int32_t n = (int32_t) strlen(pTagData->name);
*(int32_t*) pMsg = htonl(n);
pMsg += sizeof(n);
memcpy(pMsg, pTagData->name, n);
pMsg += n;
*(int32_t*)pMsg = htonl(pTagData->dataLen);
pMsg += sizeof(int32_t);
memcpy(pMsg, pTagData->data, pTagData->dataLen);
pMsg += pTagData->dataLen;
return pMsg;
}
int32_t copyTagData(STagData* dst, const STagData* src) {
dst->dataLen = src->dataLen;
tstrncpy(dst->name, src->name, tListLen(dst->name));
if (dst->dataLen > 0) {
dst->data = malloc(dst->dataLen);
if (dst->data == NULL) {
return -1;
}
memcpy(dst->data, src->data, dst->dataLen);
}
return 0;
}
STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
assert(pChild != NULL);
int32_t total = pChild->numOfColumns + pChild->numOfTags;
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
pTableMeta->tableType = TSDB_SUPER_TABLE;
pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
pTableMeta->tableInfo.precision = pChild->precision;
pTableMeta->id.tid = 0;
pTableMeta->id.uid = pChild->suid;
pTableMeta->tversion = pChild->tversion;
pTableMeta->sversion = pChild->sversion;
memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total);
int32_t num = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < num; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}
uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
return sizeof(STableMeta) + totalCols * sizeof(SSchema);
}
CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta));
cMeta->tableType = TSDB_CHILD_TABLE;
cMeta->vgId = pTableMeta->vgId;
cMeta->id = pTableMeta->id;
tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN);
return cMeta;
}
int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) {
assert(pChild != NULL);
uint32_t size = tscGetTableMetaMaxSize();
STableMeta* p = calloc(1, size);
taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1);
if (p->id.uid > 0) { // tableMeta exists, build child table meta and return
pChild->sversion = p->sversion;
pChild->tversion = p->tversion;
memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo));
int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags;
memcpy(pChild->schema, p->schema, sizeof(SSchema) *total);
tfree(p);
return TSDB_CODE_SUCCESS;
} else { // super table has been removed, current tableMeta is also expired. remove it here
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
tfree(p);
return -1;
}
}
uint32_t tscGetTableMetaMaxSize() {
return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema);
}
STableMeta* tscTableMetaClone(STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
uint32_t size = tscGetTableMetaSize(pTableMeta);
STableMeta* p = calloc(1, size);
memcpy(p, pTableMeta, size);
return p;
}

47
src/common/inc/tglobal.h
View File

@ -32,8 +32,8 @@ extern uint16_t tsSyncPort;
extern uint16_t tsArbitratorPort;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfMnodes;
extern int32_t tsEnableVnodeBak;
extern int32_t tsEnableTelemetryReporting;
extern int8_t tsEnableVnodeBak;
extern int8_t tsEnableTelemetryReporting;
extern char tsEmail[];
extern char tsArbitrator[];
@ -46,23 +46,25 @@ extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryThreads; // todo remove it
extern float tsRatioOfQueryCores;
extern int8_t tsDaylight;
extern char tsTimezone[];
extern char tsLocale[];
extern char tsCharset[]; // default encode string
extern int32_t tsEnableCoreFile;
extern int8_t tsEnableCoreFile;
extern int32_t tsCompressMsgSize;
extern char tsTempDir[];
//query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing
extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing
extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked
extern int8_t tsKeepOriginalColumnName;
// client
extern int32_t tsTableMetaKeepTimer;
extern int32_t tsMaxSQLStringLen;
extern int32_t tsTscEnableRecordSql;
extern int8_t tsTscEnableRecordSql;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
@ -91,29 +93,32 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;
extern int32_t tsUpdate;
extern int8_t tsUpdate;
extern int8_t tsCacheLastRow;
// balance
extern int32_t tsEnableBalance;
extern int32_t tsAlternativeRole;
extern int8_t tsEnableBalance;
extern int8_t tsAlternativeRole;
extern int32_t tsBalanceInterval;
extern int32_t tsOfflineThreshold;
extern int32_t tsMnodeEqualVnodeNum;
extern int32_t tsFlowCtrl;
extern int8_t tsEnableFlowCtrl;
extern int8_t tsEnableSlaveQuery;
extern int8_t tsEnableAdjustMaster;
// restful
extern int32_t tsEnableHttpModule;
extern int8_t tsEnableHttpModule;
extern int32_t tsRestRowLimit;
extern uint16_t tsHttpPort;
extern int32_t tsHttpCacheSessions;
extern int32_t tsHttpSessionExpire;
extern int32_t tsHttpMaxThreads;
extern int32_t tsHttpEnableCompress;
extern int32_t tsHttpEnableRecordSql;
extern int32_t tsTelegrafUseFieldNum;
extern int8_t tsHttpEnableCompress;
extern int8_t tsHttpEnableRecordSql;
extern int8_t tsTelegrafUseFieldNum;
// mqtt
extern int32_t tsEnableMqttModule;
extern int8_t tsEnableMqttModule;
extern char tsMqttHostName[];
extern char tsMqttPort[];
extern char tsMqttUser[];
@ -122,17 +127,17 @@ extern char tsMqttClientId[];
extern char tsMqttTopic[];
// monitor
extern int32_t tsEnableMonitorModule;
extern int8_t tsEnableMonitorModule;
extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int32_t tsMonitorInterval;
// stream
extern int32_t tsEnableStream;
extern int8_t tsEnableStream;
// internal
extern int32_t tsPrintAuth;
extern int32_t tscEmbedded;
extern int8_t tsPrintAuth;
extern int8_t tscEmbedded;
extern char configDir[];
extern char tsVnodeDir[];
extern char tsDnodeDir[];
@ -159,7 +164,7 @@ extern float tsMinimalLogDirGB;
extern float tsReservedTmpDirectorySpace;
extern float tsMinimalDataDirGB;
extern int32_t tsTotalMemoryMB;
extern int32_t tsVersion;
extern uint32_t tsVersion;
// build info
extern char version[];
@ -169,7 +174,7 @@ extern char gitinfoOfInternal[];
extern char buildinfo[];
// log
extern int32_t tsAsyncLog;
extern int8_t tsAsyncLog;
extern int32_t tsNumOfLogLines;
extern int32_t tsLogKeepDays;
extern int32_t dDebugFlag;

2
src/common/inc/tulog.h
View File

@ -23,7 +23,7 @@ extern "C" {
#include "tlog.h"
extern int32_t uDebugFlag;
extern int32_t tscEmbedded;
extern int8_t tscEmbedded;
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}

156
src/common/src/tglobal.c
View File

@ -25,7 +25,6 @@
#include "tutil.h"
#include "tlocale.h"
#include "ttimezone.h"
#include "tsync.h"
// cluster
char tsFirst[TSDB_EP_LEN] = {0};
@ -40,8 +39,8 @@ uint16_t tsSyncPort = 6040;
uint16_t tsArbitratorPort = 6042;
int32_t tsStatusInterval = 1; // second
int32_t tsNumOfMnodes = 3;
int32_t tsEnableVnodeBak = 1;
int32_t tsEnableTelemetryReporting = 1;
int8_t tsEnableVnodeBak = 1;
int8_t tsEnableTelemetryReporting = 1;
char tsEmail[TSDB_FQDN_LEN] = {0};
// common
@ -52,12 +51,12 @@ int32_t tsMaxConnections = 5000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 1;
float tsRatioOfQueryThreads = 0.5f;
float tsRatioOfQueryCores = 1.0f;
int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
int32_t tsEnableCoreFile = 0;
int8_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
@ -74,7 +73,7 @@ int32_t tsCompressMsgSize = -1;
// client
int32_t tsTableMetaKeepTimer = 7200; // second
int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
int32_t tsTscEnableRecordSql = 0;
int8_t tsTscEnableRecordSql = 0;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
@ -107,8 +106,11 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
// positive value (in MB)
int32_t tsQueryBufferSize = -1;
// only 50% cpu will be used in query processing in dnode
int32_t tsHalfCoresForQuery = 0;
// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
int32_t tsRetrieveBlockingModel = 0;
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
int8_t tsKeepOriginalColumnName = 0;
// db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
@ -124,33 +126,36 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
// balance
int32_t tsEnableBalance = 1;
int32_t tsAlternativeRole = 0;
int8_t tsEnableBalance = 1;
int8_t tsAlternativeRole = 0;
int32_t tsBalanceInterval = 300; // seconds
int32_t tsOfflineThreshold = 86400*100; // seconds 10days
int32_t tsOfflineThreshold = 86400 * 100; // seconds 10days
int32_t tsMnodeEqualVnodeNum = 4;
int32_t tsFlowCtrl = 1;
int8_t tsEnableFlowCtrl = 1;
int8_t tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1;
// restful
int32_t tsEnableHttpModule = 1;
int8_t tsEnableHttpModule = 1;
int32_t tsRestRowLimit = 10240;
uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041]
int32_t tsHttpCacheSessions = 1000;
int32_t tsHttpSessionExpire = 36000;
int32_t tsHttpMaxThreads = 2;
int32_t tsHttpEnableCompress = 1;
int32_t tsHttpEnableRecordSql = 0;
int32_t tsTelegrafUseFieldNum = 0;
int8_t tsHttpEnableCompress = 1;
int8_t tsHttpEnableRecordSql = 0;
int8_t tsTelegrafUseFieldNum = 0;
// mqtt
int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default
int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default
char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org";
char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883";
char tsMqttUser[TSDB_MQTT_USER_LEN] = {0};
@ -159,17 +164,17 @@ char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber";
char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // #
// monitor
int32_t tsEnableMonitorModule = 1;
int8_t tsEnableMonitorModule = 1;
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
int32_t tsMonitorInterval = 30; // seconds
// stream
int32_t tsEnableStream = 1;
int8_t tsEnableStream = 1;
// internal
int32_t tsPrintAuth = 0;
int32_t tscEmbedded = 0;
int8_t tsPrintAuth = 0;
int8_t tscEmbedded = 0;
char configDir[TSDB_FILENAME_LEN] = {0};
char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
@ -196,10 +201,10 @@ float tsTotalTmpDirGB = 0;
float tsTotalDataDirGB = 0;
float tsAvailTmpDirectorySpace = 0;
float tsAvailDataDirGB = 0;
float tsReservedTmpDirectorySpace = 0.1f;
float tsMinimalDataDirGB = 0.5f;
float tsReservedTmpDirectorySpace = 1.0f;
float tsMinimalDataDirGB = 1.0f;
int32_t tsTotalMemoryMB = 0;
int32_t tsVersion = 0;
uint32_t tsVersion = 0;
// log
int32_t tsNumOfLogLines = 10000000;
@ -273,12 +278,16 @@ bool taosCfgDynamicOptions(char *msg) {
for (int32_t i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
//if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue;
if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue;
if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue;
int32_t cfgLen = (int32_t)strlen(cfg->option);
if (cfgLen != olen) continue;
if (strncasecmp(option, cfg->option, olen) != 0) continue;
if (cfg->valType != TAOS_CFG_VTYPE_INT32) {
*((int32_t *)cfg->ptr) = vint;
} else {
*((int8_t *)cfg->ptr) = (int8_t)vint;
}
if (strncasecmp(cfg->option, "monitor", olen) == 0) {
if (1 == vint) {
@ -444,12 +453,12 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "ratioOfQueryThreads";
cfg.ptr = &tsRatioOfQueryThreads;
cfg.option = "ratioOfQueryCores";
cfg.ptr = &tsRatioOfQueryCores;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0.1f;
cfg.maxValue = 0.9f;
cfg.minValue = 0.0f;
cfg.maxValue = 2.0f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -466,7 +475,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "vnodeBak";
cfg.ptr = &tsEnableVnodeBak;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -476,7 +485,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "telemetryReporting";
cfg.ptr = &tsEnableTelemetryReporting;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -486,7 +495,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "balance";
cfg.ptr = &tsEnableBalance;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -507,7 +516,7 @@ static void doInitGlobalConfig(void) {
// 0-any; 1-mnode; 2-vnode
cfg.option = "role";
cfg.ptr = &tsAlternativeRole;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 2;
@ -540,7 +549,7 @@ static void doInitGlobalConfig(void) {
cfg.ptr = &tsOfflineThreshold;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 5;
cfg.minValue = 3;
cfg.maxValue = 7200000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
@ -809,7 +818,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "update";
cfg.ptr = &tsUpdate;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_UPDATE;
cfg.maxValue = TSDB_MAX_DB_UPDATE;
@ -887,8 +896,8 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg);
cfg.option = "halfCoresForQuery";
cfg.ptr = &tsHalfCoresForQuery;
cfg.option = "retrieveBlockingModel";
cfg.ptr = &tsRetrieveBlockingModel;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
@ -897,6 +906,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "keepColumnName";
cfg.ptr = &tsKeepOriginalColumnName;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// locale & charset
cfg.option = "timezone";
cfg.ptr = tsTimezone;
@ -992,8 +1011,28 @@ static void doInitGlobalConfig(void) {
// module configs
cfg.option = "flowctrl";
cfg.ptr = &tsFlowCtrl;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.ptr = &tsEnableFlowCtrl;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "slaveQuery";
cfg.ptr = &tsEnableSlaveQuery;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "adjustMaster";
cfg.ptr = &tsEnableAdjustMaster;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1003,7 +1042,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "http";
cfg.ptr = &tsEnableHttpModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1013,7 +1052,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "mqtt";
cfg.ptr = &tsEnableMqttModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1023,7 +1062,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "monitor";
cfg.ptr = &tsEnableMonitorModule;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1033,7 +1072,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "stream";
cfg.ptr = &tsEnableStream;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1043,7 +1082,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "httpEnableRecordSql";
cfg.ptr = &tsHttpEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1053,7 +1092,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "telegrafUseFieldNum";
cfg.ptr = &tsTelegrafUseFieldNum;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1104,7 +1143,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "asyncLog";
cfg.ptr = &tsAsyncLog;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1305,7 +1344,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "enableRecordSql";
cfg.ptr = &tsTscEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1315,7 +1354,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "enableCoreFile";
cfg.ptr = &tsEnableCoreFile;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
@ -1431,18 +1470,29 @@ int32_t taosCheckGlobalCfg() {
tsNumOfCores = 1;
}
if (tsMaxTablePerVnode < tsMinTablePerVnode) {
uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)",
tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode);
tsMaxTablePerVnode = tsMinTablePerVnode;
}
// todo refactor
tsVersion = 0;
for (int i = 0; i < 10; i++) {
for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) {
if (version[i] >= '0' && version[i] <= '9') {
tsVersion = tsVersion * 10 + (version[i] - '0');
ver = ver * 10 + (version[i] - '0');
} else if (version[i] == '.') {
tsVersion |= ver & 0xFF;
tsVersion <<= 8;
ver = 0;
} else if (version[i] == 0) {
tsVersion |= ver & 0xFF;
break;
}
}
tsVersion = 10 * tsVersion;
tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035]
tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp
tsSyncPort = tsServerPort + TSDB_PORT_SYNC;

2
src/common/src/tvariant.c
View File

@ -705,7 +705,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
*((int32_t *)payload) = TSDB_DATA_FLOAT_NULL;
return 0;
} else {
double value;
double value = -1;
int32_t ret;
ret = convertToDouble(pVariant->pz, pVariant->nLen, &value);
if ((errno == ERANGE && (float)value == -1) || (ret != 0)) {

2
src/connector/grafanaplugin

@ -1 +1 @@
Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944
Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df

35
src/connector/nodejs/nodetaos/cinterface.js
View File

@ -144,18 +144,9 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) {
function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) {
data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset);
let res = [];
let currOffset = 0;
// every 4 bytes, a character is encoded;
while (currOffset < data.length) {
let dataEntry = data.slice(currOffset, currOffset + nbytes); //one entry in a row under a column;
if (dataEntry.readInt64LE(0) == FieldTypes.C_NCHAR_NULL) {
res.push(null);
}
else {
res.push(dataEntry.toString("utf16le").replace(/\u0000/g, ""));
}
currOffset += nbytes;
}
let dataEntry = data.slice(0, nbytes); //one entry in a row under a column;
//TODO: should use the correct character encoding
res.push(dataEntry.toString("utf-8"));
return res;
}
@ -349,11 +340,13 @@ CTaosInterface.prototype.useResult = function useResult(result) {
return fields;
}
CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
let num_of_rows = this.libtaos.taos_fetch_block(result, pblock)
if (num_of_rows == 0) {
//let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
let pblock = this.libtaos.taos_fetch_row(result);
let num_of_rows = 1;
if (ref.isNull(pblock) == true) {
return {block:null, num_of_rows:0};
}
var fieldL = this.libtaos.taos_fetch_lengths(result);
let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
@ -361,7 +354,6 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
var fieldlens = [];
if (ref.isNull(fieldL) == false) {
for (let i = 0; i < fields.length; i ++) {
let plen = ref.reinterpret(fieldL, 4, i*4);
let len = plen.readInt32LE(0);
@ -371,17 +363,20 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
let blocks = new Array(fields.length);
blocks.fill(null);
num_of_rows = Math.abs(num_of_rows);
//num_of_rows = Math.abs(num_of_rows);
let offset = 0;
pblock = pblock.deref();
for (let i = 0; i < fields.length; i++) {
pdata = ref.reinterpret(pblock,8,i*8);
if(ref.isNull(pdata.readPointer())){
blocks[i] = new Array();
}else{
pdata = ref.ref(pdata.readPointer());
if (!convertFunctions[fields[i]['type']] ) {
throw new errors.DatabaseError("Invalid data type returned from database");
}
blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro);
}
}
return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)}
}
CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) {
@ -446,6 +441,9 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback,
}
if (numOfRows2 > 0){
for (let i = 0; i < fields.length; i++) {
if(ref.isNull(pdata.readPointer())){
blocks[i] = new Array();
}else{
if (!convertFunctions[fields[i]['type']] ) {
throw new errors.DatabaseError("Invalid data type returned from database");
}
@ -456,6 +454,7 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback,
//offset += fields[i]['bytes'] * numOfRows2;
}
}
}
callback(param2, result2, numOfRows2, blocks);
}
asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper);

4
src/connector/nodejs/nodetaos/taosresult.js
View File

@ -25,6 +25,7 @@ function TaosResult(data, fields) {
* @function pretty
* @since 1.0.6
*/
TaosResult.prototype.pretty = function pretty() {
let fieldsStr = "";
let sizing = [];
@ -46,8 +47,7 @@ TaosResult.prototype.pretty = function pretty() {
row.data.forEach((entry, i) => {
if (this.fields[i]._field.type == 9) {
entry = entry.toTaosString();
}
else {
} else {
entry = entry == null ? 'null' : entry.toString();
}
rowStr += entry

2
src/connector/nodejs/package.json
View File

@ -1,6 +1,6 @@
{
"name": "td2.0-connector",
"version": "2.0.1",
"version": "2.0.4",
"description": "A Node.js connector for TDengine.",
"main": "tdengine.js",
"scripts": {

32
src/connector/nodejs/test/test.js
View File

@ -48,6 +48,7 @@ for (let i = 0; i < 10000; i++) {
// Select
console.log('select * from td_connector_test.all_types limit 3 offset 100;');
c1.execute('select * from td_connector_test.all_types limit 2 offset 100;');
var d = c1.fetchall();
console.log(c1.fields);
console.log(d);
@ -77,13 +78,33 @@ c1.query('select stddev(_double), stddev(_bigint), stddev(_float) from all_types
})
// Binding arguments, and then using promise
var q = c1.query('select * from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100)
var q = c1.query('select _nchar from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100)
console.log(q.query);
q.execute().then(function(r) {
r.pretty();
});
// test query null value
c1.execute("create table if not exists td_connector_test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64))");
c1.execute("insert into t1 using weather tags('北京') values(now, 11.11, 11)");
c1.execute("insert into t1(ts, temperature) values(now, 22.22)");
c1.execute("insert into t1(ts, humidity) values(now, 33)");
c1.query('select * from test.t1', true).then(function (result) {
result.pretty();
});
var q = c1.query('select * from td_connector_test.weather');
console.log(q.query);
q.execute().then(function(r) {
r.pretty();
});
function sleep(sleepTime) {
for(var start = +new Date; +new Date - start <= sleepTime; ) { }
}
sleep(10000);
// Raw Async Testing (Callbacks, not promises)
function cb2(param, result, rowCount, rd) {
@ -129,16 +150,21 @@ setTimeout(function(){
c1.fetchall_a(thisRes, cb4, param);
},100);
// Async through promises
var aq = c1.query('select count(*) from td_connector_test.all_types;',false);
aq.execute_a().then(function(data) {
data.pretty();
});
c1.query('describe td_connector_test.stabletest;').execute_a().then(r=> r.pretty());
c1.query('describe td_connector_test.stabletest').execute_a().then(function(r){
r.pretty()
});
setTimeout(function(){
c1.query('drop database td_connector_test;');
},200);
setTimeout(function(){
conn.close();
},2000);

8
src/cq/src/cqMain.c
View File

@ -97,7 +97,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
pthread_mutex_init(&pContext->mutex, NULL);
cInfo("vgId:%d, CQ is opened", pContext->vgId);
cDebug("vgId:%d, CQ is opened", pContext->vgId);
return pContext;
}
@ -131,7 +131,7 @@ void cqClose(void *handle) {
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
cInfo("vgId:%d, CQ is closed", pContext->vgId);
cDebug("vgId:%d, CQ is closed", pContext->vgId);
free(pContext);
}
@ -142,7 +142,7 @@ void cqStart(void *handle) {
SCqContext *pContext = handle;
if (pContext->dbConn || pContext->master) return;
cInfo("vgId:%d, start all CQs", pContext->vgId);
cDebug("vgId:%d, start all CQs", pContext->vgId);
pthread_mutex_lock(&pContext->mutex);
pContext->master = 1;
@ -298,7 +298,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
if (pObj->pStream) {
tscSetStreamDestTable(pObj->pStream, pObj->dstTable);
pContext->num++;
cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
cDebug("vgId:%d, id:%d CQ:%s is opened", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
}

1
src/dnode/inc/dnodeCfg.h
View File

@ -25,6 +25,7 @@ int32_t dnodeInitCfg();
void dnodeCleanupCfg();
void dnodeUpdateCfg(SDnodeCfg *cfg);
int32_t dnodeGetDnodeId();
void dnodeGetClusterId(char *clusterId);
void dnodeGetCfg(int32_t *dnodeId, char *clusterId);
#ifdef __cplusplus

6
src/dnode/src/dnodeCfg.c
View File

@ -51,6 +51,12 @@ int32_t dnodeGetDnodeId() {
return dnodeId;
}
void dnodeGetClusterId(char *clusterId) {
pthread_mutex_lock(&tsCfgMutex);
tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN);
pthread_mutex_unlock(&tsCfgMutex);
}
void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
pthread_mutex_lock(&tsCfgMutex);
*dnodeId = tsCfg.dnodeId;

8
src/dnode/src/dnodeMInfos.c
View File

@ -311,6 +311,14 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) {
if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) ||
(epSet.port[i] == tsServerPort && forShell)) {
epSet.inUse = (i + 1) % epSet.numOfEps;
dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse);
}
}
epSet.port[i] = htons(epSet.port[i]);
}

2
src/dnode/src/dnodeMRead.c
View File

@ -16,9 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tqueue.h"
#include "twal.h"
#include "mnode.h"
#include "dnodeVMgmt.h"
#include "dnodeMInfos.h"
#include "dnodeMRead.h"

15
src/dnode/src/dnodeMWrite.c
View File

@ -18,7 +18,6 @@
#include "ttimer.h"
#include "tqueue.h"
#include "mnode.h"
#include "dnodeVMgmt.h"
#include "dnodeMInfos.h"
#include "dnodeMWrite.h"
@ -185,7 +184,19 @@ void dnodeReprocessMWriteMsg(void *pMsg) {
dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle,
taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
dnodeSendRedirectMsg(pMsg, true);
if (pWrite->pBatchMasterMsg) {
++pWrite->pBatchMasterMsg->received;
if (pWrite->pBatchMasterMsg->successed + pWrite->pBatchMasterMsg->received
>= pWrite->pBatchMasterMsg->expected) {
dnodeSendRedirectMsg(&pWrite->rpcMsg, true);
dnodeFreeMWriteMsg(pWrite);
}
mnodeDestroySubMsg(pWrite);
return;
}
dnodeSendRedirectMsg(&pWrite->rpcMsg, true);
dnodeFreeMWriteMsg(pWrite);
} else {
dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle,

3
src/dnode/src/dnodeMain.c
View File

@ -113,6 +113,7 @@ static void dnodeCleanupTmr() {
int32_t dnodeInitSystem() {
dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
tscEmbedded = 1;
taosIgnSIGPIPE();
taosBlockSIGPIPE();
taosResolveCRC();
taosInitGlobalCfg();
@ -120,7 +121,6 @@ int32_t dnodeInitSystem() {
taosSetCoreDump();
taosInitNotes();
dnodeInitTmr();
signal(SIGPIPE, SIG_IGN);
if (dnodeCreateDir(tsLogDir) < 0) {
printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
@ -147,6 +147,7 @@ int32_t dnodeInitSystem() {
dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
dnodeReportStep("TDengine", "initialized successfully", 1);
dInfo("TDengine is initialized successfully");
return 0;

3
src/dnode/src/dnodeShell.c
View File

@ -70,8 +70,7 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0;
if (numOfThreads < 1) {
numOfThreads = 1;
}

5
src/dnode/src/dnodeStep.c
View File

@ -57,12 +57,13 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) {
int32_t code = (*pStep->initFp)();
if (code != 0) {
dDebug("step:%s will init", pStep->name);
dDebug("step:%s will cleanup", pStep->name);
taosStepCleanupImp(pSteps, step);
return code;
}
dInfo("step:%s is initialized", pStep->name);
dnodeReportStep(pStep->name, "Initialization complete", step + 1 >= stepSize);
dnodeReportStep(pStep->name, "Initialization complete", 0);
}
return 0;

2
src/dnode/src/dnodeTelemetry.c
View File

@ -195,7 +195,7 @@ static void addRuntimeInfo(SBufferWriter* bw) {
static void sendTelemetryReport() {
char buf[128];
uint32_t ip = taosGetIpFromFqdn(TELEMETRY_SERVER);
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
if (ip == 0xffffffff) {
dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno));
return;

5
src/dnode/src/dnodeVMgmt.c
View File

@ -129,7 +129,8 @@ static void *dnodeProcessMgmtQueue(void *wparam) {
static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion);
pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize);
pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks);
@ -142,7 +143,7 @@ static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
}

17
src/dnode/src/dnodeVRead.c
View File

@ -26,16 +26,20 @@ static SWorkerPool tsVQueryWP;
static SWorkerPool tsVFetchWP;
int32_t dnodeInitVRead() {
const int32_t maxFetchThreads = 4;
// calculate the available query thread
float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
tsVQueryWP.name = "vquery";
tsVQueryWP.workerFp = dnodeProcessReadQueue;
tsVQueryWP.min = tsNumOfCores;
tsVQueryWP.max = tsNumOfCores/* * tsNumOfThreadsPerCore*/;
// if (tsVQueryWP.max <= tsVQueryWP.min * 2) tsVQueryWP.max = 2 * tsVQueryWP.min;
tsVQueryWP.min = (int32_t) threadsForQuery;
tsVQueryWP.max = tsVQueryWP.min;
if (tWorkerInit(&tsVQueryWP) != 0) return -1;
tsVFetchWP.name = "vfetch";
tsVFetchWP.workerFp = dnodeProcessReadQueue;
tsVFetchWP.min = MIN(4, tsNumOfCores);
tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores);
tsVFetchWP.max = tsVFetchWP.min;
if (tWorkerInit(&tsVFetchWP) != 0) return -1;
@ -50,6 +54,7 @@ void dnodeCleanupVRead() {
void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
int32_t queuedMsgNum = 0;
int32_t leftLen = pMsg->contLen;
int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID;
char * pCont = pMsg->pCont;
while (leftLen > 0) {
@ -60,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
assert(pHead->contLen > 0);
void *pVnode = vnodeAcquire(pHead->vgId);
if (pVnode != NULL) {
int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
vnodeRelease(pVnode);
}
@ -70,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
}
if (queuedMsgNum == 0) {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rpcRsp);
}

1
src/dnode/src/dnodeVWrite.c
View File

@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
int32_t numOfMsgs;
int32_t qtype;
taosBlockSIGPIPE();
dDebug("dnode vwrite worker:%d is running", pWorker->workerId);
while (1) {

28
src/dnode/src/dnodeVnodes.c
View File

@ -33,6 +33,8 @@ typedef struct {
extern void * tsDnodeTmr;
static void * tsStatusTimer = NULL;
static uint32_t tsRebootTime = 0;
static int32_t tsOpenVnodes = 0;
static int32_t tsTotalVnodes = 0;
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
@ -71,7 +73,8 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
if (*numOfVnodes >= TSDB_MAX_VNODES) {
dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
continue;
closedir(dir);
return TSDB_CODE_DND_TOO_MANY_VNODES;
} else {
vnodeList[*numOfVnodes - 1] = vnode;
}
@ -84,21 +87,27 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
static void *dnodeOpenVnode(void *param) {
SOpenVnodeThread *pThread = param;
char stepDesc[TSDB_STEP_DESC_LEN] = {0};
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
int32_t vgId = pThread->vnodeList[v];
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, tsOpenVnodes, tsTotalVnodes);
dnodeReportStep("open-vnodes", stepDesc, 0);
if (vnodeOpen(vgId) < 0) {
dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
pThread->failed++;
} else {
dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
dDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex);
pThread->opened++;
}
atomic_add_fetch_32(&tsOpenVnodes, 1);
}
dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
return NULL;
}
@ -107,6 +116,7 @@ int32_t dnodeInitVnodes() {
int32_t vnodeList[TSDB_MAX_VNODES] = {0};
int32_t numOfVnodes = 0;
int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
tsTotalVnodes = numOfVnodes;
if (status != TSDB_CODE_SUCCESS) {
dInfo("get dnode list failed");
@ -127,7 +137,7 @@ int32_t dnodeInitVnodes() {
pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
}
dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
@ -156,7 +166,7 @@ int32_t dnodeInitVnodes() {
}
free(threads);
dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes);
dInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes);
if (failedVnodes != 0) {
dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
@ -236,12 +246,11 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
pStatus->alternativeRole = tsAlternativeRole;
tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
// fill cluster cfg parameters
pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes);
pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance);
pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold);
pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
@ -255,6 +264,11 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
pStatus->clusterCfg.enableBalance = tsEnableBalance;
pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl;
pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery;
pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster;
vnodeBuildStatusMsg(pStatus);
contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);

2
src/inc/dnode.h
View File

@ -36,6 +36,8 @@ bool dnodeIsMasterEp(char *ep);
void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
void dnodeGetEpSetForShell(SRpcEpSet *epSet);
int32_t dnodeGetDnodeId();
void dnodeGetClusterId(char *clusterId);
void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
bool dnodeStartMnode(SMInfos *pMinfos);

10
src/inc/mnode.h
View File

@ -42,11 +42,12 @@ typedef struct SMnodeMsg {
struct SVgObj * pVgroup;
struct STableObj *pTable;
struct SSTableObj*pSTable;
struct SMnodeMsg *pBatchMasterMsg;
SMnodeRsp rpcRsp;
int8_t received;
int8_t successed;
int8_t expected;
int8_t retry;
int16_t received;
int16_t successed;
int16_t expected;
int16_t retry;
int32_t incomingTs;
int32_t code;
void * pObj;
@ -57,6 +58,7 @@ typedef struct SMnodeMsg {
void * mnodeCreateMsg(SRpcMsg *pRpcMsg);
int32_t mnodeInitMsg(SMnodeMsg *pMsg);
void mnodeCleanupMsg(SMnodeMsg *pMsg);
void mnodeDestroySubMsg(SMnodeMsg *pSubMsg);
int32_t mnodeInitSystem();
int32_t mnodeStartSystem();

6
src/inc/taosdef.h
View File

@ -369,6 +369,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_DB_UPDATE 1
#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
#define TSDB_MIN_DB_CACHE_LAST_ROW 0
#define TSDB_MAX_DB_CACHE_LAST_ROW 1
#define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
@ -432,7 +436,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_ARBITRATOR 12
#define TSDB_MAX_WAL_SIZE (1024*1024)
#define TSDB_MAX_WAL_SIZE (1024*1024*2)
typedef enum {
TAOS_QTYPE_RPC = 0,

Some files were not shown because too many files have changed in this diff Show More