xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into docs/sangshuduo/update-doc-align-with-1.0.5

This commit is contained in:
Shuduo Sang 2020-12-14 09:04:50 +08:00
parent e15f456d3d 234fd54ef5
commit cf4f0d9561
135 changed files with 5038 additions and 751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
Jenkinsfile vendored
View File

@ -1,3 +1,47 @@
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine'
}
// execute this before anything else, including requesting any time on an agent
if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) {
print "INFO: Build skipped due to trigger being Branch Indexing"
currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful
return
}
def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
sudo rmtaos
'''
}
sh '''
cd ${WKC}
rm -rf *
cd ${WK}
git reset --hard
git checkout develop
git pull
cd ${WKC}
rm -rf *
mv ${WORKSPACE}/* .
cd ${WK}
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests
'''
return 1
}
pipeline { pipeline {
agent none agent none
environment{ environment{
@ -8,85 +52,31 @@ pipeline {
stages { stages {
stage('Parallel test stage') { stage('Parallel test stage') {
parallel { parallel {
stage('pytest') { stage('python p1') {
agent{label '184'} agent{label 'p1'}
steps { steps {
pre_test()
sh ''' sh '''
date
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke ./test-all.sh p1
./test-all.sh pytest
date''' date'''
} }
} }
stage('test_b1') { stage('test_b1') {
agent{label 'master'} agent{label 'b1'}
steps { steps {
pre_test()
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh b1 ./test-all.sh b1
date''' date'''
} }
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "185"} agent{label "b2"}
steps { steps {
sh ''' pre_test()
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
@ -109,193 +99,42 @@ pipeline {
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "186"} agent{label "b3"}
steps { steps {
pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log ./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh ./handle_val_log.sh
'''
}
sh '''
date date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b3 ./test-all.sh b3
date''' date'''
} }
} }
stage('connector'){ stage('python p2'){
agent{label "release"} agent{label "p2"}
steps{ steps{
pre_test()
sh ''' sh '''
cd ${WORKSPACE} date
git checkout develop cd ${WKC}/tests
./test-all.sh p2
date
''' '''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
} }
} }
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
} }
} }
} }
post {
success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
} }

12
packaging/cfg/taos.cfg
View File

@ -29,8 +29,12 @@
# number of threads per CPU core # number of threads per CPU core
# numOfThreadsPerCore 1.0 # numOfThreadsPerCore 1.0
# the proportion of total threads responsible for query # the proportion of total CPU cores available for query processing
# ratioOfQueryThreads 0.5 # 2.0: the query threads will be set to double of the CPU cores.
# 1.0: all CPU cores are available for query processing [default].
# 0.5: only half of the CPU cores are available for query.
# 0.0: only one core available.
# tsRatioOfQueryCores 1.0
# number of management nodes in the system # number of management nodes in the system
# numOfMnodes 3 # numOfMnodes 3
@ -265,5 +269,5 @@
# enable/disable stream (continuous query) # enable/disable stream (continuous query)
# stream 1 # stream 1
# only 50% CPU resources will be used in query processing # in retrieve blocking model, only in 50% query threads will be used in query processing in dnode
# halfCoresForQuery 0 # retrieveBlockingModel 0

5
src/client/src/tscAsync.c
View File

@ -365,6 +365,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
static void tscProcessAsyncError(SSchedMsg *pMsg) { static void tscProcessAsyncError(SSchedMsg *pMsg) {
void (*fp)() = pMsg->ahandle; void (*fp)() = pMsg->ahandle;
terrno = *(int32_t*) pMsg->msg; terrno = *(int32_t*) pMsg->msg;
tfree(pMsg->msg);
(*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg); (*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg);
} }
@ -447,9 +448,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pCmd->command != TSDB_SQL_INSERT); assert(pCmd->command != TSDB_SQL_INSERT);
// in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema.
if (pCmd->command == TSDB_SQL_SELECT) { if (pCmd->command == TSDB_SQL_SELECT) {
tscDebug("%p redo parse sql string and proceed", pSql); tscDebug("%p redo parse sql string and proceed", pSql);
pCmd->parseFinished = false; pCmd->parseFinished = false;
@ -463,7 +461,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} }
tscProcessSql(pSql); tscProcessSql(pSql);
} else { // in all other cases, simple retry } else { // in all other cases, simple retry
tscProcessSql(pSql); tscProcessSql(pSql);
} }

139
src/client/src/tscFunctionImpl.c
View File

@ -3648,11 +3648,21 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN; pInfo->p.key = INT64_MIN;
pInfo->win = TSWINDOW_INITIALIZER; pInfo->win = TSWINDOW_INITIALIZER;
return true; return true;
} }
static double twa_get_area(SPoint1 s, SPoint1 e) {
if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) {
return (s.val + e.val) * (e.key - s.key) / 2;
}
double x = (s.key * e.val - e.key * s.val)/(e.val - s.val);
double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2;
return val;
}
static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
int32_t notNullElems = 0; int32_t notNullElems = 0;
TSKEY *primaryKey = pCtx->ptsList; TSKEY *primaryKey = pCtx->ptsList;
@ -3663,28 +3673,29 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
int32_t i = index; int32_t i = index;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
SPoint1* last = &pInfo->p;
if (pCtx->start.key != INT64_MIN) { if (pCtx->start.key != INT64_MIN) {
assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
(pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
assert(pInfo->lastKey == INT64_MIN); assert(last->key == INT64_MIN);
pInfo->lastKey = primaryKey[tsIndex + i]; last->key = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->dOutput += twa_get_area(pCtx->start, *last);
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key; pInfo->win.skey = pCtx->start.key;
notNullElems++; notNullElems++;
i += step; i += step;
} else if (pInfo->lastKey == INT64_MIN) { } else if (pInfo->p.key == INT64_MIN) {
pInfo->lastKey = primaryKey[tsIndex + i]; last->key = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey; pInfo->win.skey = last->key;
notNullElems++; notNullElems++;
i += step; i += step;
} }
@ -3698,9 +3709,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3711,9 +3722,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3724,9 +3735,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3737,9 +3748,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = (double) val[i]};
pInfo->lastValue = (double) val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3750,9 +3761,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3763,9 +3774,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + tsIndex]; pInfo->p = st;
} }
break; break;
} }
@ -3774,12 +3785,11 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t
// the last interpolated time window value // the last interpolated time window value
if (pCtx->end.key != INT64_MIN) { if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);
pInfo->lastValue = pCtx->end.val; pInfo->p = pCtx->end;
pInfo->lastKey = pCtx->end.key;
} }
pInfo->win.ekey = pInfo->lastKey; pInfo->win.ekey = pInfo->p.key;
return notNullElems; return notNullElems;
} }
@ -3808,6 +3818,7 @@ static void twa_function(SQLFunctionCtx *pCtx) {
} }
} }
//TODO refactor
static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
@ -3824,23 +3835,23 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
int32_t size = pCtx->size; int32_t size = pCtx->size;
if (pCtx->start.key != INT64_MIN) { if (pCtx->start.key != INT64_MIN) {
assert(pInfo->lastKey == INT64_MIN); assert(pInfo->p.key == INT64_MIN);
pInfo->lastKey = primaryKey[index]; pInfo->p.key = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->dOutput += twa_get_area(pCtx->start, pInfo->p);
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key; pInfo->win.skey = pCtx->start.key;
notNullElems++; notNullElems++;
i += 1; i += 1;
} else if (pInfo->lastKey == INT64_MIN) { } else if (pInfo->p.key == INT64_MIN) {
pInfo->lastKey = primaryKey[index]; pInfo->p.key = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey; pInfo->win.skey = pInfo->p.key;
notNullElems++; notNullElems++;
i += 1; i += 1;
} }
@ -3854,9 +3865,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3867,9 +3878,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3880,9 +3891,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3893,9 +3904,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = (double) val[i]};
pInfo->lastValue = (double) val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3906,9 +3917,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3919,9 +3930,9 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
continue; continue;
} }
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); SPoint1 st = {.key = primaryKey[i + index], .val = val[i]};
pInfo->lastValue = val[i]; pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key);
pInfo->lastKey = primaryKey[i + index]; pInfo->p = st;
} }
break; break;
} }
@ -3930,12 +3941,11 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// the last interpolated time window value // the last interpolated time window value
if (pCtx->end.key != INT64_MIN) { if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);//((pInfo->p.val + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->p.key);
pInfo->lastValue = pCtx->end.val; pInfo->p = pCtx->end;
pInfo->lastKey = pCtx->end.key;
} }
pInfo->win.ekey = pInfo->lastKey; pInfo->win.ekey = pInfo->p.key;
SET_VAL(pCtx, notNullElems, 1); SET_VAL(pCtx, notNullElems, 1);
@ -3966,7 +3976,7 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
pBuf->dOutput += pInput->dOutput; pBuf->dOutput += pInput->dOutput;
pBuf->win = pInput->win; pBuf->win = pInput->win;
pBuf->lastKey = pInput->lastKey; pBuf->p = pInput->p;
} }
SET_VAL(pCtx, numOfNotNull, 1); SET_VAL(pCtx, numOfNotNull, 1);
@ -3993,15 +4003,14 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->hasResult != DATA_SET_FLAG) { if (pInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
return; return;
} }
assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->win.ekey == pInfo->win.skey) { if (pInfo->win.ekey == pInfo->win.skey) {
*(double *)pCtx->aOutputBuf = pInfo->lastValue; *(double *)pCtx->aOutputBuf = pInfo->p.val;
} else { } else {
*(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey);
} }

15
src/client/src/tscParseInsert.c
View File

@ -1446,18 +1446,21 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
int32_t count = 0; int32_t count = 0;
int32_t maxRows = 0; int32_t maxRows = 0;
tscDestroyBlockArrayList(pSql->cmd.pDataBlocks); tfree(pCmd->pTableMetaList);
pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
if (pCmd->pTableBlockHashList == NULL) {
pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
}
STableDataBlocks *pTableDataBlock = NULL; STableDataBlocks *pTableDataBlock = NULL;
int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE,
sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
// return ret; // return ret;
} }
taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock);
tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows);
char *tokenBuf = calloc(1, 4096); char *tokenBuf = calloc(1, 4096);
while ((readLen = tgetline(&line, &n, fp)) != -1) { while ((readLen = tgetline(&line, &n, fp)) != -1) {
@ -1519,8 +1522,6 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport));
SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES);
pCmd->count = 1; pCmd->count = 1;
FILE *fp = fopen(pCmd->payload, "r"); FILE *fp = fopen(pCmd->payload, "r");

12
src/client/src/tscSQLParser.c
View File

@ -5913,12 +5913,16 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
if (pExprList->nExpr != 1) { if (pExprList->nExpr != 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }
bool server_status = false;
tSQLExpr* pExpr = pExprList->a[0].pNode; tSQLExpr* pExpr = pExprList->a[0].pNode;
if (pExpr->operand.z == NULL) { if (pExpr->operand.z == NULL) {
//handle 'select 1'
if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) {
server_status = true;
} else {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
}
// TODO redefine the function // TODO redefine the function
SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, SDNodeDynConfOption functionsInfo[5] = {{"database()", 10},
{"server_version()", 16}, {"server_version()", 16},
@ -5927,6 +5931,9 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
{"current_user()", 14}}; {"current_user()", 14}};
int32_t index = -1; int32_t index = -1;
if (server_status == true) {
index = 2;
} else {
for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { for (int32_t i = 0; i < tListLen(functionsInfo); ++i) {
if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 &&
functionsInfo[i].len == pExpr->operand.n) { functionsInfo[i].len == pExpr->operand.n) {
@ -5934,6 +5941,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
break; break;
} }
} }
}
switch (index) { switch (index) {
case 0: case 0:

13
src/client/src/tscServer.c
View File

@ -152,8 +152,14 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
SRpcEpSet * epSet = &pRsp->epSet; SRpcEpSet * epSet = &pRsp->epSet;
if (epSet->numOfEps > 0) { if (epSet->numOfEps > 0) {
tscEpSetHtons(epSet); tscEpSetHtons(epSet);
if (!tscEpSetIsEqual(&pSql->pTscObj->tscCorMgmtEpSet->epSet, epSet)) {
tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse);
for (int8_t i = 0; i < epSet->numOfEps; i++) {
tscTrace("endpoint %d: fqdn = %s, port=%d", i, epSet->fqdn[i], epSet->port[i]);
}
tscUpdateMgmtEpSet(pSql, epSet); tscUpdateMgmtEpSet(pSql, epSet);
} }
}
pSql->pTscObj->connId = htonl(pRsp->connId); pSql->pTscObj->connId = htonl(pRsp->connId);
@ -208,7 +214,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
STscObj* pObj = pSql->pTscObj; STscObj* pObj = pSql->pTscObj;
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
char *pMsg = rpcMallocCont(pCmd->payloadLen); char *pMsg = rpcMallocCont(sizeof(SMsgVersion) + pCmd->payloadLen);
if (NULL == pMsg) { if (NULL == pMsg) {
tscError("%p msg:%s malloc failed", pSql, taosMsg[pSql->cmd.msgType]); tscError("%p msg:%s malloc failed", pSql, taosMsg[pSql->cmd.msgType]);
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -219,12 +225,13 @@ int tscSendMsgToServer(SSqlObj *pSql) {
tscDumpMgmtEpSet(pSql); tscDumpMgmtEpSet(pSql);
} }
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); tstrncpy(pMsg, version, sizeof(SMsgVersion));
memcpy(pMsg + sizeof(SMsgVersion), pSql->cmd.payload, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.msgType = pSql->cmd.msgType, .msgType = pSql->cmd.msgType,
.pCont = pMsg, .pCont = pMsg,
.contLen = pSql->cmd.payloadLen, .contLen = pSql->cmd.payloadLen + sizeof(SMsgVersion),
.ahandle = (void*)pSql->self, .ahandle = (void*)pSql->self,
.handle = NULL, .handle = NULL,
.code = 0 .code = 0

20
src/client/src/tscSubquery.c
View File

@ -2172,6 +2172,15 @@ static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) {
return true; return true;
} }
static void doFreeInsertSupporter(SSqlObj* pSqlObj) {
assert(pSqlObj != NULL && pSqlObj->subState.numOfSub > 0);
for(int32_t i = 0; i < pSqlObj->subState.numOfSub; ++i) {
SSqlObj* pSql = pSqlObj->pSubs[i];
tfree(pSql->param);
}
}
static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) { static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
SInsertSupporter *pSupporter = (SInsertSupporter *)param; SInsertSupporter *pSupporter = (SInsertSupporter *)param;
SSqlObj* pParentObj = pSupporter->pSql; SSqlObj* pParentObj = pSupporter->pSql;
@ -2203,10 +2212,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
if (pParentObj->res.code == TSDB_CODE_SUCCESS) { if (pParentObj->res.code == TSDB_CODE_SUCCESS) {
tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows);
for(int32_t i = 0; i < numOfSub; ++i) { doFreeInsertSupporter(pParentObj);
SSqlObj* pSql = pParentObj->pSubs[i];
tfree(pSql->param);
}
// todo remove this parameter in async callback function definition. // todo remove this parameter in async callback function definition.
// all data has been sent to vnode, call user function // all data has been sent to vnode, call user function
@ -2214,6 +2220,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
(*pParentObj->fp)(pParentObj->param, pParentObj, v); (*pParentObj->fp)(pParentObj->param, pParentObj, v);
} else { } else {
if (!needRetryInsert(pParentObj, numOfSub)) { if (!needRetryInsert(pParentObj, numOfSub)) {
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj); tscQueueAsyncRes(pParentObj);
return; return;
} }
@ -2244,16 +2251,19 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->cmd.parseFinished = false; pParentObj->cmd.parseFinished = false;
pParentObj->subState.numOfRemain = numOfFailed; pParentObj->subState.numOfRemain = numOfFailed;
pParentObj->subState.numOfSub = numOfFailed;
tscResetSqlCmdObj(&pParentObj->cmd, false); tscResetSqlCmdObj(&pParentObj->cmd, false);
// in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.
// 2. vnode may need the schema information along with submit block to update its local table schema.
tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry++); tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry++);
int32_t code = tsParseSql(pParentObj, true); int32_t code = tsParseSql(pParentObj, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pParentObj->res.code = code; pParentObj->res.code = code;
doFreeInsertSupporter(pParentObj);
tscQueueAsyncRes(pParentObj); tscQueueAsyncRes(pParentObj);
return; return;
} }

4
src/client/src/tscUtil.c
View File

@ -2044,7 +2044,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pnCmd->numOfClause = 0; pnCmd->numOfClause = 0;
pnCmd->clauseIndex = 0; pnCmd->clauseIndex = 0;
pnCmd->pDataBlocks = NULL; pnCmd->pDataBlocks = NULL;
pnCmd->numOfTables = 0;
pnCmd->parseFinished = 1; pnCmd->parseFinished = 1;
pnCmd->pTableMetaList = NULL;
pnCmd->pTableBlockHashList = NULL;
if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;

4
src/common/inc/tglobal.h
View File

@ -46,7 +46,7 @@ extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl; extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore; extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryThreads; // todo remove it extern float tsRatioOfQueryCores;
extern int8_t tsDaylight; extern int8_t tsDaylight;
extern char tsTimezone[]; extern char tsTimezone[];
extern char tsLocale[]; extern char tsLocale[];
@ -57,7 +57,7 @@ extern char tsTempDir[];
//query buffer management //query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing
extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing extern int32_t tsRetrieveBlockingModel; // only 50% will be used in query processing
// client // client
extern int32_t tsTableMetaKeepTimer; extern int32_t tsTableMetaKeepTimer;

20
src/common/src/tglobal.c
View File

@ -52,7 +52,7 @@ int32_t tsMaxConnections = 5000;
int32_t tsShellActivityTimer = 3; // second int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f; float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 1; int32_t tsNumOfCommitThreads = 1;
float tsRatioOfQueryThreads = 0.5f; float tsRatioOfQueryCores = 1.0f;
int8_t tsDaylight = 0; int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0};
@ -107,8 +107,8 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
// positive value (in MB) // positive value (in MB)
int32_t tsQueryBufferSize = -1; int32_t tsQueryBufferSize = -1;
// only 50% cpu will be used in query processing in dnode // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
int32_t tsHalfCoresForQuery = 0; int32_t tsRetrieveBlockingModel = 0;
// db parameters // db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
@ -206,7 +206,7 @@ int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 131; int32_t mDebugFlag = 131;
int32_t sdbDebugFlag = 131; int32_t sdbDebugFlag = 131;
int32_t dDebugFlag = 135; int32_t dDebugFlag = 135;
int32_t vDebugFlag = 131; int32_t vDebugFlag = 135;
int32_t cDebugFlag = 131; int32_t cDebugFlag = 131;
int32_t jniDebugFlag = 131; int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131; int32_t odbcDebugFlag = 131;
@ -444,12 +444,12 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "ratioOfQueryThreads"; cfg.option = "ratioOfQueryCores";
cfg.ptr = &tsRatioOfQueryThreads; cfg.ptr = &tsRatioOfQueryCores;
cfg.valType = TAOS_CFG_VTYPE_FLOAT; cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0.1f; cfg.minValue = 0.0f;
cfg.maxValue = 0.9f; cfg.maxValue = 2.0f;
cfg.ptrLength = 0; cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
@ -887,8 +887,8 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_BYTE; cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "halfCoresForQuery"; cfg.option = "retrieveBlockingModel";
cfg.ptr = &tsHalfCoresForQuery; cfg.ptr = &tsRetrieveBlockingModel;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0; cfg.minValue = 0;

2
src/cq/src/cqMain.c
View File

@ -161,7 +161,7 @@ void cqStop(void *handle) {
return; return;
} }
SCqContext *pContext = handle; SCqContext *pContext = handle;
cInfo("vgId:%d, stop all CQs", pContext->vgId); cDebug("vgId:%d, stop all CQs", pContext->vgId);
if (pContext->dbConn == NULL || pContext->master == 0) return; if (pContext->dbConn == NULL || pContext->master == 0) return;
pthread_mutex_lock(&pContext->mutex); pthread_mutex_lock(&pContext->mutex);

2
src/dnode/src/dnodeEps.c
View File

@ -130,7 +130,7 @@ static void dnodePrintEps(SDnodeEps *eps) {
dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
for (int32_t i = 0; i < eps->dnodeNum; i++) { for (int32_t i = 0; i < eps->dnodeNum; i++) {
SDnodeEp *ep = &eps->dnodeEps[i]; SDnodeEp *ep = &eps->dnodeEps[i];
dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
} }
} }

2
src/dnode/src/dnodeMRead.c
View File

@ -124,8 +124,6 @@ void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
SMnodeMsg *pRead = mnodeCreateMsg(pMsg); SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
} }
rpcFreeCont(pMsg->pCont);
} }
static void dnodeFreeMReadMsg(SMnodeMsg *pRead) { static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {

2
src/dnode/src/dnodeMWrite.c
View File

@ -125,8 +125,6 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
} }
rpcFreeCont(pMsg->pCont);
} }
static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {

16
src/dnode/src/dnodeShell.c
View File

@ -70,8 +70,7 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
if (numOfThreads < 1) { if (numOfThreads < 1) {
numOfThreads = 1; numOfThreads = 1;
} }
@ -128,7 +127,20 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
} else {} } else {}
if ( dnodeProcessShellMsgFp[pMsg->msgType] ) { if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
SMsgVersion *pMsgVersion = pMsg->pCont;
if (taosCheckVersion(pMsgVersion->clientVersion, version, 3) != TSDB_CODE_SUCCESS) {
rpcMsg.code = TSDB_CODE_TSC_INVALID_VERSION;
rpcSendResponse(&rpcMsg);
rpcFreeCont(pMsg->pCont);
return; // todo change the error code
}
pMsg->pCont += sizeof(*pMsgVersion);
pMsg->contLen -= sizeof(*pMsgVersion);
(*dnodeProcessShellMsgFp[pMsg->msgType])(pMsg); (*dnodeProcessShellMsgFp[pMsg->msgType])(pMsg);
//pMsg->contLen += sizeof(*pMsgVersion);
rpcFreeCont(pMsg->pCont - sizeof(*pMsgVersion));
} else { } else {
dError("RPC %p, shell msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]); dError("RPC %p, shell msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]);
rpcMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; rpcMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;

19
src/dnode/src/dnodeSystem.c
View File

@ -16,12 +16,15 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "tgrant.h" #include "tgrant.h"
#include "tconfig.h"
#include "dnodeMain.h" #include "dnodeMain.h"
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
static tsem_t exitSem; static tsem_t exitSem;
int32_t main(int32_t argc, char *argv[]) { int32_t main(int32_t argc, char *argv[]) {
int dump_config = 0;
// Set global configuration file // Set global configuration file
for (int32_t i = 1; i < argc; ++i) { for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) { if (strcmp(argv[i], "-c") == 0) {
@ -35,6 +38,8 @@ int32_t main(int32_t argc, char *argv[]) {
printf("'-c' requires a parameter, default:%s\n", configDir); printf("'-c' requires a parameter, default:%s\n", configDir);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} else if (strcmp(argv[i], "-C") == 0) {
dump_config = 1;
} else if (strcmp(argv[i], "-V") == 0) { } else if (strcmp(argv[i], "-V") == 0) {
#ifdef _ACCT #ifdef _ACCT
char *versionStr = "enterprise"; char *versionStr = "enterprise";
@ -87,6 +92,20 @@ int32_t main(int32_t argc, char *argv[]) {
#endif #endif
} }
if (0 != dump_config) {
tscEmbedded = 1;
taosInitGlobalCfg();
taosReadGlobalLogCfg();
if (!taosReadGlobalCfg()) {
printf("TDengine read global config failed");
exit(EXIT_FAILURE);
}
taosDumpGlobalCfg();
exit(EXIT_SUCCESS);
}
if (tsem_init(&exitSem, 0, 0) != 0) { if (tsem_init(&exitSem, 0, 0) != 0) {
printf("failed to create exit semphore\n"); printf("failed to create exit semphore\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);

4
src/dnode/src/dnodeVMgmt.c
View File

@ -198,7 +198,7 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
SCreateMnodeMsg *pCfg = pMsg->pCont; SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) { if (pCfg->dnodeId != dnodeGetDnodeId()) {
dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); dDebug("dnode:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
} }
@ -207,7 +207,7 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
} }
dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); dDebug("dnode:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);

14
src/dnode/src/dnodeVRead.c
View File

@ -26,16 +26,20 @@ static SWorkerPool tsVQueryWP;
static SWorkerPool tsVFetchWP; static SWorkerPool tsVFetchWP;
int32_t dnodeInitVRead() { int32_t dnodeInitVRead() {
const int32_t maxFetchThreads = 4;
// calculate the available query thread
float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
tsVQueryWP.name = "vquery"; tsVQueryWP.name = "vquery";
tsVQueryWP.workerFp = dnodeProcessReadQueue; tsVQueryWP.workerFp = dnodeProcessReadQueue;
tsVQueryWP.min = tsNumOfCores; tsVQueryWP.min = (int32_t) threadsForQuery;
tsVQueryWP.max = tsNumOfCores/* * tsNumOfThreadsPerCore*/; tsVQueryWP.max = tsVQueryWP.min;
// if (tsVQueryWP.max <= tsVQueryWP.min * 2) tsVQueryWP.max = 2 * tsVQueryWP.min;
if (tWorkerInit(&tsVQueryWP) != 0) return -1; if (tWorkerInit(&tsVQueryWP) != 0) return -1;
tsVFetchWP.name = "vfetch"; tsVFetchWP.name = "vfetch";
tsVFetchWP.workerFp = dnodeProcessReadQueue; tsVFetchWP.workerFp = dnodeProcessReadQueue;
tsVFetchWP.min = MIN(4, tsNumOfCores); tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores);
tsVFetchWP.max = tsVFetchWP.min; tsVFetchWP.max = tsVFetchWP.min;
if (tWorkerInit(&tsVFetchWP) != 0) return -1; if (tWorkerInit(&tsVFetchWP) != 0) return -1;
@ -73,8 +77,6 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rpcRsp); rpcSendResponse(&rpcRsp);
} }
rpcFreeCont(pMsg->pCont);
} }
void *dnodeAllocVQueryQueue(void *pVnode) { void *dnodeAllocVQueryQueue(void *pVnode) {

1
src/dnode/src/dnodeVWrite.c
View File

@ -102,7 +102,6 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
} }
vnodeRelease(pVnode); vnodeRelease(pVnode);
rpcFreeCont(pRpcMsg->pCont);
} }
void *dnodeAllocVWriteQueue(void *pVnode) { void *dnodeAllocVWriteQueue(void *pVnode) {

5
src/inc/taoserror.h
View File

@ -206,9 +206,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing da
TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode") TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Database memory is full for commit failed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, 0, 0x050C, "Database memory is full for waiting commit")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Database write operation denied")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing")
// tsdb // tsdb

4
src/inc/taosmsg.h
View File

@ -198,6 +198,10 @@ typedef struct {
int32_t numOfVnodes; int32_t numOfVnodes;
} SMsgDesc; } SMsgDesc;
typedef struct SMsgVersion {
char clientVersion[TSDB_VERSION_LEN];
} SMsgVersion;
typedef struct SMsgHead { typedef struct SMsgHead {
int32_t contLen; int32_t contLen;
int32_t vgId; int32_t vgId;

1
src/kit/shell/inc/shell.h
View File

@ -45,6 +45,7 @@ typedef struct SShellArguments {
char* timezone; char* timezone;
bool is_raw_time; bool is_raw_time;
bool is_use_passwd; bool is_use_passwd;
bool dump_config;
char file[TSDB_FILENAME_LEN]; char file[TSDB_FILENAME_LEN];
char dir[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN];
int threadNum; int threadNum;

47
src/kit/shell/src/shellEngine.c
View File

@ -509,7 +509,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
static void shellPrintNChar(const char *str, int length, int width) { static void shellPrintNChar(const char *str, int length, int width) {
int pos = 0, cols = 0; wchar_t tail[3];
int pos = 0, cols = 0, totalCols = 0, tailLen = 0;
while (pos < length) { while (pos < length) {
wchar_t wc; wchar_t wc;
int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
@ -526,15 +528,44 @@ static void shellPrintNChar(const char *str, int length, int width) {
#else #else
int w = wcwidth(wc); int w = wcwidth(wc);
#endif #endif
if (w > 0) { if (w <= 0) {
if (width > 0 && cols + w > width) { continue;
}
if (width <= 0) {
printf("%lc", wc);
continue;
}
totalCols += w;
if (totalCols > width) {
break; break;
} }
if (totalCols <= (width - 3)) {
printf("%lc", wc); printf("%lc", wc);
cols += w; cols += w;
} else {
tail[tailLen] = wc;
tailLen++;
} }
} }
if (totalCols > width) {
// width could be 1 or 2, so printf("...") cannot be used
for (int i = 0; i < 3; i++) {
if (cols >= width) {
break;
}
putchar('.');
++cols;
}
} else {
for (int i = 0; i < tailLen; i++) {
printf("%lc", tail[i]);
}
cols = totalCols;
}
for (; cols < width; cols++) { for (; cols < width; cols++) {
putchar(' '); putchar(' ');
} }
@ -656,13 +687,21 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
return MAX(25, width); return MAX(25, width);
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (field->bytes > tsMaxBinaryDisplayWidth) { if (field->bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width); return MAX(tsMaxBinaryDisplayWidth, width);
} else { } else {
return MAX(field->bytes, width); return MAX(field->bytes, width);
} }
case TSDB_DATA_TYPE_NCHAR: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width);
} else {
return MAX(bytes, width);
}
}
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
if (args.is_raw_time) { if (args.is_raw_time) {
return MAX(14, width); return MAX(14, width);

4
src/kit/shell/src/shellLinux.c
View File

@ -39,6 +39,7 @@ static struct argp_option options[] = {
{"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."},
{"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."}, {"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."},
{"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."},
{"dump-config", 'C', 0, 0, "Dump configuration."},
{"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."},
{"raw-time", 'r', 0, 0, "Output time as uint64_t."}, {"raw-time", 'r', 0, 0, "Output time as uint64_t."},
{"file", 'f', "FILE", 0, "Script to run without enter the shell."}, {"file", 'f', "FILE", 0, "Script to run without enter the shell."},
@ -96,6 +97,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path); wordfree(&full_path);
break; break;
case 'C':
arguments->dump_config = true;
break;
case 's': case 's':
arguments->commands = arg; arguments->commands = arg;
break; break;

15
src/kit/shell/src/shellMain.c
View File

@ -15,6 +15,7 @@
#include "os.h" #include "os.h"
#include "shell.h" #include "shell.h"
#include "tconfig.h"
#include "tnettest.h" #include "tnettest.h"
pthread_t pid; pthread_t pid;
@ -58,6 +59,7 @@ SShellArguments args = {
.timezone = NULL, .timezone = NULL,
.is_raw_time = false, .is_raw_time = false,
.is_use_passwd = false, .is_use_passwd = false,
.dump_config = false,
.file = "\0", .file = "\0",
.dir = "\0", .dir = "\0",
.threadNum = 5, .threadNum = 5,
@ -78,6 +80,19 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args); shellParseArgument(argc, argv, &args);
if (args.dump_config) {
taosInitGlobalCfg();
taosReadGlobalLogCfg();
if (!taosReadGlobalCfg()) {
printf("TDengine read global config failed");
exit(EXIT_FAILURE);
}
taosDumpGlobalCfg();
exit(0);
}
if (args.netTestRole && args.netTestRole[0] != 0) { if (args.netTestRole && args.netTestRole[0] != 0) {
taos_init(); taos_init();
taosNetTest(args.netTestRole, args.host, args.port, args.pktLen); taosNetTest(args.netTestRole, args.host, args.port, args.pktLen);

4
src/kit/shell/src/shellWindows.c
View File

@ -35,6 +35,8 @@ void printHelp() {
printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server.");
printf("%s%s\n", indent, "-c"); printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory."); printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-C");
printf("%s%s%s\n", indent, indent, "Dump configuration.");
printf("%s%s\n", indent, "-s"); printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r"); printf("%s%s\n", indent, "-r");
@ -104,6 +106,8 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
fprintf(stderr, "Option -c requires an argument\n"); fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} else if (strcmp(argv[i], "-C") == 0) {
arguments->dump_config = true;
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
arguments->commands = argv[++i]; arguments->commands = argv[++i];

275
src/kit/taosdump/taosdump.c
View File

@ -14,6 +14,9 @@
*/ */
#include <iconv.h> #include <iconv.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include "os.h" #include "os.h"
#include "taos.h" #include "taos.h"
#include "taosdef.h" #include "taosdef.h"
@ -366,6 +369,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
static struct argp argp = {options, parse_opt, args_doc, doc}; static struct argp argp = {options, parse_opt, args_doc, doc};
static resultStatistics g_resultStatistics = {0}; static resultStatistics g_resultStatistics = {0};
static FILE *g_fpOfResult = NULL; static FILE *g_fpOfResult = NULL;
static int g_numOfCores = 1;
int taosDumpOut(struct arguments *arguments); int taosDumpOut(struct arguments *arguments);
int taosDumpIn(struct arguments *arguments); int taosDumpIn(struct arguments *arguments);
@ -378,7 +382,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName);
int taosCheckParam(struct arguments *arguments); int taosCheckParam(struct arguments *arguments);
void taosFreeDbInfos(); void taosFreeDbInfos();
static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName);
struct arguments tsArguments = { struct arguments tsArguments = {
// connection option // connection option
@ -540,6 +544,8 @@ int main(int argc, char *argv[]) {
} }
} }
g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN);
time_t tTime = time(NULL); time_t tTime = time(NULL);
struct tm tm = *localtime(&tTime); struct tm tm = *localtime(&tTime);
@ -692,65 +698,98 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu
sprintf(tmpCommand, "select tbname from %s", metric); sprintf(tmpCommand, "select tbname from %s", metric);
TAOS_RES *result = taos_query(taosCon, tmpCommand); TAOS_RES *res = taos_query(taosCon, tmpCommand);
int32_t code = taos_errno(result); int32_t code = taos_errno(res);
if (code != 0) { if (code != 0) {
fprintf(stderr, "failed to run command %s\n", tmpCommand); fprintf(stderr, "failed to run command %s\n", tmpCommand);
free(tmpCommand); free(tmpCommand);
taos_free_result(result); taos_free_result(res);
return -1; return -1;
} }
TAOS_FIELD *fields = taos_fetch_fields(result);
int32_t numOfTable = 0;
int32_t numOfThread = *totalNumOfThread;
char tmpFileName[TSDB_FILENAME_LEN + 1];
while ((row = taos_fetch_row(result)) != NULL) {
if (0 == numOfTable) {
memset(tmpFileName, 0, TSDB_FILENAME_LEN);
sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (fd == -1) {
fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
taos_free_result(result);
for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
(void)remove(tmpFileName);
}
free(tmpCommand); free(tmpCommand);
char tmpBuf[TSDB_FILENAME_LEN + 1];
memset(tmpBuf, 0, TSDB_FILENAME_LEN);
sprintf(tmpBuf, ".select-tbname.tmp");
fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (fd == -1) {
fprintf(stderr, "failed to open temp file: %s\n", tmpBuf);
taos_free_result(res);
return -1; return -1;
} }
numOfThread++; TAOS_FIELD *fields = taos_fetch_fields(res);
}
int32_t numOfTable = 0;
while ((row = taos_fetch_row(res)) != NULL) {
memset(&tableRecord, 0, sizeof(STableRecord)); memset(&tableRecord, 0, sizeof(STableRecord));
tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
taosWrite(fd, &tableRecord, sizeof(STableRecord)); taosWrite(fd, &tableRecord, sizeof(STableRecord));
numOfTable++; numOfTable++;
}
taos_free_result(res);
lseek(fd, 0, SEEK_SET);
if (numOfTable >= arguments->table_batch) { int maxThreads = arguments->thread_num;
numOfTable = 0; int tableOfPerFile ;
if (numOfTable <= arguments->thread_num) {
tableOfPerFile = 1;
maxThreads = numOfTable;
} else {
tableOfPerFile = numOfTable / arguments->thread_num;
if (0 != numOfTable % arguments->thread_num) {
tableOfPerFile += 1;
}
}
char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord));
if (NULL == tblBuf){
fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord));
close(fd); close(fd);
fd = -1; return -1;
} }
int32_t numOfThread = *totalNumOfThread;
int subFd = -1;
for (; numOfThread < maxThreads; numOfThread++) {
memset(tmpBuf, 0, TSDB_FILENAME_LEN);
sprintf(tmpBuf, ".tables.tmp.%d", numOfThread);
subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (subFd == -1) {
fprintf(stderr, "failed to open temp file: %s\n", tmpBuf);
for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
sprintf(tmpBuf, ".tables.tmp.%d", loopCnt);
(void)remove(tmpBuf);
} }
sprintf(tmpBuf, ".select-tbname.tmp");
(void)remove(tmpBuf);
close(fd);
return -1;
}
// read tableOfPerFile for fd, write to subFd
ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord));
if (readLen <= 0) {
close(subFd);
break;
}
taosWrite(subFd, tblBuf, readLen);
close(subFd);
}
sprintf(tmpBuf, ".select-tbname.tmp");
(void)remove(tmpBuf);
if (fd >= 0) { if (fd >= 0) {
close(fd); close(fd);
fd = -1; fd = -1;
} }
taos_free_result(result);
*totalNumOfThread = numOfThread; *totalNumOfThread = numOfThread;
free(tmpCommand);
return 0; return 0;
} }
@ -946,7 +985,7 @@ int taosDumpOut(struct arguments *arguments) {
} }
// start multi threads to dumpout // start multi threads to dumpout
taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name); taosStartDumpOutWorkThreads(taos, arguments, totalNumOfThread, dbInfos[0]->name);
char tmpFileName[TSDB_FILENAME_LEN + 1]; char tmpFileName[TSDB_FILENAME_LEN + 1];
_clean_tmp_file: _clean_tmp_file:
@ -1181,34 +1220,34 @@ void* taosDumpOutWorkThreadFp(void *arg)
STableRecord tableRecord; STableRecord tableRecord;
int fd; int fd;
char tmpFileName[TSDB_FILENAME_LEN*4] = {0}; char tmpBuf[TSDB_FILENAME_LEN*4] = {0};
sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex); sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex);
fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (fd == -1) { if (fd == -1) {
fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName); fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpBuf);
return NULL; return NULL;
} }
FILE *fp = NULL; FILE *fp = NULL;
memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128); memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128);
if (tsArguments.outpath[0] != 0) { if (tsArguments.outpath[0] != 0) {
sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); sprintf(tmpBuf, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex);
} else { } else {
sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex);
} }
fp = fopen(tmpFileName, "w"); fp = fopen(tmpBuf, "w");
if (fp == NULL) { if (fp == NULL) {
fprintf(stderr, "failed to open file %s\n", tmpFileName); fprintf(stderr, "failed to open file %s\n", tmpBuf);
close(fd); close(fd);
return NULL; return NULL;
} }
memset(tmpFileName, 0, TSDB_FILENAME_LEN); memset(tmpBuf, 0, TSDB_FILENAME_LEN);
sprintf(tmpFileName, "use %s", pThread->dbName); sprintf(tmpBuf, "use %s", pThread->dbName);
TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName); TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf);
int32_t code = taos_errno(tmpResult); int32_t code = taos_errno(tmpResult);
if (code != 0) { if (code != 0) {
fprintf(stderr, "invalid database %s\n", pThread->dbName); fprintf(stderr, "invalid database %s\n", pThread->dbName);
@ -1218,6 +1257,9 @@ void* taosDumpOutWorkThreadFp(void *arg)
return NULL; return NULL;
} }
int fileNameIndex = 1;
int tablesInOneFile = 0;
int64_t lastRowsPrint = 5000000;
fprintf(fp, "USE %s;\n\n", pThread->dbName); fprintf(fp, "USE %s;\n\n", pThread->dbName);
while (1) { while (1) {
ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
@ -1228,6 +1270,33 @@ void* taosDumpOutWorkThreadFp(void *arg)
// TODO: sum table count and table rows by self // TODO: sum table count and table rows by self
pThread->tablesOfDumpOut++; pThread->tablesOfDumpOut++;
pThread->rowsOfDumpOut += ret; pThread->rowsOfDumpOut += ret;
if (pThread->rowsOfDumpOut >= lastRowsPrint) {
printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName);
lastRowsPrint += 5000000;
}
tablesInOneFile++;
if (tablesInOneFile >= tsArguments.table_batch) {
fclose(fp);
tablesInOneFile = 0;
memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128);
if (tsArguments.outpath[0] != 0) {
sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex);
} else {
sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex);
}
fileNameIndex++;
fp = fopen(tmpBuf, "w");
if (fp == NULL) {
fprintf(stderr, "failed to open file %s\n", tmpBuf);
close(fd);
taos_free_result(tmpResult);
return NULL;
}
}
} }
} }
@ -1238,7 +1307,7 @@ void* taosDumpOutWorkThreadFp(void *arg)
return NULL; return NULL;
} }
static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName) static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName)
{ {
pthread_attr_t thattr; pthread_attr_t thattr;
SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj));
@ -1249,12 +1318,7 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh
pThread->threadIndex = t; pThread->threadIndex = t;
pThread->totalThreads = numOfThread; pThread->totalThreads = numOfThread;
tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN);
pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); pThread->taosCon = taosCon;
if (pThread->taosCon == NULL) {
fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL));
exit(0);
}
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
@ -1273,7 +1337,6 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh
int64_t totalRowsOfDumpOut = 0; int64_t totalRowsOfDumpOut = 0;
int64_t totalChildTblsOfDumpOut = 0; int64_t totalChildTblsOfDumpOut = 0;
for (int32_t t = 0; t < numOfThread; ++t) { for (int32_t t = 0; t < numOfThread; ++t) {
taos_close(threadObj[t].taosCon);
totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut;
totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut;
} }
@ -1398,29 +1461,20 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
return -1; return -1;
} }
TAOS_FIELD *fields = taos_fetch_fields(res); char tmpBuf[TSDB_FILENAME_LEN + 1];
memset(tmpBuf, 0, TSDB_FILENAME_LEN);
int32_t numOfTable = 0; sprintf(tmpBuf, ".show-tables.tmp");
int32_t numOfThread = 0; fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
char tmpFileName[TSDB_FILENAME_LEN + 1];
while ((row = taos_fetch_row(res)) != NULL) {
if (0 == numOfTable) {
memset(tmpFileName, 0, TSDB_FILENAME_LEN);
sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (fd == -1) { if (fd == -1) {
fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); fprintf(stderr, "failed to open temp file: %s\n", tmpBuf);
taos_free_result(res); taos_free_result(res);
for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
(void)remove(tmpFileName);
}
return -1; return -1;
} }
numOfThread++; TAOS_FIELD *fields = taos_fetch_fields(res);
}
int32_t numOfTable = 0;
while ((row = taos_fetch_row(res)) != NULL) {
memset(&tableRecord, 0, sizeof(STableRecord)); memset(&tableRecord, 0, sizeof(STableRecord));
tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
@ -1428,13 +1482,59 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
taosWrite(fd, &tableRecord, sizeof(STableRecord)); taosWrite(fd, &tableRecord, sizeof(STableRecord));
numOfTable++; numOfTable++;
}
taos_free_result(res);
lseek(fd, 0, SEEK_SET);
if (numOfTable >= arguments->table_batch) { int maxThreads = tsArguments.thread_num;
numOfTable = 0; int tableOfPerFile ;
if (numOfTable <= tsArguments.thread_num) {
tableOfPerFile = 1;
maxThreads = numOfTable;
} else {
tableOfPerFile = numOfTable / tsArguments.thread_num;
if (0 != numOfTable % tsArguments.thread_num) {
tableOfPerFile += 1;
}
}
char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord));
if (NULL == tblBuf){
fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord));
close(fd); close(fd);
fd = -1; return -1;
} }
int32_t numOfThread = 0;
int subFd = -1;
for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) {
memset(tmpBuf, 0, TSDB_FILENAME_LEN);
sprintf(tmpBuf, ".tables.tmp.%d", numOfThread);
subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
if (subFd == -1) {
fprintf(stderr, "failed to open temp file: %s\n", tmpBuf);
for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
sprintf(tmpBuf, ".tables.tmp.%d", loopCnt);
(void)remove(tmpBuf);
} }
sprintf(tmpBuf, ".show-tables.tmp");
(void)remove(tmpBuf);
close(fd);
return -1;
}
// read tableOfPerFile for fd, write to subFd
ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord));
if (readLen <= 0) {
close(subFd);
break;
}
taosWrite(subFd, tblBuf, readLen);
close(subFd);
}
sprintf(tmpBuf, ".show-tables.tmp");
(void)remove(tmpBuf);
if (fd >= 0) { if (fd >= 0) {
close(fd); close(fd);
@ -1444,10 +1544,10 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
taos_free_result(res); taos_free_result(res);
// start multi threads to dumpout // start multi threads to dumpout
taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name); taosStartDumpOutWorkThreads(taosCon, arguments, numOfThread, dbInfo->name);
for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); sprintf(tmpBuf, ".tables.tmp.%d", loopCnt);
(void)remove(tmpFileName); (void)remove(tmpBuf);
} }
return 0; return 0;
@ -1552,7 +1652,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
} }
int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) { int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) {
/* char temp[MAX_COMMAND_SIZE] = "\0"; */ int64_t lastRowsPrint = 5000000;
int64_t totalRows = 0; int64_t totalRows = 0;
int count = 0; int count = 0;
char *pstr = NULL; char *pstr = NULL;
@ -1684,6 +1784,11 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
count++; count++;
fprintf(fp, "%s", tmpBuffer); fprintf(fp, "%s", tmpBuffer);
if (totalRows >= lastRowsPrint) {
printf(" %"PRId64 " rows already be dumpout from %s.%s\n", totalRows, dbName, tbname);
lastRowsPrint += 5000000;
}
total_sqlstr_len += curr_sqlstr_len; total_sqlstr_len += curr_sqlstr_len;
if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) {
@ -2048,6 +2153,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c
return -1; return -1;
} }
int lastRowsPrint = 5000000;
int lineNo = 0; int lineNo = 0;
while ((read_len = getline(&line, &line_len, fp)) != -1) { while ((read_len = getline(&line, &line_len, fp)) != -1) {
++lineNo; ++lineNo;
@ -2075,6 +2181,11 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c
memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN);
cmd_len = 0; cmd_len = 0;
if (lineNo >= lastRowsPrint) {
printf(" %d lines already be executed from file %s\n", lineNo, fileName);
lastRowsPrint += 5000000;
}
} }
tfree(cmd); tfree(cmd);
@ -2101,7 +2212,7 @@ void* taosDumpInWorkThreadFp(void *arg)
return NULL; return NULL;
} }
static void taosStartDumpInWorkThreads(struct arguments *args) static void taosStartDumpInWorkThreads(void* taosCon, struct arguments *args)
{ {
pthread_attr_t thattr; pthread_attr_t thattr;
SThreadParaObj *pThread; SThreadParaObj *pThread;
@ -2116,11 +2227,7 @@ static void taosStartDumpInWorkThreads(struct arguments *args)
pThread = threadObj + t; pThread = threadObj + t;
pThread->threadIndex = t; pThread->threadIndex = t;
pThread->totalThreads = totalThreads; pThread->totalThreads = totalThreads;
pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); pThread->taosCon = taosCon;
if (pThread->taosCon == NULL) {
fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL));
exit(0);
}
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
@ -2169,7 +2276,7 @@ int taosDumpIn(struct arguments *arguments) {
taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile);
} }
taosStartDumpInWorkThreads(arguments); taosStartDumpInWorkThreads(taos, arguments);
taos_close(taos); taos_close(taos);
taosFreeSQLFiles(); taosFreeSQLFiles();

4
src/mnode/inc/mnodeMnode.h
View File

@ -43,8 +43,8 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode);
void mnodeDecMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode);
char * mnodeGetMnodeRoleStr(); char * mnodeGetMnodeRoleStr();
void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect);
void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect);
char* mnodeGetMnodeMasterEp(); char* mnodeGetMnodeMasterEp();
void mnodeGetMnodeInfos(void *mnodes); void mnodeGetMnodeInfos(void *mnodes);

2
src/mnode/src/mnodeDnode.c
View File

@ -303,7 +303,7 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
int32_t code = sdbUpdateRow(&row); int32_t code = sdbUpdateRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("dnodeId:%d, failed update", pDnode->dnodeId); mError("dnode:%d, failed update", pDnode->dnodeId);
} }
} }

8
src/mnode/src/mnodeMnode.c
View File

@ -273,14 +273,14 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) {
mnodeMnodeUnLock(); mnodeMnodeUnLock();
} }
void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect) {
mnodeMnodeRdLock(); mnodeMnodeRdLock();
*epSet = tsMEpForPeer; *epSet = tsMEpForPeer;
mnodeMnodeUnLock(); mnodeMnodeUnLock();
mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse); mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse);
for (int32_t i = 0; i < epSet->numOfEps; ++i) { for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps; epSet->inUse = (i + 1) % epSet->numOfEps;
mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else { } else {
@ -289,14 +289,14 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) {
} }
} }
void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect) {
mnodeMnodeRdLock(); mnodeMnodeRdLock();
*epSet = tsMEpForShell; *epSet = tsMEpForShell;
mnodeMnodeUnLock(); mnodeMnodeUnLock();
mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse); mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse);
for (int32_t i = 0; i < epSet->numOfEps; ++i) { for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
epSet->inUse = (i + 1) % epSet->numOfEps; epSet->inUse = (i + 1) % epSet->numOfEps;
mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else { } else {

2
src/mnode/src/mnodePeer.c
View File

@ -54,7 +54,7 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeEpSetForPeer(epSet); mnodeGetMnodeEpSetForPeer(epSet, true);
rpcRsp->rsp = epSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet); rpcRsp->len = sizeof(SRpcEpSet);

21
src/mnode/src/mnodeProfile.c
View File

@ -282,27 +282,34 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
// not thread safe, need optimized // not thread safe, need optimized
int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) {
pConn->numOfQueries = htonl(pHBMsg->numOfQueries); pConn->numOfQueries = 0;
if (pConn->numOfQueries > 0) { pConn->numOfStreams = 0;
int32_t numOfQueries = htonl(pHBMsg->numOfQueries);
int32_t numOfStreams = htonl(pHBMsg->numOfStreams);
if (numOfQueries > 0) {
if (pConn->pQueries == NULL) { if (pConn->pQueries == NULL) {
pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE);
} }
int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfQueries) * sizeof(SQueryDesc); pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries);
int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc);
if (saveSize > 0 && pConn->pQueries != NULL) { if (saveSize > 0 && pConn->pQueries != NULL) {
memcpy(pConn->pQueries, pHBMsg->pData, saveSize); memcpy(pConn->pQueries, pHBMsg->pData, saveSize);
} }
} }
pConn->numOfStreams = htonl(pHBMsg->numOfStreams); if (numOfStreams > 0) {
if (pConn->numOfStreams > 0) {
if (pConn->pStreams == NULL) { if (pConn->pStreams == NULL) {
pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE);
} }
int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfStreams) * sizeof(SStreamDesc); pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams);
int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc);
if (saveSize > 0 && pConn->pStreams != NULL) { if (saveSize > 0 && pConn->pStreams != NULL) {
memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), saveSize); memcpy(pConn->pStreams, pHBMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize);
} }
} }

2
src/mnode/src/mnodeRead.c
View File

@ -50,7 +50,7 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeEpSetForShell(epSet); mnodeGetMnodeEpSetForShell(epSet, true);
rpcRsp->rsp = epSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet); rpcRsp->len = sizeof(SRpcEpSet);

4
src/mnode/src/mnodeShow.c
View File

@ -282,7 +282,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
mnodeGetMnodeEpSetForShell(&pRsp->epSet); mnodeGetMnodeEpSetForShell(&pRsp->epSet, false);
pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.rsp = pRsp;
pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); pMsg->rpcRsp.len = sizeof(SHeartBeatRsp);
@ -349,7 +349,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->writeAuth = pUser->writeAuth;
pConnectRsp->superAuth = pUser->superAuth; pConnectRsp->superAuth = pUser->superAuth;
mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet); mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false);
connect_over: connect_over:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {

3
src/mnode/src/mnodeVgroup.c
View File

@ -315,7 +315,8 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
if (pVgid->pDnode == pDnode) { if (pVgid->pDnode == pDnode) {
mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d:%s", pDnode->dnodeId, pVgroup->vgId, pVgid->role, syncRole[pVgid->role]); mTrace("dnode:%d, receive status from dnode, vgId:%d status:%s last:%s", pDnode->dnodeId, pVgroup->vgId,
syncRole[pVload->role], syncRole[pVgid->role]);
pVgid->role = pVload->role; pVgid->role = pVload->role;
if (pVload->role == TAOS_SYNC_ROLE_MASTER) { if (pVload->role == TAOS_SYNC_ROLE_MASTER) {
pVgroup->inUse = i; pVgroup->inUse = i;

2
src/mnode/src/mnodeWrite.c
View File

@ -50,7 +50,7 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
if (!sdbIsMaster()) { if (!sdbIsMaster()) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeEpSetForShell(epSet); mnodeGetMnodeEpSetForShell(epSet, true);
rpcRsp->rsp = epSet; rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet); rpcRsp->len = sizeof(SRpcEpSet);

3
src/query/inc/tsqlfunction.h
View File

@ -250,10 +250,9 @@ enum {
}; };
typedef struct STwaInfo { typedef struct STwaInfo {
TSKEY lastKey;
int8_t hasResult; // flag to denote has value int8_t hasResult; // flag to denote has value
double dOutput; double dOutput;
double lastValue; SPoint1 p;
STimeWindow win; STimeWindow win;
} STwaInfo; } STwaInfo;

108
src/query/src/qExecutor.c
View File

@ -703,39 +703,11 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
return forwardStep; return forwardStep;
} }
/** static int32_t updateResultRowCurrentIndex(SResultRowInfo* pWindowResInfo, TSKEY lastKey, bool ascQuery) {
* NOTE: the query status only set for the first scan of master scan.
*/
static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) {
SQuery *pQuery = pRuntimeEnv->pQuery;
if (pRuntimeEnv->scanFlag != MASTER_SCAN) {
return pWindowResInfo->size;
}
// for group by normal column query, close time window and return.
if (!QUERY_IS_INTERVAL_QUERY(pQuery)) {
closeAllTimeWindow(pWindowResInfo);
return pWindowResInfo->size;
}
// no qualified results exist, abort check
int32_t numOfClosed = 0;
if (pWindowResInfo->size == 0) {
return pWindowResInfo->size;
}
// query completed
if ((lastKey >= pQuery->current->win.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(lastKey <= pQuery->current->win.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
closeAllTimeWindow(pWindowResInfo);
pWindowResInfo->curIndex = pWindowResInfo->size - 1;
setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
} else { // set the current index to be the last unclosed window
int32_t i = 0; int32_t i = 0;
int64_t skey = TSKEY_INITIAL_VAL; int64_t skey = TSKEY_INITIAL_VAL;
int32_t numOfClosed = 0;
for (i = 0; i < pWindowResInfo->size; ++i) { for (i = 0; i < pWindowResInfo->size; ++i) {
SResultRow *pResult = pWindowResInfo->pResult[i]; SResultRow *pResult = pWindowResInfo->pResult[i];
if (pResult->closed) { if (pResult->closed) {
@ -744,8 +716,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
} }
TSKEY ekey = pResult->win.ekey; TSKEY ekey = pResult->win.ekey;
if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || if ((ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) {
(pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) {
closeTimeWindow(pWindowResInfo, i); closeTimeWindow(pWindowResInfo, i);
} else { } else {
skey = pResult->win.skey; skey = pResult->win.skey;
@ -759,9 +730,33 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
pWindowResInfo->curIndex = pWindowResInfo->size - 1; pWindowResInfo->curIndex = pWindowResInfo->size - 1;
} else { } else {
pWindowResInfo->curIndex = i; pWindowResInfo->curIndex = i;
pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey;
} }
pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; return numOfClosed;
}
/**
* NOTE: the query status only set for the first scan of master scan.
*/
static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) {
SQuery *pQuery = pRuntimeEnv->pQuery;
if (pRuntimeEnv->scanFlag != MASTER_SCAN || pWindowResInfo->size == 0) {
return pWindowResInfo->size;
}
// no qualified results exist, abort check
int32_t numOfClosed = 0;
bool ascQuery = QUERY_IS_ASC_QUERY(pQuery);
// query completed
if ((lastKey >= pQuery->current->win.ekey && ascQuery) || (lastKey <= pQuery->current->win.ekey && (!ascQuery))) {
closeAllTimeWindow(pWindowResInfo);
pWindowResInfo->curIndex = pWindowResInfo->size - 1;
setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
} else { // set the current index to be the last unclosed window
numOfClosed = updateResultRowCurrentIndex(pWindowResInfo, lastKey, ascQuery);
// the number of completed slots are larger than the threshold, return current generated results to client. // the number of completed slots are larger than the threshold, return current generated results to client.
if (numOfClosed > pQuery->rec.threshold) { if (numOfClosed > pQuery->rec.threshold) {
@ -1050,24 +1045,6 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in
} }
} }
//static double getTSWindowInterpoVal(SColumnInfoData* pColInfo, int16_t srcColIndex, int16_t rowIndex, TSKEY key, char** prevRow, TSKEY* tsCols, int32_t step) {
// TSKEY start = tsCols[rowIndex];
// TSKEY prevTs = (rowIndex == 0)? *(TSKEY *) prevRow[0] : tsCols[rowIndex - step];
//
// double v1 = 0, v2 = 0, v = 0;
// char *prevVal = (rowIndex == 0)? prevRow[srcColIndex] : ((char*)pColInfo->pData) + (rowIndex - step) * pColInfo->info.bytes;
//
// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal);
// GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + rowIndex * pColInfo->info.bytes);
//
// SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
// SPoint point2 = (SPoint){.key = start, .val = &v2};
// SPoint point = (SPoint){.key = key, .val = &v};
// taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point);
//
// return v;
//}
// window start key interpolation // window start key interpolation
static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) {
SQuery* pQuery = pRuntimeEnv->pQuery; SQuery* pQuery = pRuntimeEnv->pQuery;
@ -1238,6 +1215,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
if (interp) { if (interp) {
setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
} }
} else {
setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
} }
done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
@ -1249,6 +1228,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
if (interp) { if (interp) {
setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
} }
} else {
setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
} }
} }
@ -1289,6 +1270,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
if (interp) { if (interp) {
setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
} }
} else {
setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
} }
done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
@ -1299,6 +1282,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
if (interp) { if (interp) {
setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
} }
} else {
setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
} }
} }
@ -1802,9 +1787,12 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
// interval query with limit applied // interval query with limit applied
int32_t numOfRes = 0; int32_t numOfRes = 0;
if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
} else { } else if (pRuntimeEnv->groupbyNormalCol) {
closeAllTimeWindow(pWindowResInfo);
numOfRes = pWindowResInfo->size;
} else { // projection query
numOfRes = (int32_t)getNumOfResult(pRuntimeEnv); numOfRes = (int32_t)getNumOfResult(pRuntimeEnv);
// update the number of output result // update the number of output result
@ -4487,6 +4475,18 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc
} else { } else {
blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);
} }
if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
bool ascQuery = QUERY_IS_ASC_QUERY(pQuery);
// TODO refactor
if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) {
closeAllTimeWindow(pWindowResInfo);
pWindowResInfo->curIndex = pWindowResInfo->size - 1;
} else {
updateResultRowCurrentIndex(pWindowResInfo, pTableQueryInfo->lastKey, ascQuery);
}
}
} }
bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) {
@ -7635,7 +7635,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (tsHalfCoresForQuery) { if (tsRetrieveBlockingModel) {
pQInfo->rspContext = pRspContext; pQInfo->rspContext = pRspContext;
tsem_wait(&pQInfo->ready); tsem_wait(&pQInfo->ready);
*buildRes = true; *buildRes = true;

73
src/query/src/qFilterfunc.c
View File

@ -21,6 +21,12 @@
#include "tcompare.h" #include "tcompare.h"
#include "tsqlfunction.h" #include "tsqlfunction.h"
#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (4 * FLT_EPSILON))
#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y)))
#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y)))
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); return (*(int8_t *)minval < pFilter->filterInfo.upperBndi);
} }
@ -38,35 +44,35 @@ bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
} }
bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval < pFilter->filterInfo.upperBndd); return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd);
} }
bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)minval < pFilter->filterInfo.upperBndd); return *(double *)minval < pFilter->filterInfo.upperBndd;
} }
////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////
bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi);
} }
bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi);
} }
bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi);
} }
bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi);
} }
bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval > pFilter->filterInfo.lowerBndd); return FLT_GREATER(*(float*)maxval, pFilter->filterInfo.lowerBndd);
} }
bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool larger_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval > pFilter->filterInfo.lowerBndd); return (*(double *)maxval > pFilter->filterInfo.lowerBndd);
} }
///////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////
@ -88,10 +94,14 @@ bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
} }
bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd); return FLT_LESSEQUAL(*(float*)minval, pFilter->filterInfo.upperBndd);
} }
bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if ((fabs(*(double*)minval) - pFilter->filterInfo.upperBndd) <= 2 * DBL_EPSILON) {
return true;
}
return (*(double *)minval <= pFilter->filterInfo.upperBndd); return (*(double *)minval <= pFilter->filterInfo.upperBndd);
} }
@ -113,11 +123,15 @@ bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
} }
bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); return FLT_GREATEREQUAL(*(float*)maxval, pFilter->filterInfo.lowerBndd);
} }
bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); if (fabs(*(double *)maxval - pFilter->filterInfo.lowerBndd) <= 2 * DBL_EPSILON) {
return true;
}
return (*(double *)maxval - pFilter->filterInfo.lowerBndd > (2 * DBL_EPSILON));
} }
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
@ -162,10 +176,12 @@ bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
} }
} }
// user specified input filter value and the original saved float value may needs to
// increase the tolerance to obtain the correct result.
bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) { if (*(float *)minval == *(float *)maxval) {
return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); return FLT_EQUAL(*(float*)minval, pFilter->filterInfo.lowerBndd);
} else { /* range filter */ } else { // range filter
assert(*(float *)minval < *(float *)maxval); assert(*(float *)minval < *(float *)maxval);
return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd;
} }
@ -173,10 +189,9 @@ bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(double *)minval == *(double *)maxval) { if (*(double *)minval == *(double *)maxval) {
return (*(double *)minval == pFilter->filterInfo.lowerBndd); return (fabs(*(double *)minval - pFilter->filterInfo.lowerBndd) <= 2 * DBL_EPSILON);
} else { /* range filter */ } else { // range filter
assert(*(double *)minval < *(double *)maxval); assert(*(double *)minval < *(double *)maxval);
return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi;
} }
} }
@ -255,7 +270,7 @@ bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) {
bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) {
if (*(float *)minval == *(float *)maxval) { if (*(float *)minval == *(float *)maxval) {
return (*(float *)minval != pFilter->filterInfo.lowerBndd); return !FLT_EQUAL(*(float *)minval, pFilter->filterInfo.lowerBndd);
} }
return true; return true;
@ -364,7 +379,8 @@ bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval)
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); return FLT_LESSEQUAL(*(float *)minval, pFilter->filterInfo.upperBndd) &&
FLT_GREATEREQUAL(*(float *)maxval, pFilter->filterInfo.lowerBndd);
} }
bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) {
@ -376,7 +392,8 @@ bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) {
} }
bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); return FLT_GREATER(*(float *)maxval, pFilter->filterInfo.lowerBndd) &&
FLT_LESSEQUAL(*(float *)minval, pFilter->filterInfo.upperBndd);
} }
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
@ -400,7 +417,7 @@ bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) {
bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_i8, less_i8,
large_i8, larger_i8,
equal_i8, equal_i8,
lessEqual_i8, lessEqual_i8,
largeEqual_i8, largeEqual_i8,
@ -413,7 +430,7 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_i16, less_i16,
large_i16, larger_i16,
equal_i16, equal_i16,
lessEqual_i16, lessEqual_i16,
largeEqual_i16, largeEqual_i16,
@ -426,7 +443,7 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_i32, less_i32,
large_i32, larger_i32,
equal_i32, equal_i32,
lessEqual_i32, lessEqual_i32,
largeEqual_i32, largeEqual_i32,
@ -439,7 +456,7 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_i64, less_i64,
large_i64, larger_i64,
equal_i64, equal_i64,
lessEqual_i64, lessEqual_i64,
largeEqual_i64, largeEqual_i64,
@ -452,7 +469,7 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_ds, less_ds,
large_ds, larger_ds,
equal_ds, equal_ds,
lessEqual_ds, lessEqual_ds,
largeEqual_ds, largeEqual_ds,
@ -465,7 +482,7 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
NULL, NULL,
less_dd, less_dd,
large_dd, larger_dd,
equal_dd, equal_dd,
lessEqual_dd, lessEqual_dd,
largeEqual_dd, largeEqual_dd,
@ -551,7 +568,7 @@ bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *max
__filter_func_t* getRangeFilterFuncArray(int32_t type) { __filter_func_t* getRangeFilterFuncArray(int32_t type) {
switch(type) { switch(type) {
case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16;
case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32;
@ -565,7 +582,7 @@ __filter_func_t* getRangeFilterFuncArray(int32_t type) {
__filter_func_t* getValueFilterFuncArray(int32_t type) { __filter_func_t* getValueFilterFuncArray(int32_t type) {
switch(type) { switch(type) {
case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8;
case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16;
case TSDB_DATA_TYPE_INT: return filterFunc_i32; case TSDB_DATA_TYPE_INT: return filterFunc_i32;

8
src/rpc/src/rpcMain.c
View File

@ -631,6 +631,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
// if there is an outgoing message, free it // if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) { if (pConn->outType && pConn->pReqMsg) {
SRpcReqContext *pContext = pConn->pContext; SRpcReqContext *pContext = pConn->pContext;
if (pContext) {
if (pContext->pRsp) { if (pContext->pRsp) {
// for synchronous API, post semaphore to unblock app // for synchronous API, post semaphore to unblock app
pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR;
@ -640,6 +641,9 @@ static void rpcReleaseConn(SRpcConn *pConn) {
} }
pContext->pConn = NULL; pContext->pConn = NULL;
taosRemoveRef(tsRpcRefId, pContext->rid); taosRemoveRef(tsRpcRefId, pContext->rid);
} else {
assert(0);
}
} }
} }
@ -1083,7 +1087,11 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) {
rpcCloseConn(pConn); rpcCloseConn(pConn);
} }
if (pHead->msgType + 1 > 1 && pHead->msgType+1 < TSDB_MSG_TYPE_MAX) {
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
} else {
tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], code);
}
} }
} else { // msg is passed to app only parsing is ok } else { // msg is passed to app only parsing is ok
rpcProcessIncomingMsg(pConn, pHead, pContext); rpcProcessIncomingMsg(pConn, pHead, pContext);

9
src/rpc/src/rpcTcp.c
View File

@ -242,7 +242,14 @@ static void *taosAcceptTcpConnection(void *arg) {
taosKeepTcpAlive(connFd); taosKeepTcpAlive(connFd);
struct timeval to={1, 0}; struct timeval to={1, 0};
taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
if (ret != 0) {
taosCloseSocket(connFd);
tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno),
taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
continue;
}
// pick up the thread to handle this connection // pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj[threadId]; pThreadObj = pServerObj->pThreadObj[threadId];

3
src/rpc/test/rclient.c
View File

@ -188,7 +188,8 @@ int main(int argc, char *argv[]) {
tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
getchar(); int ch = getchar();
UNUSED(ch);
taosCloseLog(); taosCloseLog();

4
src/sync/inc/syncInt.h
View File

@ -62,12 +62,15 @@ typedef struct {
typedef struct { typedef struct {
SSyncHead syncHead; SSyncHead syncHead;
uint16_t port; uint16_t port;
uint16_t tranId;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
int32_t sourceId; // only for arbitrator int32_t sourceId; // only for arbitrator
} SFirstPkt; } SFirstPkt;
typedef struct { typedef struct {
int8_t sync; int8_t sync;
int8_t reserved;
uint16_t tranId;
} SFirstPktRsp; } SFirstPktRsp;
typedef struct { typedef struct {
@ -187,6 +190,7 @@ void syncRestartConnection(SSyncPeer *pPeer);
void syncBroadcastStatus(SSyncNode *pNode); void syncBroadcastStatus(SSyncNode *pNode);
void syncAddPeerRef(SSyncPeer *pPeer); void syncAddPeerRef(SSyncPeer *pPeer);
int32_t syncDecPeerRef(SSyncPeer *pPeer); int32_t syncDecPeerRef(SSyncPeer *pPeer);
uint16_t syncGenTranId();
#ifdef __cplusplus #ifdef __cplusplus
} }

27
src/sync/src/syncMain.c
View File

@ -396,9 +396,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
pFwdRsp->code = code; pFwdRsp->code = code;
int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen); if (taosWriteMsg(pPeer->peerFd, msg, msgLen) == msgLen) {
if (retLen == msgLen) {
sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version);
} else { } else {
sDebug("%s, failed to send forward ack, restart", pPeer->id); sDebug("%s, failed to send forward ack, restart", pPeer->id);
@ -795,7 +793,7 @@ void syncRestartConnection(SSyncPeer *pPeer) {
static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode; SSyncNode *pNode = pPeer->pSyncNode;
sDebug("%s, sync-req is received", pPeer->id); sInfo("%s, sync-req is received", pPeer->id);
if (pPeer->ip == 0) return; if (pPeer->ip == 0) return;
@ -873,6 +871,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ; firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ;
firstPkt.syncHead.vgId = pNode->vgId; firstPkt.syncHead.vgId = pNode->vgId;
firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead); firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
firstPkt.tranId = syncGenTranId();
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort; firstPkt.port = tsSyncPort;
taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
@ -880,8 +879,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
sError("%s, failed to send sync-req to peer", pPeer->id); sError("%s, failed to send sync-req to peer", pPeer->id);
} else { } else {
nodeSStatus = TAOS_SYNC_STATUS_START; sInfo("%s, sync-req is sent to peer, tranId:%u, sstatus:%s", pPeer->id, firstPkt.tranId, syncStatus[nodeSStatus]);
sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
} }
} }
@ -1018,8 +1016,7 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type
pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
} }
int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (taosWriteMsg(pPeer->peerFd, msg, statusMsgLen) == statusMsgLen) {
if (retLen == statusMsgLen) {
sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d",
pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role],
syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId,
@ -1053,10 +1050,11 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
firstPkt.syncHead.type = TAOS_SMSG_STATUS; firstPkt.syncHead.type = TAOS_SMSG_STATUS;
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort; firstPkt.port = tsSyncPort;
firstPkt.tranId = syncGenTranId();
firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId
if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd); sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, firstPkt.tranId);
pPeer->peerFd = connFd; pPeer->peerFd = connFd;
pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
@ -1093,7 +1091,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
pthread_attr_destroy(&thattr); pthread_attr_destroy(&thattr);
if (ret < 0) { if (ret < 0) {
sError("%s, failed to create sync thread", pPeer->id); SSyncNode *pNode = pPeer->pSyncNode;
nodeSStatus = TAOS_SYNC_STATUS_INIT;
sError("%s, failed to create sync thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
taosClose(pPeer->syncFd); taosClose(pPeer->syncFd);
syncDecPeerRef(pPeer); syncDecPeerRef(pPeer);
} else { } else {
@ -1123,6 +1123,8 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
return; return;
} }
sDebug("vgId:%d, firstPkt is received, tranId:%u", vgId, firstPkt.tranId);
SSyncNode *pNode = *ppNode; SSyncNode *pNode = *ppNode;
pthread_mutex_lock(&pNode->mutex); pthread_mutex_lock(&pNode->mutex);
@ -1141,6 +1143,9 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
// first packet tells what kind of link // first packet tells what kind of link
if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) { if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) {
pPeer->syncFd = connFd; pPeer->syncFd = connFd;
nodeSStatus = TAOS_SYNC_STATUS_START;
sInfo("%s, sync-data pkt from master is received, tranId:%u, set sstatus:%s", pPeer->id, firstPkt.tranId,
syncStatus[nodeSStatus]);
syncCreateRestoreDataThread(pPeer); syncCreateRestoreDataThread(pPeer);
} else { } else {
sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
@ -1312,7 +1317,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
} }
// always update version // always update version
sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, sTrace("vgId:%d, update version, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
nodeVersion = pWalHead->version; nodeVersion = pWalHead->version;

24
src/sync/src/syncRestore.c
View File

@ -36,6 +36,8 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
if (sindex < 0 || eindex < sindex) return; if (sindex < 0 || eindex < sindex) return;
sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex);
while (1) { while (1) {
name[0] = 0; name[0] = 0;
magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion);
@ -43,7 +45,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
(void)remove(fname); (void)remove(fname);
sDebug("%s, %s is removed", pPeer->id, fname); sInfo("%s, %s is removed for its extra", pPeer->id, fname);
index++; index++;
if (index > eindex) break; if (index > eindex) break;
@ -61,11 +63,12 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
bool fileChanged = false; bool fileChanged = false;
*fversion = 0; *fversion = 0;
sinfo.index = 0; sinfo.index = -1;
while (1) { while (1) {
// read file info // read file info
int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); minfo.index = -1;
if (ret < 0) { int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo));
if (ret != sizeof(SFileInfo) || minfo.index == -1) {
sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno)); sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno));
break; break;
} }
@ -75,7 +78,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
sDebug("%s, no more files to restore", pPeer->id); sDebug("%s, no more files to restore", pPeer->id);
// remove extra files after the current index // remove extra files after the current index
syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX);
code = 0; code = 0;
break; break;
} }
@ -96,7 +99,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
// send file ack // send file ack
ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck));
if (ret < 0) { if (ret != sizeof(fileAck)) {
sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno));
break; break;
} }
@ -154,7 +157,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
while (1) { while (1) {
ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
if (ret < 0) { if (ret != sizeof(SWalHead)) {
sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno));
break; break;
} }
@ -166,7 +169,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
} // wal sync over } // wal sync over
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret < 0) { if (ret != pHead->len) {
sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno));
break; break;
} }
@ -286,11 +289,12 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
uint64_t fversion = 0; uint64_t fversion = 0;
sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
SFirstPktRsp firstPktRsp = {.sync = 1}; SFirstPktRsp firstPktRsp = {.sync = 1, .tranId = syncGenTranId()};
if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) != sizeof(SFirstPktRsp)) {
sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno)); sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno));
return -1; return -1;
} }
sDebug("%s, send firstPktRsp to peer, tranId:%u", pPeer->id, firstPktRsp.tranId);
sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
int32_t code = syncRestoreFile(pPeer, &fversion); int32_t code = syncRestoreFile(pPeer, &fversion);

36
src/sync/src/syncRetrieve.c
View File

@ -58,7 +58,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver; uint64_t fver, wver;
int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
if (code != 0) { if (code != 0) {
sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
return -1; return -1;
} }
@ -92,7 +92,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
int32_t code = -1; int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0}; char name[TSDB_FILENAME_LEN * 2] = {0};
if (syncGetFileVersion(pNode, pPeer) < 0) return -1; if (syncGetFileVersion(pNode, pPeer) < 0) {
pPeer->fileChanged = 1;
return -1;
}
while (1) { while (1) {
// retrieve file info // retrieve file info
@ -100,12 +103,11 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
fileInfo.size = 0; fileInfo.size = 0;
fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
&fileInfo.size, &fileInfo.fversion); &fileInfo.size, &fileInfo.fversion);
// fileInfo.size = htonl(size);
sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
// send the file info // send the file info
int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
if (ret < 0) { if (ret != sizeof(fileInfo)) {
code = -1; code = -1;
sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break; break;
@ -119,8 +121,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
} }
// wait for the ack from peer // wait for the ack from peer
ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
if (ret < 0) { if (ret != sizeof(SFileAck)) {
code = -1; code = -1;
sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break; break;
@ -384,12 +386,15 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
} }
if (code == 0) { if (code == 0) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
SWalHead walHead; SWalHead walHead;
memset(&walHead, 0, sizeof(walHead)); memset(&walHead, 0, sizeof(walHead));
taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); if (taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)) == sizeof(walHead)) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
} else {
sError("%s, failed to send last wal record since %s", pPeer->id, strerror(errno));
code = -1;
}
} else { } else {
sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code); sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code);
} }
@ -404,20 +409,23 @@ static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) {
memset(&firstPkt, 0, sizeof(firstPkt)); memset(&firstPkt, 0, sizeof(firstPkt));
firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA; firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA;
firstPkt.syncHead.vgId = pNode->vgId; firstPkt.syncHead.vgId = pNode->vgId;
firstPkt.tranId = syncGenTranId();
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort; firstPkt.port = tsSyncPort;
if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) { if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno)); sError("%s, failed to send sync firstPkt since %s, tranId:%u", pPeer->id, strerror(errno), firstPkt.tranId);
return -1; return -1;
} }
sDebug("%s, send sync-data pkt to peer, tranId:%u", pPeer->id, firstPkt.tranId);
SFirstPktRsp firstPktRsp; SFirstPktRsp firstPktRsp;
if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) != sizeof(SFirstPktRsp)) {
sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno)); sError("%s, failed to read sync firstPkt rsp since %s, tranId:%u", pPeer->id, strerror(errno), firstPkt.tranId);
return -1; return -1;
} }
sDebug("%s, recv firstPktRsp from peer, tranId:%u", pPeer->id, firstPkt.tranId);
return 0; return 0;
} }

1
src/util/inc/tconfig.h
View File

@ -78,6 +78,7 @@ extern char * tsCfgStatusStr[];
void taosReadGlobalLogCfg(); void taosReadGlobalLogCfg();
bool taosReadGlobalCfg(); bool taosReadGlobalCfg();
void taosPrintGlobalCfg(); void taosPrintGlobalCfg();
void taosDumpGlobalCfg();
void taosInitConfigOption(SGlobalCfg cfg); void taosInitConfigOption(SGlobalCfg cfg);
SGlobalCfg * taosGetConfigOption(const char *option); SGlobalCfg * taosGetConfigOption(const char *option);

54
src/util/src/tconfig.c
View File

@ -397,3 +397,57 @@ void taosPrintGlobalCfg() {
taosPrintOsInfo(); taosPrintOsInfo();
} }
static void taosDumpCfg(SGlobalCfg *cfg) {
int optionLen = (int)strlen(cfg->option);
int blankLen = TSDB_CFG_PRINT_LEN - optionLen;
blankLen = blankLen < 0 ? 0 : blankLen;
char blank[TSDB_CFG_PRINT_LEN];
memset(blank, ' ', TSDB_CFG_PRINT_LEN);
blank[blankLen] = 0;
switch (cfg->valType) {
case TAOS_CFG_VTYPE_INT16:
printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_INT32:
printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_FLOAT:
printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_STRING:
case TAOS_CFG_VTYPE_IPSTR:
case TAOS_CFG_VTYPE_DIRECTORY:
printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]);
break;
default:
break;
}
}
void taosDumpGlobalCfg() {
printf("taos global config:\n");
printf("==================================\n");
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue;
taosDumpCfg(cfg);
}
printf("\ntaos local config:\n");
printf("==================================\n");
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue;
taosDumpCfg(cfg);
}
}

5
src/util/src/ttimer.c
View File

@ -225,10 +225,11 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
} }
static bool removeFromWheel(tmr_obj_t* timer) { static bool removeFromWheel(tmr_obj_t* timer) {
if (timer->wheel >= tListLen(wheels)) { uint8_t wheelIdx = timer->wheel;
if (wheelIdx >= tListLen(wheels)) {
return false; return false;
} }
time_wheel_t* wheel = wheels + timer->wheel; time_wheel_t* wheel = wheels + wheelIdx;
bool removed = false; bool removed = false;
pthread_mutex_lock(&wheel->mutex); pthread_mutex_lock(&wheel->mutex);

1
src/vnode/inc/vnodeInt.h
View File

@ -46,6 +46,7 @@ typedef struct {
int8_t isFull; int8_t isFull;
int8_t isCommiting; int8_t isCommiting;
uint64_t version; // current version uint64_t version; // current version
uint64_t cversion; // version while commit start
uint64_t fversion; // version on saved data file uint64_t fversion; // version on saved data file
void * wqueue; // write queue void * wqueue; // write queue
void * qqueue; // read query queue void * qqueue; // read query queue

7
src/vnode/src/vnodeMain.c
View File

@ -203,8 +203,8 @@ int32_t vnodeOpen(int32_t vgId) {
code = vnodeReadVersion(pVnode); code = vnodeReadVersion(pVnode);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to read version, generate it from data file", pVnode->vgId); vError("vgId:%d, failed to read file version, generate it from data file", pVnode->vgId);
// Allow vnode start even when read version fails, set version as walVersion or zero // Allow vnode start even when read file version fails, set file version as wal version or zero
// vnodeCleanUp(pVnode); // vnodeCleanUp(pVnode);
// return code; // return code;
} }
@ -447,6 +447,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
if (status == TSDB_STATUS_COMMIT_START) { if (status == TSDB_STATUS_COMMIT_START) {
pVnode->isCommiting = 1; pVnode->isCommiting = 1;
pVnode->cversion = pVnode->version;
vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
if (!vnodeInInitStatus(pVnode)) { if (!vnodeInInitStatus(pVnode)) {
return walRenew(pVnode->wal); return walRenew(pVnode->wal);
@ -457,7 +458,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
if (status == TSDB_STATUS_COMMIT_OVER) { if (status == TSDB_STATUS_COMMIT_OVER) {
pVnode->isCommiting = 0; pVnode->isCommiting = 0;
pVnode->isFull = 0; pVnode->isFull = 0;
pVnode->fversion = pVnode->version; pVnode->fversion = pVnode->cversion;
vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
if (!vnodeInInitStatus(pVnode)) { if (!vnodeInInitStatus(pVnode)) {
walRemoveOneOldFile(pVnode->wal); walRemoveOneOldFile(pVnode->wal);

18
src/vnode/src/vnodeRead.c
View File

@ -133,7 +133,7 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void
int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
vDebug("QInfo:%p add to vread queue for exec query", *qhandle); vTrace("QInfo:%p add to vread queue for exec query", *qhandle);
} }
return code; return code;
@ -164,7 +164,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle,
} }
} else { } else {
*freeHandle = true; *freeHandle = true;
vDebug("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); vTrace("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle);
} }
} else { } else {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@ -266,7 +266,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
} }
if (handle != NULL) { if (handle != NULL) {
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); vTrace("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pRsp->code = code; pRsp->code = code;
@ -278,10 +278,10 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
assert(pCont != NULL); assert(pCont != NULL);
void **qhandle = (void **)pRead->qhandle; void **qhandle = (void **)pRead->qhandle;
vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
// In the retrieve blocking model, only 50% CPU will be used in query processing // In the retrieve blocking model, only 50% CPU will be used in query processing
if (tsHalfCoresForQuery) { if (tsRetrieveBlockingModel) {
qTableQuery(*qhandle); // do execute query qTableQuery(*qhandle); // do execute query
qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false);
} else { } else {
@ -294,7 +294,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
assert(pRead->rpcHandle != NULL); assert(pRead->rpcHandle != NULL);
vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, vTrace("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
pRead->rpcHandle); pRead->rpcHandle);
// set the real rsp error code // set the real rsp error code
@ -327,7 +327,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
pRetrieve->free = htons(pRetrieve->free); pRetrieve->free = htons(pRetrieve->free);
pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
pRetrieve->free, pRead->rpcHandle); pRetrieve->free, pRead->rpcHandle);
memset(pRet, 0, sizeof(SRspRet)); memset(pRet, 0, sizeof(SRspRet));
@ -380,7 +380,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
freeHandle = true; freeHandle = true;
} else { // result is not ready, return immediately } else { // result is not ready, return immediately
// Only effects in the non-blocking model // Only effects in the non-blocking model
if (!tsHalfCoresForQuery) { if (!tsRetrieveBlockingModel) {
if (!buildRes) { if (!buildRes) {
assert(pRead->rpcHandle != NULL); assert(pRead->rpcHandle != NULL);
@ -410,6 +410,6 @@ int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) {
pMsg->header.vgId = htonl(vgId); pMsg->header.vgId = htonl(vgId);
pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle);
return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg)); return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg));
} }

10
src/vnode/src/vnodeWrite.c
View File

@ -243,8 +243,10 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar
int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1);
if (queued > MAX_QUEUED_MSG_NUM) { if (queued > MAX_QUEUED_MSG_NUM) {
vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3;
taosMsleep(1); if (ms > 100) ms = 100;
vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms);
taosMsleep(ms);
} }
code = vnodePerformFlowCtrl(pWrite); code = vnodePerformFlowCtrl(pWrite);
@ -271,6 +273,8 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) {
SVnodeObj * pVnode = pWrite->pVnode; SVnodeObj * pVnode = pWrite->pVnode;
int32_t code = TSDB_CODE_VND_SYNCING; int32_t code = TSDB_CODE_VND_SYNCING;
if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL;
pWrite->processedCount++; pWrite->processedCount++;
if (pWrite->processedCount > 100) { if (pWrite->processedCount > 100) {
vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code),
@ -290,8 +294,8 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) {
static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) {
SVnodeObj *pVnode = pWrite->pVnode; SVnodeObj *pVnode = pWrite->pVnode;
if (pVnode->flowctrlLevel <= 0) return 0;
if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (pWrite->qtype != TAOS_QTYPE_RPC) return 0;
if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0;
if (tsFlowCtrl == 0) { if (tsFlowCtrl == 0) {
int32_t ms = pow(2, pVnode->flowctrlLevel + 2); int32_t ms = pow(2, pVnode->flowctrlLevel + 2);

2
src/wal/inc/walInt.h
View File

@ -38,7 +38,7 @@ extern int32_t wDebugFlag;
#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE)) #define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_FILE_NUM 3 #define WAL_FILE_NUM 1 // 3
typedef struct { typedef struct {
uint64_t version; uint64_t version;

4
src/wal/src/walWrite.c
View File

@ -173,7 +173,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
continue; continue;
} }
wInfo("vgId:%d, file:%s, restore success", pWal->vgId, walName); wInfo("vgId:%d, file:%s, restore success, wver:%" PRIu64, pWal->vgId, walName, pWal->version);
count++; count++;
} }
@ -267,8 +267,6 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
return TAOS_SYSTEM_ERROR(errno); return TAOS_SYSTEM_ERROR(errno);
} }
wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int64_t offset = 0; int64_t offset = 0;
SWalHead *pHead = buffer; SWalHead *pHead = buffer;

205
tests/Jenkinsfile vendored
View File

@ -1,16 +1,3 @@
// execute this before anything else, including requesting any time on an agent
if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) {
print "INFO: Build skipped due to trigger being Branch Indexing"
currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful
return
}
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine.git'
}
def pre_test(){ def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
@ -19,15 +6,14 @@ def pre_test(){
} }
sh ''' sh '''
cd ${WKC} cd ${WKC}
rm -rf * git reset --hard
git checkout ${BRANCH}
git pull
git submodule update
cd ${WK} cd ${WK}
git reset --hard git reset --hard
git checkout develop git checkout ${BRANCH}
git pull git pull
cd ${WKC}
rm -rf *
mv ${WORKSPACE}/* .
cd ${WK}
export TZ=Asia/Harbin export TZ=Asia/Harbin
date date
rm -rf ${WK}/debug rm -rf ${WK}/debug
@ -36,13 +22,13 @@ def pre_test(){
cmake .. > /dev/null cmake .. > /dev/null
make > /dev/null make > /dev/null
make install > /dev/null make install > /dev/null
cd ${WKC}/tests
''' '''
return 1 return 1
} }
pipeline { pipeline {
agent none agent none
environment{ environment{
BRANCH = 'develop'
WK = '/var/lib/jenkins/workspace/TDinternal' WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDinternal/community' WKC= '/var/lib/jenkins/workspace/TDinternal/community'
} }
@ -50,13 +36,13 @@ pipeline {
stages { stages {
stage('Parallel test stage') { stage('Parallel test stage') {
parallel { parallel {
stage('python p1') { stage('pytest') {
agent{label 'p1'} agent{label '184'}
steps { steps {
pre_test() pre_test()
sh ''' sh '''
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh p1 ./test-all.sh pytest
date''' date'''
} }
} }
@ -64,6 +50,12 @@ pipeline {
agent{label 'master'} agent{label 'master'}
steps { steps {
pre_test() pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
python3 concurrent_inquiry.py -c 1
'''
}
sh ''' sh '''
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b1 ./test-all.sh b1
@ -72,9 +64,12 @@ pipeline {
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "b2"} agent{label "185"}
steps { steps {
pre_test() pre_test()
sh '''
cd ${WKC}/tests/pytest
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
@ -88,7 +83,6 @@ pipeline {
''' '''
} }
sh ''' sh '''
date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b2 ./test-all.sh b2
date date
@ -97,42 +91,177 @@ pipeline {
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "b3"} agent{label "186"}
steps { steps {
pre_test() pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log ./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh ./handle_val_log.sh
'''
}
sh '''
date date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b3 ./test-all.sh b3
date''' date'''
} }
} }
stage('python p2'){ stage('connector'){
agent{label "p2"} agent{label "release"}
steps{ steps{
pre_test()
sh''' sh'''
date cd ${WORKSPACE}
cd ${WKC}/tests git checkout develop
./test-all.sh p2
date
''' '''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
} }
} }
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
} }
} }
} }
post {
success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
} }

33
tests/examples/JDBC/taosdemo/.gitignore vendored Normal file
View File

@ -0,0 +1,33 @@
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/

118
tests/examples/JDBC/taosdemo/.mvn/wrapper/MavenWrapperDownloader.java vendored Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if (mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if (mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if (!outputFile.getParentFile().exists()) {
if (!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
String username = System.getenv("MVNW_USERNAME");
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
}
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

BIN
tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.jar vendored Normal file
View File

Binary file not shown.

2
tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.properties vendored Normal file
View File

@ -0,0 +1,2 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

322
tests/examples/JDBC/taosdemo/mvnw vendored Executable file
View File

@ -0,0 +1,322 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ]; then
if [ -f /etc/mavenrc ]; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ]; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false
darwin=false
mingw=false
case "$(uname)" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true ;;
Darwin*)
darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="$(/usr/libexec/java_home)"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ]; then
if [ -r /etc/gentoo-release ]; then
JAVA_HOME=$(java-config --jre-home)
fi
fi
if [ -z "$M2_HOME" ]; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ]; do
ls=$(ls -ld "$PRG")
link=$(expr "$ls" : '.*-> \(.*\)$')
if expr "$link" : '/.*' >/dev/null; then
PRG="$link"
else
PRG="$(dirname "$PRG")/$link"
fi
done
saveddir=$(pwd)
M2_HOME=$(dirname "$PRG")/..
# make it fully qualified
M2_HOME=$(cd "$M2_HOME" && pwd)
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=$(cygpath --unix "$M2_HOME")
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
[ -n "$CLASSPATH" ] &&
CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw; then
[ -n "$M2_HOME" ] &&
M2_HOME="$( (
cd "$M2_HOME"
pwd
))"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="$( (
cd "$JAVA_HOME"
pwd
))"
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="$(which javac)"
if [ -n "$javaExecutable" ] && ! [ "$(expr \"$javaExecutable\" : '\([^ ]*\)')" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=$(which readlink)
if [ ! $(expr "$readLink" : '\([^ ]*\)') = "no" ]; then
if $darwin; then
javaHome="$(dirname \"$javaExecutable\")"
javaExecutable="$(cd \"$javaHome\" && pwd -P)/javac"
else
javaExecutable="$(readlink -f \"$javaExecutable\")"
fi
javaHome="$(dirname \"$javaExecutable\")"
javaHome=$(expr "$javaHome" : '\(.*\)/bin')
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ]; then
if [ -n "$JAVA_HOME" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="$(which java)"
fi
fi
if [ ! -x "$JAVACMD" ]; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ]; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]; then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ]; do
if [ -d "$wdir"/.mvn ]; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=$(
cd "$wdir/.."
pwd
)
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' <"$1")"
fi
}
BASE_DIR=$(find_maven_basedir "$(pwd)")
if [ -z "$BASE_DIR" ]; then
exit 1
fi
##########################################################################################
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
fi
while IFS="=" read key value; do
case "$key" in wrapperUrl)
jarUrl="$value"
break
;;
esac
done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
fi
if command -v wget >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=$(cygpath --path --windows "$javaClass")
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
##########################################################################################
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=$(cygpath --path --windows "$M2_HOME")
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
[ -n "$CLASSPATH" ] &&
CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
fi
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

182
tests/examples/JDBC/taosdemo/mvnw.cmd vendored Normal file
View File

@ -0,0 +1,182 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM https://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM set title of command window
title %0
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
)
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
if exist %WRAPPER_JAR% (
if "%MVNW_VERBOSE%" == "true" (
echo Found %WRAPPER_JAR%
)
) else (
if not "%MVNW_REPOURL%" == "" (
SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
)
if "%MVNW_VERBOSE%" == "true" (
echo Couldn't find %WRAPPER_JAR%, downloading it ...
echo Downloading from: %DOWNLOAD_URL%
)
powershell -Command "&{"^
"$webclient = new-object System.Net.WebClient;"^
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
"}"^
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
"}"
if "%MVNW_VERBOSE%" == "true" (
echo Finished downloading %WRAPPER_JAR%
)
)
@REM End of extension
@REM Provide a "standardized" way to retrieve the CLI args that will
@REM work with both Windows and non-Windows executions.
set MAVEN_CMD_LINE_ARGS=%*
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
exit /B %ERROR_CODE%

117
tests/examples/JDBC/taosdemo/pom.xml Normal file
View File

@ -0,0 +1,117 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.4.0</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.taosdata</groupId>
<artifactId>taosdemo</artifactId>
<version>2.0</version>
<name>taosdemo</name>
<description>Demo project for TDengine</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<!-- taos jdbc -->
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.14</version>
</dependency>
<!-- mysql -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
<!-- mybatis-plus -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.1.2</version>
</dependency>
<!-- log4j -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- springboot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>2.1.4</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

15
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosdemoApplication.java Normal file
View File

@ -0,0 +1,15 @@
package com.taosdata.taosdemo;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@MapperScan(basePackages = {"com.taosdata.taosdemo.mapper"})
@SpringBootApplication
public class TaosdemoApplication {
public static void main(String[] args) {
SpringApplication.run(TaosdemoApplication.class, args);
}
}

174
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/TaosDemoCommandLineRunner.java Normal file
View File

@ -0,0 +1,174 @@
package com.taosdata.taosdemo.components;
import com.taosdata.taosdemo.domain.*;
import com.taosdata.taosdemo.service.DatabaseService;
import com.taosdata.taosdemo.service.SubTableService;
import com.taosdata.taosdemo.service.SuperTableService;
import com.taosdata.taosdemo.service.data.SubTableMetaGenerator;
import com.taosdata.taosdemo.service.data.SubTableValueGenerator;
import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator;
import com.taosdata.taosdemo.utils.JdbcTaosdemoConfig;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.TimeUnit;
@Component
public class TaosDemoCommandLineRunner implements CommandLineRunner {
private static Logger logger = Logger.getLogger(TaosDemoCommandLineRunner.class);
@Autowired
private DatabaseService databaseService;
@Autowired
private SuperTableService superTableService;
@Autowired
private SubTableService subTableService;
private SuperTableMeta superTableMeta;
private List<SubTableMeta> subTableMetaList;
private List<SubTableValue> subTableValueList;
private List<List<SubTableValue>> dataList;
@Override
public void run(String... args) throws Exception {
// 读配置参数
JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args);
boolean isHelp = Arrays.asList(args).contains("--help");
if (isHelp) {
JdbcTaosdemoConfig.printHelp();
System.exit(0);
}
// 准备数据
prepareData(config);
// 创建数据库
createDatabaseTask(config);
// 建表
createTableTask(config);
// 插入
insertTask(config);
// 查询: 1. 生成查询语句, 2. 执行查询
// 删除表
if (config.dropTable) {
superTableService.drop(config.database, config.superTable);
}
System.exit(0);
}
private void createDatabaseTask(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
Map<String, String> databaseParam = new HashMap<>();
databaseParam.put("database", config.database);
databaseParam.put("keep", Integer.toString(config.keep));
databaseParam.put("days", Integer.toString(config.days));
databaseParam.put("replica", Integer.toString(config.replica));
//TODO: other database parameters
databaseService.dropDatabase(config.database);
databaseService.createDatabase(databaseParam);
databaseService.useDatabase(config.database);
long end = System.currentTimeMillis();
logger.info(">>> insert time cost : " + (end - start) + " ms.");
}
// 建超级表，三种方式：1. 指定SQL，2. 指定field和tags的个数，3. 默认
private void createTableTask(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
if (config.doCreateTable) {
superTableService.create(superTableMeta);
// 批量建子表
subTableService.createSubTable(subTableMetaList, config.numOfThreadsForCreate);
}
long end = System.currentTimeMillis();
logger.info(">>> create table time cost : " + (end - start) + " ms.");
}
private void insertTask(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
int numOfThreadsForInsert = config.numOfThreadsForInsert;
int sleep = config.sleep;
if (config.autoCreateTable) {
// 批量插入，自动建表
dataList.stream().forEach(subTableValues -> {
subTableService.insertAutoCreateTable(subTableValues, numOfThreadsForInsert);
sleep(sleep);
});
} else {
dataList.stream().forEach(subTableValues -> {
subTableService.insert(subTableValues, numOfThreadsForInsert);
sleep(sleep);
});
}
long end = System.currentTimeMillis();
logger.info(">>> insert time cost : " + (end - start) + " ms.");
}
private void prepareData(JdbcTaosdemoConfig config) {
long start = System.currentTimeMillis();
// 超级表的meta
superTableMeta = createSupertable(config);
// 子表的meta
subTableMetaList = SubTableMetaGenerator.generate(superTableMeta, config.numOfTables, config.tablePrefix);
// 子表的data
subTableValueList = SubTableValueGenerator.generate(subTableMetaList, config.numOfRowsPerTable, config.startTime, config.timeGap);
// 如果有乱序，给数据搞乱
if (config.order != 0) {
SubTableValueGenerator.disrupt(subTableValueList, config.rate, config.range);
}
// 分割数据
int numOfTables = config.numOfTables;
int numOfTablesPerSQL = config.numOfTablesPerSQL;
int numOfRowsPerTable = config.numOfRowsPerTable;
int numOfValuesPerSQL = config.numOfValuesPerSQL;
dataList = SubTableValueGenerator.split(subTableValueList, numOfTables, numOfTablesPerSQL, numOfRowsPerTable, numOfValuesPerSQL);
long end = System.currentTimeMillis();
logger.info(">>> prepare data time cost : " + (end - start) + " ms.");
}
private SuperTableMeta createSupertable(JdbcTaosdemoConfig config) {
SuperTableMeta tableMeta;
// create super table
logger.info(">>> create super table <<<");
if (config.superTableSQL != null) {
// use a sql to create super table
tableMeta = SuperTableMetaGenerator.generate(config.superTableSQL);
} else if (config.numOfFields == 0) {
// default sql = "create table test.weather (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)";
SuperTableMeta superTableMeta = new SuperTableMeta();
superTableMeta.setDatabase(config.database);
superTableMeta.setName(config.superTable);
List<FieldMeta> fields = new ArrayList<>();
fields.add(new FieldMeta("ts", "timestamp"));
fields.add(new FieldMeta("temperature", "float"));
fields.add(new FieldMeta("humidity", "int"));
superTableMeta.setFields(fields);
List<TagMeta> tags = new ArrayList<>();
tags.add(new TagMeta("location", "nchar(64)"));
tags.add(new TagMeta("groupId", "int"));
superTableMeta.setTags(tags);
return superTableMeta;
} else {
// create super table with specified field size and tag size
tableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, config.prefixOfFields, config.numOfTags, config.prefixOfTags);
}
return tableMeta;
}
private static void sleep(int sleep) {
if (sleep <= 0)
return;
try {
TimeUnit.MILLISECONDS.sleep(sleep);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

40
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/DatabaseController.java Normal file
View File

@ -0,0 +1,40 @@
package com.taosdata.taosdemo.controller;
import com.taosdata.taosdemo.service.DatabaseService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
@RestController
@RequestMapping
public class DatabaseController {
@Autowired
private DatabaseService databaseService;
/**
* create database
***/
@PostMapping
public int create(@RequestBody Map<String, String> map) {
return databaseService.createDatabase(map);
}
/**
* drop database
**/
@DeleteMapping("/{dbname}")
public int delete(@PathVariable("dbname") String dbname) {
return databaseService.dropDatabase(dbname);
}
/**
* use database
**/
@GetMapping("/{dbname}")
public int use(@PathVariable("dbname") String dbname) {
return databaseService.useDatabase(dbname);
}
}

17
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/InsertController.java Normal file
View File

@ -0,0 +1,17 @@
package com.taosdata.taosdemo.controller;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class InsertController {
//TODO：多线程写一张表, thread = 10, table = 1
//TODO：一个批次写多张表, insert into t1 using weather values() t2 using weather values()
//TODO：插入的频率，
//TODO：指定一张表内的records数量
//TODO：是否乱序，
//TODO：乱序的比例，乱序的范围
//TODO：先建表，自动建表
//TODO：一个批次写多张表
}

45
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SubTableController.java Normal file
View File

@ -0,0 +1,45 @@
package com.taosdata.taosdemo.controller;
import com.taosdata.taosdemo.domain.TableValue;
import com.taosdata.taosdemo.service.SuperTableService;
import com.taosdata.taosdemo.service.TableService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class SubTableController {
@Autowired
private TableService tableService;
@Autowired
private SuperTableService superTableService;
//TODO: 使用supertable创建一个子表
//TODO：使用supertable创建多个子表
//TODO：使用supertable多线程创建子表
//TODO：使用supertable多线程创建子表，指定子表的name_prefix，子表的数量，使用线程的个数
/**
* 创建表，超级表或者普通表
**/
/**
* 创建超级表的子表
**/
@PostMapping("/{database}/{superTable}")
public int createTable(@PathVariable("database") String database,
@PathVariable("superTable") String superTable,
@RequestBody TableValue tableMetadta) {
tableMetadta.setDatabase(database);
return 0;
}
}

26
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SuperTableController.java Normal file
View File

@ -0,0 +1,26 @@
package com.taosdata.taosdemo.controller;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import com.taosdata.taosdemo.service.SuperTableService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
public class SuperTableController {
@Autowired
private SuperTableService superTableService;
@PostMapping("/{database}")
public int createTable(@PathVariable("database") String database, @RequestBody SuperTableMeta tableMetadta) {
tableMetadta.setDatabase(database);
return superTableService.create(tableMetadta);
}
//TODO: 删除超级表
//TODO：查询超级表
//TODO：统计查询表
}

11
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/TableController.java Normal file
View File

@ -0,0 +1,11 @@
package com.taosdata.taosdemo.controller;
public class TableController {
//TODO：创建普通表，create table(ts timestamp, temperature float)
//TODO：创建普通表，指定表的列数，包括第一列timestamp
//TODO：创建普通表，指定表每列的name和type
}

17
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldMeta.java Normal file
View File

@ -0,0 +1,17 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
@Data
public class FieldMeta {
private String name;
private String type;
public FieldMeta() {
}
public FieldMeta(String name, String type) {
this.name = name;
this.type = type;
}
}

17
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldValue.java Normal file
View File

@ -0,0 +1,17 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
@Data
public class FieldValue<T> {
private String name;
private T value;
public FieldValue() {
}
public FieldValue(String name, T value) {
this.name = name;
this.value = value;
}
}

15
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java Normal file
View File

@ -0,0 +1,15 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class RowValue {
private List<FieldValue> fields;
public RowValue(List<FieldValue> fields) {
this.fields = fields;
}
}

15
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableMeta.java Normal file
View File

@ -0,0 +1,15 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class SubTableMeta {
private String database;
private String supertable;
private String name;
private List<TagValue> tags;
private List<FieldMeta> fields;
}

15
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableValue.java Normal file
View File

@ -0,0 +1,15 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class SubTableValue {
private String database;
private String supertable;
private String name;
private List<TagValue> tags;
private List<RowValue> values;
}

14
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SuperTableMeta.java Normal file
View File

@ -0,0 +1,14 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class SuperTableMeta {
private String database;
private String name;
private List<FieldMeta> fields;
private List<TagMeta> tags;
}

13
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableMeta.java Normal file
View File

@ -0,0 +1,13 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class TableMeta {
private String database;
private String name;
private List<FieldMeta> fields;
}

15
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableValue.java Normal file
View File

@ -0,0 +1,15 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
import java.util.List;
@Data
public class TableValue {
private String database;
private String name;
private List<FieldMeta> columns;
private List<RowValue> values;
}

18
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagMeta.java Normal file
View File

@ -0,0 +1,18 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
@Data
public class TagMeta {
private String name;
private String type;
public TagMeta() {
}
public TagMeta(String name, String type) {
this.name = name;
this.type = type;
}
}

17
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagValue.java Normal file
View File

@ -0,0 +1,17 @@
package com.taosdata.taosdemo.domain;
import lombok.Data;
@Data
public class TagValue<T> {
private String name;
private T value;
public TagValue() {
}
public TagValue(String name, T value) {
this.name = name;
this.value = value;
}
}

27
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.java Normal file
View File

@ -0,0 +1,27 @@
package com.taosdata.taosdemo.mapper;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.Map;
@Repository
public interface DatabaseMapper {
// create database if not exists XXX
int createDatabase(@Param("database") String dbname);
// drop database if exists XXX
int dropDatabase(@Param("database") String dbname);
// create database if not exists XXX keep XX days XX replica XX
int createDatabaseWithParameters(Map<String, String> map);
// use XXX
int useDatabase(@Param("database") String dbname);
//TODO: alter database
//TODO: show database
}

48
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.xml Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.taosdata.taosdemo.mapper.DatabaseMapper">
<!-- create database XXX -->
<update id="createDatabase" parameterType="java.lang.String">
create database if not exists ${database}
</update>
<update id="dropDatabase" parameterType="java.lang.String">
DROP database if exists ${database}
</update>
<update id="createDatabaseWithParameters" parameterType="map">
CREATE database if not exists ${database}
<if test="keep != null">
KEEP ${keep}
</if>
<if test="days != null">
DAYS ${days}
</if>
<if test="replica != null">
REPLICA ${replica}
</if>
<if test="cache != null">
cache ${cache}
</if>
<if test="blocks != null">
blocks ${blocks}
</if>
<if test="minrows != null">
minrows ${minrows}
</if>
<if test="maxrows != null">
maxrows ${maxrows}
</if>
</update>
<update id="useDatabase" parameterType="java.lang.String">
use ${database}
</update>
<!-- TODO: alter database -->
<!-- TODO: show database -->
</mapper>

30
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.java Normal file
View File

@ -0,0 +1,30 @@
package com.taosdata.taosdemo.mapper;
import com.taosdata.taosdemo.domain.SubTableMeta;
import com.taosdata.taosdemo.domain.SubTableValue;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface SubTableMapper {
// 创建：子表
int createUsingSuperTable(SubTableMeta subTableMeta);
// 插入：一张子表多个values
int insertOneTableMultiValues(SubTableValue subTableValue);
// 插入：一张子表多个values, 自动建表
int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue);
// 插入：多张表多个values
int insertMultiTableMultiValues(@Param("tables") List<SubTableValue> tables);
// 插入：多张表多个values，自动建表
int insertMultiTableMultiValuesUsingSuperTable(@Param("tables") List<SubTableValue> tables);
//<!-- TODO:修改子表标签值 alter table ${tablename} set tag tagName=newTagValue-->
}

81
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.xml Normal file
View File

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.taosdata.taosdemo.mapper.SubTableMapper">
<!-- 创建子表 -->
<update id="createUsingSuperTable">
CREATE table IF NOT EXISTS ${database}.${name} USING ${supertable} TAGS
<foreach collection="tags" item="tag" index="index" open="(" close=")" separator=",">
#{tag.value}
</foreach>
</update>
<!-- 插入：向一张表中插入多张表 -->
<insert id="insertOneTableMultiValues">
INSERT INTO ${database}.${name}
VALUES
<foreach collection="values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
#{field.value}
</foreach>
</foreach>
</insert>
<!-- 插入：使用自动建表模式，向一张表中插入多条数据 -->
<insert id="insertOneTableMultiValuesUsingSuperTable">
INSERT INTO ${database}.${name} USING ${supertable} TAGS
<foreach collection="tags" item="tag" index="index" open="(" close=")" separator=",">
#{tag.value}
</foreach>
VALUES
<foreach collection="values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
#{field.value}
</foreach>
</foreach>
</insert>
<!-- TODO:插入：向一张表中插入多张表, 指定列 -->
<!-- TODO:插入：向一张表中插入多张表, 自动建表，指定列 -->
<!-- 插入：向多张表中插入多条数据 -->
<insert id="insertMultiTableMultiValues">
INSERT INTO
<foreach collection="tables" item="table">
${table.database}.${table.name}
VALUES
<foreach collection="table.values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
#{field.value}
</foreach>
</foreach>
</foreach>
</insert>
<!-- 插入：向多张表中插入多条数据，自动建表 -->
<insert id="insertMultiTableMultiValuesUsingSuperTable">
INSERT INTO
<foreach collection="tables" item="table">
${table.database}.${table.name} USING ${table.supertable} TAGS
<foreach collection="table.tags" item="tag" index="index" open="(" close=")" separator=",">
#{tag.value}
</foreach>
VALUES
<foreach collection="table.values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
#{field.value}
</foreach>
</foreach>
</foreach>
</insert>
<!-- TODO:插入：向多张表中插入多张表, 指定列 -->
<!-- TODO:插入：向多张表中插入多张表, 自动建表，指定列 -->
<!-- TODO:修改子表标签值 alter table ${tablename} set tag tagName=newTagValue -->
<!-- TODO: -->
</mapper>

33
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.java Normal file
View File

@ -0,0 +1,33 @@
package com.taosdata.taosdemo.mapper;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
@Repository
public interface SuperTableMapper {
// 创建超级表，使用自己定义的SQL语句
int createSuperTableUsingSQL(@Param("createSuperTableSQL") String sql);
// 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...)
int createSuperTable(SuperTableMeta tableMetadata);
// 删除超级表 drop table if exists xxx;
int dropSuperTable(@Param("database") String database, @Param("name") String name);
//<!-- TODO:查询所有超级表信息 show stables -->
//<!-- TODO:查询表结构 describe stable -->
//<!-- TODO:增加列 alter table ${tablename} add column fieldName dataType -->
//<!-- TODO:删除列 alter table ${tablename} drop column fieldName -->
//<!-- TODO:添加标签 alter table ${tablename} add tag new_tagName tag_type -->
//<!-- TODO:删除标签 alter table ${tablename} drop tag_name -->
//<!-- TODO:修改标签名 alter table ${tablename} change tag old_tagName new_tagName -->
}

41
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.xml Normal file
View File

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.taosdata.taosdemo.mapper.SuperTableMapper">
<update id="createSuperTableUsingSQL">
${createSuperTableSQL}
</update>
<!-- 创建超级表 -->
<update id="createSuperTable">
create table if not exists ${database}.${name}
<foreach collection="fields" item="field" index="index" open="(" close=")" separator=",">
${field.name} ${field.type}
</foreach>
tags
<foreach collection="tags" item="tag" index="index" open="(" close=")" separator=",">
${tag.name} ${tag.type}
</foreach>
</update>
<!-- 删除超级表：drop super table -->
<delete id="dropSuperTable">
drop table if exists ${database}.${name}
</delete>
<!-- TODO:查询所有超级表信息 show stables -->
<!-- TODO:查询表结构 describe stable -->
<!-- TODO:增加列 alter table ${tablename} add column fieldName dataType -->
<!-- TODO:删除列 alter table ${tablename} drop column fieldName -->
<!-- TODO:添加标签 alter table ${tablename} add tag new_tagName tag_type -->
<!-- TODO:删除标签 alter table ${tablename} drop tag_name -->
<!-- TODO:修改标签名 alter table ${tablename} change tag old_tagName new_tagName -->
</mapper>

28
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.java Normal file
View File

@ -0,0 +1,28 @@
package com.taosdata.taosdemo.mapper;
import com.taosdata.taosdemo.domain.TableMeta;
import com.taosdata.taosdemo.domain.TableValue;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface TableMapper {
// 创建：普通表
int create(TableMeta tableMeta);
// 插入：一张表多个value
int insertOneTableMultiValues(TableValue values);
// 插入: 一张表多个value，指定的列
int insertOneTableMultiValuesWithColumns(TableValue values);
// 插入：多个表多个value
int insertMultiTableMultiValues(@Param("tables") List<TableValue> tables);
// 插入：多个表多个value, 指定的列
int insertMultiTableMultiValuesWithColumns(@Param("tables") List<TableValue> tables);
}

68
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.xml Normal file
View File

@ -0,0 +1,68 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.taosdata.taosdemo.mapper.TableMapper">
<!-- 创建普通表 -->
<update id="create" parameterType="com.taosdata.taosdemo.domain.TableMeta">
create table if not exists ${database}.${name}
<foreach collection="fields" item="field" index="index" open="(" close=")" separator=",">
${field.name} ${field.type}
</foreach>
</update>
<!-- 插入：向一张普通表中插入多条数据 -->
<insert id="insertOneTableMultiValues" parameterType="com.taosdata.taosdemo.domain.TableValue">
insert into ${database}.${name} values
<foreach collection="values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
${field.value}
</foreach>
</foreach>
</insert>
<!-- 向一张表中插入指定列的数据 insert into XXX.xx (f1,f2,f3...) values(v1,v2,v3...) -->
<insert id="insertOneTableMultiValuesWithColumns" parameterType="com.taosdata.taosdemo.domain.TableValue">
insert into ${database}.${name}
<foreach collection="columns" item="column" open="(" close=")" separator=",">
${column.name}
</foreach>
values
<foreach collection="values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
${field.value}
</foreach>
</foreach>
</insert>
<!-- 向多个表中插入多条数据 -->
<insert id="insertMultiTableMultiValues">
insert into
<foreach collection="tables" item="table">
${table.database}.${table.name} values
<foreach collection="table.values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
${field.value}
</foreach>
</foreach>
</foreach>
</insert>
<!-- 向多张表中指定的列插入多条数据 -->
<insert id="insertMultiTableMultiValuesWithColumns">
insert into
<foreach collection="tables" item="table">
${table.database}.${table.name}
<foreach collection="table.columns" item="column" open="(" close=")" separator=",">
${column.name}
</foreach>
values
<foreach collection="table.values" item="value">
<foreach collection="value.fields" item="field" open="(" close=")" separator=",">
${field.value}
</foreach>
</foreach>
</foreach>
</insert>
</mapper>

35
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/AbstractService.java Normal file
View File

@ -0,0 +1,35 @@
package com.taosdata.taosdemo.service;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
public class AbstractService {
protected int getAffectRows(List<Future<Integer>> futureList) {
int count = 0;
for (Future<Integer> future : futureList) {
try {
count += future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
return count;
}
protected int getAffectRows(Future<Integer> future) {
int count = 0;
try {
count += future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
return count;
}
}

38
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java Normal file
View File

@ -0,0 +1,38 @@
package com.taosdata.taosdemo.service;
import com.taosdata.taosdemo.mapper.DatabaseMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Map;
@Service
public class DatabaseService {
@Autowired
private DatabaseMapper databaseMapper;
// 建库，指定 name
public int createDatabase(String database) {
return databaseMapper.createDatabase(database);
}
// 建库，指定参数 keep,days,replica等
public int createDatabase(Map<String, String> map) {
if (map.isEmpty())
return 0;
if (map.containsKey("database") && map.size() == 1)
return databaseMapper.createDatabase(map.get("database"));
return databaseMapper.createDatabaseWithParameters(map);
}
// drop database
public int dropDatabase(String dbname) {
return databaseMapper.dropDatabase(dbname);
}
// use database
public int useDatabase(String dbname) {
return databaseMapper.useDatabase(dbname);
}
}

118
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java Normal file
View File

@ -0,0 +1,118 @@
package com.taosdata.taosdemo.service;
import com.taosdata.taosdemo.domain.SubTableMeta;
import com.taosdata.taosdemo.domain.SubTableValue;
import com.taosdata.taosdemo.mapper.SubTableMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@Service
public class SubTableService extends AbstractService {
@Autowired
private SubTableMapper mapper;
/**
* 1. 选择database，找到所有supertable
* 2. 选择supertable，可以拿到表结构，包括field和tag
* 3. 指定子表的前缀和个数
* 4. 指定创建子表的线程数
*/
//TODO：指定database、supertable、子表前缀、子表个数、线程数
// 多线程创建表，指定线程个数
public int createSubTable(List<SubTableMeta> subTables, int threadSize) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
List<Future<Integer>> futureList = new ArrayList<>();
for (SubTableMeta subTableMeta : subTables) {
Future<Integer> future = executor.submit(() -> createSubTable(subTableMeta));
futureList.add(future);
}
executor.shutdown();
return getAffectRows(futureList);
}
// 创建一张子表，可以指定database，supertable，tablename，tag值
public int createSubTable(SubTableMeta subTableMeta) {
return mapper.createUsingSuperTable(subTableMeta);
}
// 单线程创建多张子表，每张子表分别可以指定自己的database，supertable，tablename，tag值
public int createSubTable(List<SubTableMeta> subTables) {
return createSubTable(subTables, 1);
}
/*************************************************************************************************************************/
// 插入：多线程，多表
public int insert(List<SubTableValue> subTableValues, int threadSize) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
Future<Integer> future = executor.submit(() -> insert(subTableValues));
executor.shutdown();
return getAffectRows(future);
}
// 插入：多线程，多表, 自动建表
public int insertAutoCreateTable(List<SubTableValue> subTableValues, int threadSize) {
ExecutorService executor = Executors.newFixedThreadPool(threadSize);
Future<Integer> future = executor.submit(() -> insertAutoCreateTable(subTableValues));
executor.shutdown();
return getAffectRows(future);
}
// 插入：单表，insert into xxx values(),()...
public int insert(SubTableValue subTableValue) {
return mapper.insertOneTableMultiValues(subTableValue);
}
// 插入: 多表，insert into xxx values(),()... xxx values(),()...
public int insert(List<SubTableValue> subTableValues) {
return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues);
}
// 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()...
public int insertAutoCreateTable(SubTableValue subTableValue) {
return mapper.insertOneTableMultiValuesUsingSuperTable(subTableValue);
}
// 插入：多表，自动建表, insert into xxx using XXX tags(...) values(),()... xxx using XXX tags(...) values(),()...
public int insertAutoCreateTable(List<SubTableValue> subTableValues) {
return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues);
}
// ExecutorService executors = Executors.newFixedThreadPool(threadSize);
// int count = 0;
//
// //
// List<SubTableValue> subTableValues = new ArrayList<>();
// for (int tableIndex = 1; tableIndex <= numOfTablesPerSQL; tableIndex++) {
// // each table
// SubTableValue subTableValue = new SubTableValue();
// subTableValue.setDatabase();
// subTableValue.setName();
// subTableValue.setSupertable();
//
// List<RowValue> values = new ArrayList<>();
// for (int valueCnt = 0; valueCnt < numOfValuesPerSQL; valueCnt++) {
// List<FieldValue> fields = new ArrayList<>();
// for (int fieldInd = 0; fieldInd <; fieldInd++) {
// FieldValue<Object> field = new FieldValue<>("", "");
// fields.add(field);
// }
// RowValue row = new RowValue();
// row.setFields(fields);
// values.add(row);
// }
// subTableValue.setValues(values);
// subTableValues.add(subTableValue);
// }
}

22
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java Normal file
View File

@ -0,0 +1,22 @@
package com.taosdata.taosdemo.service;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import com.taosdata.taosdemo.mapper.SuperTableMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class SuperTableService {
@Autowired
private SuperTableMapper superTableMapper;
// 创建超级表，指定每个field的名称和类型，每个tag的名称和类型
public int create(SuperTableMeta superTableMeta) {
return superTableMapper.createSuperTable(superTableMeta);
}
public void drop(String database, String name) {
superTableMapper.dropSuperTable(database, name);
}
}

42
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java Normal file
View File

@ -0,0 +1,42 @@
package com.taosdata.taosdemo.service;
import com.taosdata.taosdemo.domain.TableMeta;
import com.taosdata.taosdemo.mapper.TableMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@Service
public class TableService extends AbstractService {
@Autowired
private TableMapper tableMapper;
//创建一张表
public int create(TableMeta tableMeta) {
return tableMapper.create(tableMeta);
}
//创建多张表
public int create(List<TableMeta> tables) {
return create(tables, 1);
}
//多线程创建多张表
public int create(List<TableMeta> tables, int threadSize) {
ExecutorService executors = Executors.newFixedThreadPool(threadSize);
List<Future<Integer>> futures = new ArrayList<>();
for (TableMeta table : tables) {
Future<Integer> future = executors.submit(() -> create(table));
futures.add(future);
}
return getAffectRows(futures);
}
}

48
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java Normal file
View File

@ -0,0 +1,48 @@
package com.taosdata.taosdemo.service.data;
import com.taosdata.taosdemo.domain.FieldMeta;
import com.taosdata.taosdemo.domain.FieldValue;
import com.taosdata.taosdemo.domain.RowValue;
import com.taosdata.taosdemo.utils.DataGenerator;
import java.util.*;
public class FieldValueGenerator {
public static Random random = new Random(System.currentTimeMillis());
// 生成start到end的时间序列，时间戳为顺序，不含有乱序，field的value为随机生成
public static List<RowValue> generate(long start, long end, long timeGap, List<FieldMeta> fieldMetaList) {
List<RowValue> values = new ArrayList<>();
for (long ts = start; ts < end; ts += timeGap) {
List<FieldValue> fieldValues = new ArrayList<>();
// timestamp
fieldValues.add(new FieldValue(fieldMetaList.get(0).getName(), ts));
// other values
for (int fieldInd = 1; fieldInd < fieldMetaList.size(); fieldInd++) {
FieldMeta fieldMeta = fieldMetaList.get(fieldInd);
fieldValues.add(new FieldValue(fieldMeta.getName(), DataGenerator.randomValue(fieldMeta.getType())));
}
values.add(new RowValue(fieldValues));
}
return values;
}
// 生成start到end的时间序列，时间戳为顺序，含有乱序，rate为乱序的比例，range为乱序前跳范围，field的value为随机生成
public static List<RowValue> disrupt(List<RowValue> values, int rate, long range) {
long timeGap = (long) (values.get(1).getFields().get(0).getValue()) - (long) (values.get(0).getFields().get(0).getValue());
int bugSize = values.size() * rate / 100;
Set<Integer> bugIndSet = new HashSet<>();
while (bugIndSet.size() < bugSize) {
bugIndSet.add(random.nextInt(values.size()));
}
for (Integer bugInd : bugIndSet) {
Long timestamp = (Long) values.get(bugInd).getFields().get(0).getValue();
Long newTimestamp = timestamp - timeGap - random.nextInt((int) range);
values.get(bugInd).getFields().get(0).setValue(newTimestamp);
}
return values;
}
}

30
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java Normal file
View File

@ -0,0 +1,30 @@
package com.taosdata.taosdemo.service.data;
import com.taosdata.taosdemo.domain.SubTableMeta;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import com.taosdata.taosdemo.domain.TagValue;
import java.util.ArrayList;
import java.util.List;
public class SubTableMetaGenerator {
// 创建tableSize张子表，使用tablePrefix作为子表名的前缀，使用superTableMeta的元数据
// create table xxx using XXX tags(XXX)
public static List<SubTableMeta> generate(SuperTableMeta superTableMeta, int tableSize, String tablePrefix) {
List<SubTableMeta> subTableMetaList = new ArrayList<>();
for (int i = 1; i <= tableSize; i++) {
SubTableMeta subTableMeta = new SubTableMeta();
// create table xxx.xxx using xxx tags(...)
subTableMeta.setDatabase(superTableMeta.getDatabase());
subTableMeta.setName(tablePrefix + i);
subTableMeta.setSupertable(superTableMeta.getName());
subTableMeta.setFields(superTableMeta.getFields());
List<TagValue> tagValues = TagValueGenerator.generate(superTableMeta.getTags());
subTableMeta.setTags(tagValues);
subTableMetaList.add(subTableMeta);
}
return subTableMetaList;
}
}

84
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java Normal file
View File

@ -0,0 +1,84 @@
package com.taosdata.taosdemo.service.data;
import com.taosdata.taosdemo.domain.RowValue;
import com.taosdata.taosdemo.domain.SubTableMeta;
import com.taosdata.taosdemo.domain.SubTableValue;
import com.taosdata.taosdemo.utils.TimeStampUtil;
import org.springframework.beans.BeanUtils;
import java.util.ArrayList;
import java.util.List;
public class SubTableValueGenerator {
public static List<SubTableValue> generate(List<SubTableMeta> subTableMetaList, int numOfRowsPerTable, long start, long timeGap) {
List<SubTableValue> subTableValueList = new ArrayList<>();
subTableMetaList.stream().forEach((subTableMeta) -> {
// insert into xxx.xxx using xxxx tags(...) values(),()...
SubTableValue subTableValue = new SubTableValue();
subTableValue.setDatabase(subTableMeta.getDatabase());
subTableValue.setName(subTableMeta.getName());
subTableValue.setSupertable(subTableMeta.getSupertable());
subTableValue.setTags(subTableMeta.getTags());
TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(start, timeGap, numOfRowsPerTable);
List<RowValue> values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields());
subTableValue.setValues(values);
subTableValueList.add(subTableValue);
});
return subTableValueList;
}
public static void disrupt(List<SubTableValue> subTableValueList, int rate, long range) {
subTableValueList.stream().forEach((tableValue) -> {
List<RowValue> values = tableValue.getValues();
FieldValueGenerator.disrupt(values, rate, range);
});
}
public static List<List<SubTableValue>> split(List<SubTableValue> subTableValueList, int numOfTables, int numOfTablesPerSQL, int numOfRowsPerTable, int numOfValuesPerSQL) {
List<List<SubTableValue>> dataList = new ArrayList<>();
if (numOfRowsPerTable < numOfValuesPerSQL)
numOfValuesPerSQL = numOfRowsPerTable;
if (numOfTables < numOfTablesPerSQL)
numOfTablesPerSQL = numOfTables;
//table
for (int tableCnt = 0; tableCnt < numOfTables; ) {
int tableSize = numOfTablesPerSQL;
if (tableCnt + tableSize > numOfTables) {
tableSize = numOfTables - tableCnt;
}
// row
for (int rowCnt = 0; rowCnt < numOfRowsPerTable; ) {
int rowSize = numOfValuesPerSQL;
if (rowCnt + rowSize > numOfRowsPerTable) {
rowSize = numOfRowsPerTable - rowCnt;
}
// System.out.println("rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", tableCnt: " + tableCnt + ", tableSize: " + tableSize);
// split
List<SubTableValue> blocks = subTableValueList.subList(tableCnt, tableCnt + tableSize);
List<SubTableValue> newBlocks = new ArrayList<>();
for (int i = 0; i < blocks.size(); i++) {
SubTableValue subTableValue = blocks.get(i);
SubTableValue newSubTableValue = new SubTableValue();
BeanUtils.copyProperties(subTableValue, newSubTableValue);
List<RowValue> values = subTableValue.getValues().subList(rowCnt, rowCnt + rowSize);
newSubTableValue.setValues(values);
newBlocks.add(newSubTableValue);
}
dataList.add(newBlocks);
rowCnt += rowSize;
}
tableCnt += tableSize;
}
return dataList;
}
public static void main(String[] args) {
split(null, 99, 10, 99, 10);
}
}

80
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java Normal file
View File

@ -0,0 +1,80 @@
package com.taosdata.taosdemo.service.data;
import com.taosdata.taosdemo.domain.FieldMeta;
import com.taosdata.taosdemo.domain.SuperTableMeta;
import com.taosdata.taosdemo.domain.TagMeta;
import com.taosdata.taosdemo.utils.TaosConstants;
import java.util.ArrayList;
import java.util.List;
public class SuperTableMetaGenerator {
// 创建超级表，使用指定SQL语句
public static SuperTableMeta generate(String superTableSQL) {
SuperTableMeta tableMeta = new SuperTableMeta();
// for example : create table superTable (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)
superTableSQL = superTableSQL.trim().toLowerCase();
if (!superTableSQL.startsWith("create"))
throw new RuntimeException("invalid create super table SQL");
if (superTableSQL.contains("tags")) {
String tagSQL = superTableSQL.substring(superTableSQL.indexOf("tags") + 4).trim();
tagSQL = tagSQL.substring(tagSQL.indexOf("(") + 1, tagSQL.lastIndexOf(")"));
String[] tagPairs = tagSQL.split(",");
List<TagMeta> tagMetaList = new ArrayList<>();
for (String tagPair : tagPairs) {
String name = tagPair.trim().split("\\s+")[0];
String type = tagPair.trim().split("\\s+")[1];
tagMetaList.add(new TagMeta(name, type));
}
tableMeta.setTags(tagMetaList);
superTableSQL = superTableSQL.substring(0, superTableSQL.indexOf("tags"));
}
if (superTableSQL.contains("(")) {
String fieldSQL = superTableSQL.substring(superTableSQL.indexOf("(") + 1, superTableSQL.indexOf(")"));
String[] fieldPairs = fieldSQL.split(",");
List<FieldMeta> fieldList = new ArrayList<>();
for (String fieldPair : fieldPairs) {
String name = fieldPair.trim().split("\\s+")[0];
String type = fieldPair.trim().split("\\s+")[1];
fieldList.add(new FieldMeta(name, type));
}
tableMeta.setFields(fieldList);
superTableSQL = superTableSQL.substring(0, superTableSQL.indexOf("("));
}
superTableSQL = superTableSQL.substring(superTableSQL.indexOf("table") + 5).trim();
if (superTableSQL.contains(".")) {
String database = superTableSQL.split("\\.")[0];
tableMeta.setDatabase(database);
superTableSQL = superTableSQL.substring(superTableSQL.indexOf(".") + 1);
}
tableMeta.setName(superTableSQL.trim());
return tableMeta;
}
// 创建超级表,指定field和tag的个数
public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, String tagPrefix) {
if (fieldSize < 2 || tagSize < 1) {
throw new RuntimeException("create super table but fieldSize less than 2 or tagSize less than 1");
}
SuperTableMeta tableMetadata = new SuperTableMeta();
tableMetadata.setDatabase(database);
tableMetadata.setName(name);
// fields
List<FieldMeta> fields = new ArrayList<>();
fields.add(new FieldMeta("ts", "timestamp"));
for (int i = 1; i <= fieldSize; i++) {
fields.add(new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length]));
}
tableMetadata.setFields(fields);
// tags
List<TagMeta> tags = new ArrayList<>();
for (int i = 1; i <= tagSize; i++) {
tags.add(new TagMeta(tagPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length]));
}
tableMetadata.setTags(tags);
return tableMetadata;
}
}

24
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java Normal file
View File

@ -0,0 +1,24 @@
package com.taosdata.taosdemo.service.data;
import com.taosdata.taosdemo.domain.TagMeta;
import com.taosdata.taosdemo.domain.TagValue;
import com.taosdata.taosdemo.utils.DataGenerator;
import java.util.ArrayList;
import java.util.List;
public class TagValueGenerator {
// 创建标签值：使用tagMetas
public static List<TagValue> generate(List<TagMeta> tagMetas) {
List<TagValue> tagValues = new ArrayList<>();
for (int i = 0; i < tagMetas.size(); i++) {
TagMeta tagMeta = tagMetas.get(i);
TagValue tagValue = new TagValue();
tagValue.setName(tagMeta.getName());
tagValue.setValue(DataGenerator.randomValue(tagMeta.getType()));
tagValues.add(tagValue);
}
return tagValues;
}
}

120
tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/DataGenerator.java Normal file
View File

@ -0,0 +1,120 @@
package com.taosdata.taosdemo.utils;
import java.util.Random;
public class DataGenerator {
private static Random random = new Random(System.currentTimeMillis());
private static final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
// "timestamp", "int", "bigint", "float", "double", "binary(64)", "smallint", "tinyint", "bool", "nchar(64)",
public static Object randomValue(String type) {
int length = 64;
if (type.contains("(")) {
length = Integer.parseInt(type.substring(type.indexOf("(") + 1, type.indexOf(")")));
type = type.substring(0, type.indexOf("("));
}
switch (type.trim().toLowerCase()) {
case "timestamp":
return randomTimestamp();
case "int":
return randomInt();
case "bigint":
return randomBigint();
case "float":
return randomFloat();
case "double":
return randomDouble();
case "binary":
return randomBinary(length);
case "smallint":
return randomSmallint();
case "tinyint":
return randomTinyint();
case "bool":
return randomBoolean();
case "nchar":
return randomNchar(length);
default:
throw new IllegalArgumentException("Unexpected value: " + type);
}
}
public static Long randomTimestamp() {
long start = System.currentTimeMillis();
return randomTimestamp(start, start + 60l * 60l * 1000l);
}
public static Long randomTimestamp(Long start, Long end) {
return start + (long) random.nextInt((int) (end - start));
}
public static String randomNchar(int length) {
return randomChinese(length);
}
public static Boolean randomBoolean() {
return random.nextBoolean();
}
public static Integer randomTinyint() {
return randomInt(-127, 127);
}
public static Integer randomSmallint() {
return randomInt(-32767, 32767);
}
public static String randomBinary(int length) {
return randomString(length);
}
public static String randomString(int length) {
String zh_en = "";
for (int i = 0; i < length; i++) {
zh_en += alphabet.charAt(random.nextInt(alphabet.length()));
}
return zh_en;
}
public static String randomChinese(int length) {
String zh_cn = "";
int bottom = Integer.parseInt("4e00", 16);
int top = Integer.parseInt("9fa5", 16);
for (int i = 0; i < length; i++) {
char c = (char) (random.nextInt(top - bottom + 1) + bottom);
zh_cn += new String(new char[]{c});
}
return zh_cn;
}
public static Double randomDouble() {
return randomDouble(0, 100);
}
public static Double randomDouble(double bottom, double top) {
return bottom + (top - bottom) * random.nextDouble();
}
public static Float randomFloat() {
return randomFloat(0, 100);
}
public static Float randomFloat(float bottom, float top) {
return bottom + (top - bottom) * random.nextFloat();
}
public static Long randomBigint() {
return random.nextLong();
}
public static Integer randomInt(int bottom, int top) {
return bottom + random.nextInt((top - bottom));
}
public static Integer randomInt() {
return randomInt(0, 100);
}
}

Some files were not shown because too many files have changed in this diff Show More