Merge branch 'master' of github.com:taosdata/TDengine into test/chr
This commit is contained in:
tomchon
commit
0eafd19005
|
|
@ -1,12 +1,12 @@
|
|||
[submodule "src/connector/go"]
|
||||
path = src/connector/go
|
||||
url = https://github.com/taosdata/driver-go
|
||||
url = git@github.com:taosdata/driver-go.git
|
||||
[submodule "src/connector/grafanaplugin"]
|
||||
path = src/connector/grafanaplugin
|
||||
url = https://github.com/taosdata/grafanaplugin
|
||||
url = git@github.com:taosdata/grafanaplugin.git
|
||||
[submodule "src/connector/hivemq-tdengine-extension"]
|
||||
path = src/connector/hivemq-tdengine-extension
|
||||
url = https://github.com/huskar-t/hivemq-tdengine-extension.git
|
||||
url = git@github.com:taosdata/hivemq-tdengine-extension.git
|
||||
[submodule "tests/examples/rust"]
|
||||
path = tests/examples/rust
|
||||
url = https://github.com/songtianyi/tdengine-rust-bindings.git
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ PROJECT(TDengine)
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "2.0.20.2")
|
||||
SET(TD_VER_NUMBER "2.0.20.5")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
|
|||
|
|
@ -58,7 +58,12 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat
|
|||
cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include
|
||||
cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include
|
||||
cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples
|
||||
cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_path}/connector
|
||||
if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then
|
||||
cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin
|
||||
else
|
||||
echo "grafanaplugin bundled directory not found!"
|
||||
exit 1
|
||||
fi
|
||||
cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
|
||||
cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector
|
||||
cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector
|
||||
|
|
|
|||
|
|
@ -66,7 +66,12 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
|
|||
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
|
||||
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
|
||||
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include
|
||||
cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector
|
||||
if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then
|
||||
cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin
|
||||
else
|
||||
echo grafanaplugin bundled directory not found!
|
||||
exit 1
|
||||
fi
|
||||
cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
|
||||
cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
|
||||
cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector
|
||||
|
|
|
|||
|
|
@ -243,9 +243,17 @@ function install_data() {
|
|||
}
|
||||
|
||||
function install_connector() {
|
||||
${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin ${install_main_dir}/connector
|
||||
if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then
|
||||
${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin
|
||||
else
|
||||
echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
|
||||
fi
|
||||
if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then
|
||||
${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector
|
||||
else
|
||||
echo "WARNING: go connector not found, please check if want to use it!"
|
||||
fi
|
||||
${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector
|
||||
${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector
|
||||
|
||||
${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,10 +117,18 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
|
|||
if [ "$osType" != "Darwin" ]; then
|
||||
cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
|
||||
fi
|
||||
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
cp -r ${connector_dir}/nodejs ${install_dir}/connector
|
||||
if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
|
||||
cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
|
||||
else
|
||||
echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
|
||||
fi
|
||||
if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
else
|
||||
echo "WARNING: go connector not found, please check if want to use it!"
|
||||
fi
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector
|
||||
cp -r ${connector_dir}/nodejs ${install_dir}/connector
|
||||
fi
|
||||
# Copy release note
|
||||
# cp ${script_dir}/release_note ${install_dir}
|
||||
|
|
|
|||
|
|
@ -144,9 +144,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
|
|||
if [ "$osType" != "Darwin" ]; then
|
||||
cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
|
||||
fi
|
||||
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
|
||||
cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
|
||||
else
|
||||
echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!"
|
||||
fi
|
||||
if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
else
|
||||
echo "WARNING: go connector not found, please check if want to use it!"
|
||||
fi
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector
|
||||
|
||||
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py
|
||||
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py
|
||||
|
|
|
|||
|
|
@ -114,6 +114,25 @@ mkdir -p ${install_dir}/examples
|
|||
examples_dir="${top_dir}/tests/examples"
|
||||
cp -r ${examples_dir}/c ${install_dir}/examples
|
||||
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
|
||||
if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/connectionPools/target
|
||||
fi
|
||||
if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/JDBCDemo/target
|
||||
fi
|
||||
if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target
|
||||
fi
|
||||
if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/springbootdemo/target
|
||||
fi
|
||||
if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target
|
||||
fi
|
||||
if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then
|
||||
rm -rf ${examples_dir}/JDBC/taosdemo/target
|
||||
fi
|
||||
|
||||
cp -r ${examples_dir}/JDBC ${install_dir}/examples
|
||||
cp -r ${examples_dir}/matlab ${install_dir}/examples
|
||||
cp -r ${examples_dir}/python ${install_dir}/examples
|
||||
|
|
@ -131,9 +150,17 @@ connector_dir="${code_dir}/connector"
|
|||
mkdir -p ${install_dir}/connector
|
||||
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
|
||||
cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
|
||||
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
|
||||
cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
|
||||
else
|
||||
echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!"
|
||||
fi
|
||||
if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
else
|
||||
echo "WARNING: go connector not found, please check if want to use it!"
|
||||
fi
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector
|
||||
cp -r ${connector_dir}/nodejs ${install_dir}/connector
|
||||
fi
|
||||
# Copy release note
|
||||
|
|
|
|||
|
|
@ -166,9 +166,18 @@ connector_dir="${code_dir}/connector"
|
|||
mkdir -p ${install_dir}/connector
|
||||
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
|
||||
cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
|
||||
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
|
||||
|
||||
if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
|
||||
cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
|
||||
else
|
||||
echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
|
||||
fi
|
||||
if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
else
|
||||
echo "WARNING: go connector not found, please check if want to use it!"
|
||||
fi
|
||||
cp -r ${connector_dir}/python ${install_dir}/connector/
|
||||
cp -r ${connector_dir}/go ${install_dir}/connector
|
||||
|
||||
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py
|
||||
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
name: tdengine
|
||||
base: core18
|
||||
version: '2.0.20.2'
|
||||
version: '2.0.20.5'
|
||||
icon: snap/gui/t-dengine.svg
|
||||
summary: an open-source big data platform designed and optimized for IoT.
|
||||
description: |
|
||||
|
|
@ -72,7 +72,7 @@ parts:
|
|||
- usr/bin/taosd
|
||||
- usr/bin/taos
|
||||
- usr/bin/taosdemo
|
||||
- usr/lib/libtaos.so.2.0.20.2
|
||||
- usr/lib/libtaos.so.2.0.20.5
|
||||
- usr/lib/libtaos.so.1
|
||||
- usr/lib/libtaos.so
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5
|
||||
Subproject commit 8ce6d86558afc8c0b50c10f990fd2b4270cf06fc
|
||||
|
|
@ -391,7 +391,7 @@ typedef struct SuperQueryInfo_S {
|
|||
uint64_t sqlCount;
|
||||
char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
|
||||
char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
|
||||
int resubAfterConsume[MAX_QUERY_SQL_COUNT];
|
||||
int resubAfterConsume;
|
||||
TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT];
|
||||
|
||||
char* childTblName;
|
||||
|
|
@ -1114,21 +1114,21 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
|
|||
|
||||
static void appendResultBufToFile(char *resultBuf, char *resultFile)
|
||||
{
|
||||
FILE *fp = NULL;
|
||||
if (resultFile[0] != 0) {
|
||||
fp = fopen(resultFile, "at");
|
||||
if (fp == NULL) {
|
||||
errorPrint(
|
||||
"%s() LN%d, failed to open result file: %s, result will not save to file\n",
|
||||
__func__, __LINE__, resultFile);
|
||||
return;
|
||||
FILE *fp = NULL;
|
||||
if (resultFile[0] != 0) {
|
||||
fp = fopen(resultFile, "at");
|
||||
if (fp == NULL) {
|
||||
errorPrint(
|
||||
"%s() LN%d, failed to open result file: %s, result will not save to file\n",
|
||||
__func__, __LINE__, resultFile);
|
||||
return;
|
||||
}
|
||||
fprintf(fp, "%s", resultBuf);
|
||||
tmfclose(fp);
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(fp, "%s", resultBuf);
|
||||
tmfclose(fp);
|
||||
}
|
||||
|
||||
|
||||
static void appendResultToFile(TAOS_RES *res, char* resultFile) {
|
||||
TAOS_ROW row = NULL;
|
||||
int num_rows = 0;
|
||||
|
|
@ -4447,6 +4447,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
|
|||
g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
|
||||
}
|
||||
|
||||
cJSON* superResubAfterConsume =
|
||||
cJSON_GetObjectItem(superQuery, "resubAfterConsume");
|
||||
if (superResubAfterConsume
|
||||
&& superResubAfterConsume->type == cJSON_Number) {
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume =
|
||||
superResubAfterConsume->valueint;
|
||||
} else if (!superResubAfterConsume) {
|
||||
//printf("failed to read json, subscribe interval no found\n");
|
||||
////goto PARSE_OVER;
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume = 1;
|
||||
}
|
||||
|
||||
// sqls
|
||||
cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
|
||||
if (!subsqls) {
|
||||
|
|
@ -4478,18 +4490,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
|
|||
tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
|
||||
MAX_QUERY_SQL_LENGTH);
|
||||
|
||||
cJSON* superResubAfterConsume =
|
||||
cJSON_GetObjectItem(sql, "resubAfterConsume");
|
||||
if (superResubAfterConsume
|
||||
&& superResubAfterConsume->type == cJSON_Number) {
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume[j] =
|
||||
superResubAfterConsume->valueint;
|
||||
} else if (!superResubAfterConsume) {
|
||||
//printf("failed to read json, subscribe interval no found\n");
|
||||
//goto PARSE_OVER;
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
|
||||
}
|
||||
|
||||
cJSON *result = cJSON_GetObjectItem(sql, "result");
|
||||
if (result != NULL && result->type == cJSON_String
|
||||
&& result->valuestring != NULL){
|
||||
|
|
@ -6612,15 +6612,6 @@ static void *superSubscribe(void *sarg) {
|
|||
exit(-1);
|
||||
}
|
||||
|
||||
if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
|
||||
errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n",
|
||||
g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables,
|
||||
g_queryInfo.superQueryInfo.sqlCount,
|
||||
pThreadInfo->ntables,
|
||||
MAX_QUERY_SQL_COUNT);
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
if (pThreadInfo->taos == NULL) {
|
||||
TAOS * taos = NULL;
|
||||
taos = taos_connect(g_queryInfo.host,
|
||||
|
|
@ -6688,6 +6679,8 @@ static void *superSubscribe(void *sarg) {
|
|||
}
|
||||
TAOS_RES* res = NULL;
|
||||
|
||||
uint64_t st = 0, et = 0;
|
||||
|
||||
while(1) {
|
||||
for (uint64_t i = pThreadInfo->start_table_from;
|
||||
i <= pThreadInfo->end_table_to; i++) {
|
||||
|
|
@ -6696,7 +6689,12 @@ static void *superSubscribe(void *sarg) {
|
|||
continue;
|
||||
}
|
||||
|
||||
st = taosGetTimestampMs();
|
||||
performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
|
||||
res = taos_consume(tsub[tsubSeq]);
|
||||
et = taosGetTimestampMs();
|
||||
performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));
|
||||
|
||||
if (res) {
|
||||
if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
|
||||
sprintf(pThreadInfo->fp, "%s-%d",
|
||||
|
|
@ -6714,7 +6712,7 @@ static void *superSubscribe(void *sarg) {
|
|||
|
||||
if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
|
||||
&& (consumed[tsubSeq] >=
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
|
||||
g_queryInfo.superQueryInfo.resubAfterConsume)) {
|
||||
printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
|
||||
g_queryInfo.superQueryInfo.subscribeKeepProgress,
|
||||
pThreadInfo->querySeq);
|
||||
|
|
|
|||
|
|
@ -709,7 +709,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
|
|||
}
|
||||
|
||||
static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
|
||||
int32_t onlineNum = 0;
|
||||
int32_t onlineNum = 0, arbOnlineNum = 0;
|
||||
int32_t masterIndex = -1;
|
||||
int32_t replica = pNode->replica;
|
||||
|
||||
|
|
@ -723,13 +723,15 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
|
|||
SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
|
||||
if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
|
||||
onlineNum++;
|
||||
++arbOnlineNum;
|
||||
replica = pNode->replica + 1;
|
||||
}
|
||||
|
||||
if (onlineNum <= replica * 0.5) {
|
||||
if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
|
||||
if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) {
|
||||
if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && ((replica > 2 && onlineNum - arbOnlineNum > 1) || pNode->replica < 3)) {
|
||||
sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica);
|
||||
masterIndex = pNode->selfIndex;
|
||||
} else {
|
||||
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
|
||||
sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
|
||||
|
|
@ -1002,6 +1004,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
|
|||
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
|
||||
// nodeVersion = pHead->version;
|
||||
code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
|
||||
syncConfirmForward(pNode->rid, pHead->version, code, false);
|
||||
} else {
|
||||
if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
|
||||
code = syncSaveIntoBuffer(pPeer, pHead);
|
||||
|
|
@ -1404,7 +1407,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
|
|||
pthread_mutex_lock(&pNode->mutex);
|
||||
for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
|
||||
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS;
|
||||
if (ABS(time - pFwdInfo->time) < 2000) break;
|
||||
if (ABS(time - pFwdInfo->time) < 10000) break;
|
||||
|
||||
sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId,
|
||||
pFwdInfo->version, time, pFwdInfo->time);
|
||||
|
|
|
|||
|
|
@ -613,7 +613,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
|
|||
|
||||
// todo memory leak if there are object with refcount greater than 0 in hash table?
|
||||
taosHashCleanup(pCacheObj->pHashTable);
|
||||
taosTrashcanEmpty(pCacheObj, true);
|
||||
taosTrashcanEmpty(pCacheObj, false);
|
||||
|
||||
__cache_lock_destroy(pCacheObj);
|
||||
|
||||
|
|
|
|||
|
|
@ -64,18 +64,25 @@ function runQueryPerfTest {
|
|||
[ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT
|
||||
nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 &
|
||||
echoInfo "Wait TDengine to start"
|
||||
sleep 300
|
||||
sleep 60
|
||||
echoInfo "Run Performance Test"
|
||||
cd $WORK_DIR/TDengine/tests/pytest
|
||||
|
||||
python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
|
||||
|
||||
mkdir -p /var/lib/perf/
|
||||
mkdir -p /var/log/perf/
|
||||
rm -rf /var/lib/perf/*
|
||||
rm -rf /var/log/perf/*
|
||||
nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 &
|
||||
echoInfo "Wait TDengine to start"
|
||||
sleep 10
|
||||
echoInfo "Run Performance Test"
|
||||
cd $WORK_DIR/TDengine/tests/pytest
|
||||
|
||||
python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
|
||||
|
||||
python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
|
||||
|
||||
#python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ python3 ./test.py -f stable/query_after_reset.py
|
|||
# perfbenchmark
|
||||
python3 ./test.py -f perfbenchmark/bug3433.py
|
||||
#python3 ./test.py -f perfbenchmark/bug3589.py
|
||||
|
||||
python3 ./test.py -f perfbenchmark/taosdemoInsert.py
|
||||
|
||||
#query
|
||||
python3 ./test.py -f query/filter.py
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class insertFromCSVPerformace:
|
|||
self.host = "127.0.0.1"
|
||||
self.user = "root"
|
||||
self.password = "taosdata"
|
||||
self.config = "/etc/taosperf"
|
||||
self.config = "/etc/perf"
|
||||
self.conn = taos.connect(
|
||||
self.host,
|
||||
self.user,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,387 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import taos
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
import subprocess
|
||||
import datetime
|
||||
import re
|
||||
|
||||
|
||||
from multiprocessing import cpu_count
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.dnodes import TDDnode
|
||||
|
||||
class Taosdemo:
|
||||
def __init__(self, clearCache, dbName, keep):
|
||||
self.clearCache = clearCache
|
||||
self.dbname = dbName
|
||||
self.drop = "yes"
|
||||
self.keep = keep
|
||||
self.host = "127.0.0.1"
|
||||
self.user = "root"
|
||||
self.password = "taosdata"
|
||||
# self.config = "/etc/taosperf"
|
||||
# self.conn = taos.connect(
|
||||
# self.host,
|
||||
# self.user,
|
||||
# self.password,
|
||||
# self.config)
|
||||
|
||||
# env config
|
||||
def getBuildPath(self) -> str:
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/debug/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def getExeToolsDir(self) -> str:
|
||||
self.debugdir = self.getBuildPath() + "/debug/build/bin"
|
||||
return self.debugdir
|
||||
|
||||
def getCfgDir(self) -> str:
|
||||
self.config = self.getBuildPath() + "/sim/dnode1/cfg"
|
||||
return self.config
|
||||
|
||||
# taodemo insert file config
|
||||
def dbinfocfg(self) -> dict:
|
||||
return {
|
||||
"name": self.dbname,
|
||||
"drop": self.drop,
|
||||
"replica": 1,
|
||||
"days": 10,
|
||||
"cache": 16,
|
||||
"blocks": 8,
|
||||
"precision": "ms",
|
||||
"keep": self.keep,
|
||||
"minRows": 100,
|
||||
"maxRows": 4096,
|
||||
"comp": 2,
|
||||
"walLevel": 1,
|
||||
"cachelast": 0,
|
||||
"quorum": 1,
|
||||
"fsync": 3000,
|
||||
"update": 0
|
||||
}
|
||||
|
||||
def type_check(func):
|
||||
def wrapper(self, **kwargs):
|
||||
num_types = ["int", "float", "bigint", "tinyint", "smallint", "double"]
|
||||
str_types = ["binary", "nchar"]
|
||||
for k ,v in kwargs.items():
|
||||
if k.lower() not in num_types and k.lower() not in str_types:
|
||||
return f"args {k} type error, not allowed"
|
||||
elif not isinstance(v, (int, list, tuple)):
|
||||
return f"value {v} type error, not allowed"
|
||||
elif k.lower() in num_types and not isinstance(v, int):
|
||||
return f"arg {v} takes 1 positional argument must be type int "
|
||||
elif isinstance(v, (list,tuple)) and len(v) > 2:
|
||||
return f"arg {v} takes from 1 to 2 positional arguments but more than 2 were given "
|
||||
elif isinstance(v,(list,tuple)) and [ False for _ in v if not isinstance(_, int) ]:
|
||||
return f"arg {v} takes from 1 to 2 positional arguments must be type int "
|
||||
else:
|
||||
pass
|
||||
return func(self, **kwargs)
|
||||
return wrapper
|
||||
|
||||
@type_check
|
||||
def column_tag_count(self, **column_tag) -> list :
|
||||
init_column_tag = []
|
||||
for k, v in column_tag.items():
|
||||
if re.search(k, "int, float, bigint, tinyint, smallint, double", re.IGNORECASE):
|
||||
init_column_tag.append({"type": k, "count": v})
|
||||
elif re.search(k, "binary, nchar", re.IGNORECASE):
|
||||
if isinstance(v, int):
|
||||
init_column_tag.append({"type": k, "count": v, "len":8})
|
||||
elif len(v) == 1:
|
||||
init_column_tag.append({"type": k, "count": v[0], "len": 8})
|
||||
else:
|
||||
init_column_tag.append({"type": k, "count": v[0], "len": v[1]})
|
||||
return init_column_tag
|
||||
|
||||
def stbcfg(self, stb: str, child_tab_count: int, rows: int, prechildtab: str, columns: dict, tags: dict) -> dict:
|
||||
return {
|
||||
"name": stb,
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": child_tab_count,
|
||||
"childtable_prefix": prechildtab,
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 10,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": rows,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset": 0,
|
||||
"rows_per_tbl": 1,
|
||||
"max_sql_len": 65480,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 10,
|
||||
"start_timestamp": f"{datetime.datetime.now():%F %X}",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": self.column_tag_count(**columns),
|
||||
"tags": self.column_tag_count(**tags)
|
||||
}
|
||||
|
||||
def schemecfg(self,intcount=1,floatcount=0,bcount=0,tcount=0,scount=0,doublecount=0,binarycount=0,ncharcount=0):
|
||||
return {
|
||||
"INT": intcount,
|
||||
"FLOAT": floatcount,
|
||||
"BIGINT": bcount,
|
||||
"TINYINT": tcount,
|
||||
"SMALLINT": scount,
|
||||
"DOUBLE": doublecount,
|
||||
"BINARY": binarycount,
|
||||
"NCHAR": ncharcount
|
||||
}
|
||||
|
||||
def insertcfg(self,db: dict, stbs: list) -> dict:
|
||||
return {
|
||||
"filetype": "insert",
|
||||
"cfgdir": self.config,
|
||||
"host": self.host,
|
||||
"port": 6030,
|
||||
"user": self.user,
|
||||
"password": self.password,
|
||||
"thread_count": cpu_count(),
|
||||
"thread_count_create_tbl": cpu_count(),
|
||||
"result_file": "/tmp/insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"insert_interval": 0,
|
||||
"num_of_records_per_req": 100,
|
||||
"max_sql_len": 1024000,
|
||||
"databases": [{
|
||||
"dbinfo": db,
|
||||
"super_tables": stbs
|
||||
}]
|
||||
}
|
||||
|
||||
def createinsertfile(self,db: dict, stbs: list) -> str:
|
||||
date = datetime.datetime.now()
|
||||
file_create_table = f"/tmp/insert_{date:%F-%H%M}.json"
|
||||
|
||||
with open(file_create_table, 'w') as f:
|
||||
json.dump(self.insertcfg(db, stbs), f)
|
||||
|
||||
return file_create_table
|
||||
|
||||
# taosdemo query file config
|
||||
def querysqls(self, sql: str) -> list:
|
||||
return [{"sql":sql,"result":""}]
|
||||
|
||||
def querycfg(self, sql: str) -> dict:
|
||||
return {
|
||||
"filetype": "query",
|
||||
"cfgdir": self.config,
|
||||
"host": self.host,
|
||||
"port": 6030,
|
||||
"user": self.user,
|
||||
"password": self.password,
|
||||
"confirm_parameter_prompt": "yes",
|
||||
"query_times": 10,
|
||||
"query_mode": "taosc",
|
||||
"databases": self.dbname,
|
||||
"specified_table_query": {
|
||||
"query_interval": 0,
|
||||
"concurrent": cpu_count(),
|
||||
"sqls": self.querysqls(sql)
|
||||
}
|
||||
}
|
||||
|
||||
def createqueryfile(self, sql: str):
|
||||
date = datetime.datetime.now()
|
||||
file_query_table = f"/tmp/query_{date:%F-%H%M}.json"
|
||||
|
||||
with open(file_query_table,"w") as f:
|
||||
json.dump(self.querycfg(sql), f)
|
||||
|
||||
return file_query_table
|
||||
|
||||
# Execute taosdemo, and delete temporary files when finished
|
||||
def taosdemotable(self, filepath: str, resultfile="/dev/null"):
|
||||
taosdemopath = self.getBuildPath() + "/debug/build/bin"
|
||||
with open(filepath,"r") as f:
|
||||
filetype = json.load(f)["filetype"]
|
||||
if filetype == "insert":
|
||||
taosdemo_table_cmd = f"{taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1"
|
||||
else:
|
||||
taosdemo_table_cmd = f"yes | {taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1"
|
||||
try:
|
||||
_ = subprocess.check_output(taosdemo_table_cmd, shell=True).decode("utf-8")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_ = e.output
|
||||
|
||||
def droptmpfile(self, filepath: str):
|
||||
drop_file_cmd = f"[ -f {filepath} ] && rm -f {filepath}"
|
||||
try:
|
||||
_ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_ = e.output
|
||||
|
||||
# TODO:需要完成TD-4153的数据插入和客户端请求的性能查询。
|
||||
def td4153insert(self):
|
||||
|
||||
tdLog.printNoPrefix("========== start to create table and insert data ==========")
|
||||
self.dbname = "td4153"
|
||||
db = self.dbinfocfg()
|
||||
stblist = []
|
||||
|
||||
columntype = self.schemecfg(intcount=1, ncharcount=100)
|
||||
tagtype = self.schemecfg(intcount=1)
|
||||
stbname = "stb1"
|
||||
prechild = "t1"
|
||||
stable = self.stbcfg(
|
||||
stb=stbname,
|
||||
prechildtab=prechild,
|
||||
child_tab_count=2,
|
||||
rows=10000,
|
||||
columns=columntype,
|
||||
tags=tagtype
|
||||
)
|
||||
stblist.append(stable)
|
||||
insertfile = self.createinsertfile(db=db, stbs=stblist)
|
||||
|
||||
nmon_file = f"/tmp/insert_{datetime.datetime.now():%F-%H%M}.nmon"
|
||||
cmd = f"nmon -s5 -F {nmon_file} -m /tmp/"
|
||||
try:
|
||||
_ = subprocess.check_output(cmd, shell=True).decode("utf-8")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_ = e.output
|
||||
|
||||
self.taosdemotable(insertfile)
|
||||
self.droptmpfile(insertfile)
|
||||
self.droptmpfile("/tmp/insert_res.txt")
|
||||
|
||||
# In order to prevent too many performance files from being generated, the nmon file is deleted.
|
||||
# and the delete statement can be cancelled during the actual test.
|
||||
self.droptmpfile(nmon_file)
|
||||
|
||||
cmd = f"ps -ef|grep -w nmon| grep -v grep | awk '{{print $2}}'"
|
||||
try:
|
||||
time.sleep(10)
|
||||
_ = subprocess.check_output(cmd,shell=True).decode("utf-8")
|
||||
except BaseException as e:
|
||||
raise e
|
||||
|
||||
def td4153query(self):
|
||||
tdLog.printNoPrefix("========== start to query operation ==========")
|
||||
|
||||
sqls = {
|
||||
"select_all": "select * from stb1",
|
||||
"select_join": "select * from t10, t11 where t10.ts=t11.ts"
|
||||
}
|
||||
for type, sql in sqls.items():
|
||||
result_file = f"/tmp/queryResult_{type}.log"
|
||||
query_file = self.createqueryfile(sql)
|
||||
try:
|
||||
self.taosdemotable(query_file, resultfile=result_file)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out_put = e.output
|
||||
if result_file:
|
||||
print(f"execute rows {type.split('_')[1]} sql, the sql is: {sql}")
|
||||
max_sql_time_cmd = f'''
|
||||
grep -o Spent.*s {result_file} |awk 'NR==1{{max=$2;next}}{{max=max>$2?max:$2}}END{{print "Max=",max,"s"}}'
|
||||
'''
|
||||
max_sql_time = subprocess.check_output(max_sql_time_cmd, shell=True).decode("UTF-8")
|
||||
print(f"{type.split('_')[1]} rows sql time : {max_sql_time}")
|
||||
|
||||
min_sql_time_cmd = f'''
|
||||
grep -o Spent.*s {result_file} |awk 'NR==1{{min=$2;next}}{{min=min<$2?min:$2}}END{{print "Min=",min,"s"}}'
|
||||
'''
|
||||
min_sql_time = subprocess.check_output(min_sql_time_cmd, shell=True).decode("UTF-8")
|
||||
print(f"{type.split('_')[1]} rows sql time : {min_sql_time}")
|
||||
|
||||
avg_sql_time_cmd = f'''
|
||||
grep -o Spent.*s {result_file} |awk '{{sum+=$2}}END{{print "Average=",sum/NR,"s"}}'
|
||||
'''
|
||||
avg_sql_time = subprocess.check_output(avg_sql_time_cmd, shell=True).decode("UTF-8")
|
||||
print(f"{type.split('_')[1]} rows sql time : {avg_sql_time}")
|
||||
|
||||
self.droptmpfile(query_file)
|
||||
self.droptmpfile(result_file)
|
||||
|
||||
drop_query_tmt_file_cmd = " find ./ -name 'querySystemInfo-*' -type f -exec rm {} \; "
|
||||
try:
|
||||
_ = subprocess.check_output(drop_query_tmt_file_cmd, shell=True).decode("utf-8")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_ = e.output
|
||||
pass
|
||||
|
||||
def td4153(self):
|
||||
self.td4153insert()
|
||||
self.td4153query()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-r',
|
||||
'--remove-cache',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='clear cache before query (default: False)')
|
||||
parser.add_argument(
|
||||
'-d',
|
||||
'--database-name',
|
||||
action='store',
|
||||
default='db',
|
||||
type=str,
|
||||
help='Database name to be created (default: db)')
|
||||
parser.add_argument(
|
||||
'-k',
|
||||
'--keep-time',
|
||||
action='store',
|
||||
default=3650,
|
||||
type=int,
|
||||
help='Database keep parameters (default: 3650)')
|
||||
|
||||
args = parser.parse_args()
|
||||
taosdemo = Taosdemo(args.remove_cache, args.database_name, args.keep_time)
|
||||
# taosdemo.conn = taos.connect(
|
||||
# taosdemo.host,
|
||||
# taosdemo.user,
|
||||
# taosdemo.password,
|
||||
# taosdemo.config
|
||||
# )
|
||||
|
||||
debugdir = taosdemo.getExeToolsDir()
|
||||
cfgdir = taosdemo.getCfgDir()
|
||||
cmd = f"{debugdir}/taosd -c {cfgdir} >/dev/null 2>&1 &"
|
||||
try:
|
||||
_ = subprocess.check_output(cmd, shell=True).decode("utf-8")
|
||||
except subprocess.CalledProcessError as e:
|
||||
_ = e.output
|
||||
|
||||
if taosdemo.clearCache:
|
||||
# must be root permission
|
||||
subprocess.check_output("echo 3 > /proc/sys/vm/drop_caches", shell=True).decode("utf-8")
|
||||
|
||||
taosdemo.td4153()
|
||||
|
|
@ -24,7 +24,7 @@ class taosdemoPerformace:
|
|||
self.host = "127.0.0.1"
|
||||
self.user = "root"
|
||||
self.password = "taosdata"
|
||||
self.config = "/etc/taosperf"
|
||||
self.config = "/etc/perf"
|
||||
self.conn = taos.connect(
|
||||
self.host,
|
||||
self.user,
|
||||
|
|
@ -77,7 +77,7 @@ class taosdemoPerformace:
|
|||
|
||||
insert_data = {
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taosperf",
|
||||
"cfgdir": "/etc/perf",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ if $dnode4Vtatus != offline then
|
|||
sleep 2000
|
||||
goto wait_dnode4_vgroup_offline
|
||||
endi
|
||||
if $dnode3Vtatus != master then
|
||||
if $dnode3Vtatus != unsynced then
|
||||
sleep 2000
|
||||
goto wait_dnode4_vgroup_offline
|
||||
endi
|
||||
|
|
|
|||
Loading…
Reference in New Issue