From 3062b1dabb3aa85ea5088edbfe815d33f01d1114 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 21 May 2021 14:37:52 +0800 Subject: [PATCH 1/4] [TD-4153] add case to resolve use taosdemo tools insert and query operation --- tests/pytest/fulltest.sh | 2 +- tests/pytest/perfbenchmark/taosdemoInsert.py | 387 +++++++++++++++++++ 2 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/perfbenchmark/taosdemoInsert.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index a748c9dd2d..d8e2a31e70 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -183,7 +183,7 @@ python3 ./test.py -f stable/query_after_reset.py # perfbenchmark python3 ./test.py -f perfbenchmark/bug3433.py #python3 ./test.py -f perfbenchmark/bug3589.py - +python3 ./test.py -f perfbenchmark/taosdemoInsert.py #query python3 ./test.py -f query/filter.py diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py new file mode 100644 index 0000000000..59a8143d5a --- /dev/null +++ b/tests/pytest/perfbenchmark/taosdemoInsert.py @@ -0,0 +1,387 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys +import os +import json +import argparse +import subprocess +import datetime +import re + + +from multiprocessing import cpu_count +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode + +class Taosdemo: + def __init__(self, clearCache, dbName, keep): + self.clearCache = clearCache + self.dbname = dbName + self.drop = "yes" + self.keep = keep + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + # self.config = "/etc/taosperf" + # self.conn = taos.connect( + # self.host, + # self.user, + # self.password, + # self.config) + + # env config + def getBuildPath(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getExeToolsDir(self) -> str: + self.debugdir = self.getBuildPath() + "/debug/build/bin" + return self.debugdir + + def getCfgDir(self) -> str: + self.config = self.getBuildPath() + "/sim/dnode1/cfg" + return self.config + + # taodemo insert file config + def dbinfocfg(self) -> dict: + return { + "name": self.dbname, + "drop": self.drop, + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": self.keep, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + def type_check(func): + def wrapper(self, **kwargs): + num_types = ["int", "float", "bigint", "tinyint", "smallint", "double"] + str_types = ["binary", "nchar"] + for k ,v in kwargs.items(): + if k.lower() not in num_types and k.lower() not in str_types: + return f"args {k} type error, not allowed" + elif not isinstance(v, (int, list, tuple)): + return f"value {v} type error, not allowed" + elif k.lower() in num_types and not isinstance(v, int): + return f"arg {v} takes 1 positional argument must be type int " + elif isinstance(v, (list,tuple)) and len(v) > 2: + return f"arg {v} takes from 1 to 2 positional arguments but more than 2 were given " + elif isinstance(v,(list,tuple)) and [ False for _ in v if not isinstance(_, int) ]: + return f"arg {v} takes from 1 to 2 positional arguments must be type int " + else: + pass + return func(self, **kwargs) + return wrapper + + @type_check + def column_tag_count(self, **column_tag) -> list : + init_column_tag = [] + for k, v in column_tag.items(): + if re.search(k, "int, float, bigint, tinyint, smallint, double", re.IGNORECASE): + init_column_tag.append({"type": k, "count": v}) + elif re.search(k, "binary, nchar", re.IGNORECASE): + if isinstance(v, int): + init_column_tag.append({"type": k, "count": v, "len":8}) + elif len(v) == 1: + init_column_tag.append({"type": k, "count": v[0], "len": 8}) + else: + init_column_tag.append({"type": k, "count": v[0], "len": v[1]}) + return init_column_tag + + def stbcfg(self, stb: str, child_tab_count: int, rows: int, prechildtab: str, columns: dict, tags: dict) -> dict: + return { + "name": stb, + "child_table_exists": "no", + "childtable_count": child_tab_count, + "childtable_prefix": prechildtab, + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": rows, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": f"{datetime.datetime.now():%F %X}", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": self.column_tag_count(**columns), + "tags": self.column_tag_count(**tags) + } + + def schemecfg(self,intcount=1,floatcount=0,bcount=0,tcount=0,scount=0,doublecount=0,binarycount=0,ncharcount=0): + return { + "INT": intcount, + "FLOAT": floatcount, + "BIGINT": bcount, + "TINYINT": tcount, + "SMALLINT": scount, + "DOUBLE": doublecount, + "BINARY": binarycount, + "NCHAR": ncharcount + } + + def insertcfg(self,db: dict, stbs: list) -> dict: + return { + "filetype": "insert", + "cfgdir": self.config, + "host": self.host, + "port": 6030, + "user": self.user, + "password": self.password, + "thread_count": cpu_count(), + "thread_count_create_tbl": cpu_count(), + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": db, + "super_tables": stbs + }] + } + + def createinsertfile(self,db: dict, stbs: list) -> str: + date = datetime.datetime.now() + file_create_table = f"/tmp/insert_{date:%F-%H%M}.json" + + with open(file_create_table, 'w') as f: + json.dump(self.insertcfg(db, stbs), f) + + return file_create_table + + # taosdemo query file config + def querysqls(self, sql: str) -> list: + return [{"sql":sql,"result":""}] + + def querycfg(self, sql: str) -> dict: + return { + "filetype": "query", + "cfgdir": self.config, + "host": self.host, + "port": 6030, + "user": self.user, + "password": self.password, + "confirm_parameter_prompt": "yes", + "query_times": 10, + "query_mode": "taosc", + "databases": self.dbname, + "specified_table_query": { + "query_interval": 0, + "concurrent": cpu_count(), + "sqls": self.querysqls(sql) + } + } + + def createqueryfile(self, sql: str): + date = datetime.datetime.now() + file_query_table = f"/tmp/query_{date:%F-%H%M}.json" + + with open(file_query_table,"w") as f: + json.dump(self.querycfg(sql), f) + + return file_query_table + + # Execute taosdemo, and delete temporary files when finished + def taosdemotable(self, filepath: str, resultfile="/dev/null"): + taosdemopath = self.getBuildPath() + "/debug/build/bin" + with open(filepath,"r") as f: + filetype = json.load(f)["filetype"] + if filetype == "insert": + taosdemo_table_cmd = f"{taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1" + else: + taosdemo_table_cmd = f"yes | {taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1" + try: + _ = subprocess.check_output(taosdemo_table_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + def droptmpfile(self, filepath: str): + drop_file_cmd = f"[ -f {filepath} ] && rm -f {filepath}" + try: + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + # TODO:需要完成TD-4153的数据插入和客户端请求的性能查询。 + def td4153insert(self): + + tdLog.printNoPrefix("========== start to create table and insert data ==========") + self.dbname = "td4153" + db = self.dbinfocfg() + stblist = [] + + columntype = self.schemecfg(intcount=1, ncharcount=100) + tagtype = self.schemecfg(intcount=1) + stbname = "stb1" + prechild = "t1" + stable = self.stbcfg( + stb=stbname, + prechildtab=prechild, + child_tab_count=2, + rows=10000, + columns=columntype, + tags=tagtype + ) + stblist.append(stable) + insertfile = self.createinsertfile(db=db, stbs=stblist) + + nmon_file = f"/tmp/insert_{datetime.datetime.now():%F-%H%M}.nmon" + cmd = f"nmon -s5 -F {nmon_file} -m /tmp/" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + self.taosdemotable(insertfile) + self.droptmpfile(insertfile) + self.droptmpfile("/tmp/insert_res.txt") + + # In order to prevent too many performance files from being generated, the nmon file is deleted. + # and the delete statement can be cancelled during the actual test. + self.droptmpfile(nmon_file) + + cmd = f"ps -ef|grep -w nmon| grep -v grep | awk '{{print $2}}'" + try: + time.sleep(10) + _ = subprocess.check_output(cmd,shell=True).decode("utf-8") + except BaseException as e: + raise e + + def td4153query(self): + tdLog.printNoPrefix("========== start to query operation ==========") + + sqls = { + "select_all": "select * from stb1", + "select_join": "select * from t10, t11 where t10.ts=t11.ts" + } + for type, sql in sqls.items(): + result_file = f"/tmp/queryResult_{type}.log" + query_file = self.createqueryfile(sql) + try: + self.taosdemotable(query_file, resultfile=result_file) + except subprocess.CalledProcessError as e: + out_put = e.output + if result_file: + print(f"execute rows {type.split('_')[1]} sql, the sql is: {sql}") + max_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk 'NR==1{{max=$2;next}}{{max=max>$2?max:$2}}END{{print "Max=",max,"s"}}' + ''' + max_sql_time = subprocess.check_output(max_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {max_sql_time}") + + min_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk 'NR==1{{min=$2;next}}{{min=min<$2?min:$2}}END{{print "Min=",min,"s"}}' + ''' + min_sql_time = subprocess.check_output(min_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {min_sql_time}") + + avg_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk '{{sum+=$2}}END{{print "Average=",sum/NR,"s"}}' + ''' + avg_sql_time = subprocess.check_output(avg_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {avg_sql_time}") + + self.droptmpfile(query_file) + self.droptmpfile(result_file) + + drop_query_tmt_file_cmd = " find ./ -name 'querySystemInfo-*' -type f -exec rm {} \; " + try: + _ = subprocess.check_output(drop_query_tmt_file_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + pass + + def td4153(self): + self.td4153insert() + self.td4153query() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + '--remove-cache', + action='store_true', + default=False, + help='clear cache before query (default: False)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='db', + type=str, + help='Database name to be created (default: db)') + parser.add_argument( + '-k', + '--keep-time', + action='store', + default=3650, + type=int, + help='Database keep parameters (default: 3650)') + + args = parser.parse_args() + taosdemo = Taosdemo(args.remove_cache, args.database_name, args.keep_time) + # taosdemo.conn = taos.connect( + # taosdemo.host, + # taosdemo.user, + # taosdemo.password, + # taosdemo.config + # ) + + debugdir = taosdemo.getExeToolsDir() + cfgdir = taosdemo.getCfgDir() + cmd = f"{debugdir}/taosd -c {cfgdir} >/dev/null 2>&1 &" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + if taosdemo.clearCache: + # must be root permission + subprocess.check_output("echo 3 > /proc/sys/vm/drop_caches", shell=True).decode("utf-8") + + taosdemo.td4153() From 9ef6b68030f4162ee502f5e9d57d9a2e74669787 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sat, 22 May 2021 13:53:36 +0800 Subject: [PATCH 2/4] [TD-4263]: update performance test script --- tests/perftest-scripts/perftest-query.sh | 15 +++++++++++---- tests/pytest/insert/insertFromCSVPerformance.py | 2 +- tests/pytest/tools/taosdemoPerformance.py | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index c6d4687ed7..bf62f401bf 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -64,18 +64,25 @@ function runQueryPerfTest { [ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 & echoInfo "Wait TDengine to start" - sleep 300 + sleep 60 echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + mkdir -p /var/lib/perf/ + mkdir -p /var/log/perf/ + rm -rf /var/lib/perf/* + rm -rf /var/log/perf/* + nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 & + echoInfo "Wait TDengine to start" + sleep 10 + echoInfo "Run Performance Test" + cd $WORK_DIR/TDengine/tests/pytest + python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - - #python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT - } diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py index 84fd1d7cca..e1f741bd12 100644 --- a/tests/pytest/insert/insertFromCSVPerformance.py +++ b/tests/pytest/insert/insertFromCSVPerformance.py @@ -31,7 +31,7 @@ class insertFromCSVPerformace: self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" - self.config = "/etc/taosperf" + self.config = "/etc/perf" self.conn = taos.connect( self.host, self.user, diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index a32cba167e..a45393e222 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -24,7 +24,7 @@ class taosdemoPerformace: self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" - self.config = "/etc/taosperf" + self.config = "/etc/perf" self.conn = taos.connect( self.host, self.user, @@ -77,7 +77,7 @@ class taosdemoPerformace: insert_data = { "filetype": "insert", - "cfgdir": "/etc/taosperf", + "cfgdir": "/etc/perf", "host": "127.0.0.1", "port": 6030, "user": "root", From 696c784377cd9a3f4cb095b566b7c378660d646d Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Sat, 22 May 2021 15:55:32 +0800 Subject: [PATCH 3/4] Fix/td 4086 (#6153) * [TD-3086]: clean out unnecessary file in packaging scripts * [TD-4224]: reduce grafana module size in packaging * change * [TD-4228]: error or warn while lost grafanaplugin or go submodule Conflicts: packaging/deb/makedeb.sh packaging/rpm/tdengine.spec packaging/tools/make_install.sh packaging/tools/makeclient.sh packaging/tools/makeclient_power.sh packaging/tools/makepkg.sh packaging/tools/makepkg_power.sh Co-authored-by: Huo Linhe --- packaging/deb/makedeb.sh | 7 +++++- packaging/rpm/tdengine.spec | 7 +++++- packaging/tools/make_install.sh | 12 +++++++++-- packaging/tools/makeclient.sh | 16 ++++++++++---- packaging/tools/makeclient_power.sh | 14 +++++++++--- packaging/tools/makepkg.sh | 33 ++++++++++++++++++++++++++--- packaging/tools/makepkg_power.sh | 13 ++++++++++-- src/connector/go | 2 +- 8 files changed, 87 insertions(+), 17 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 36870b2ebe..28be037e6c 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,7 +58,12 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples -cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_path}/connector +if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then + cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin +else + echo "grafanaplugin bundled directory not found!" + exit 1 +fi cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 92c917cb3d..9910e20bfe 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -66,7 +66,12 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include -cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector +if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then + cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin +else + echo grafanaplugin bundled directory not found! + exit 1 +fi cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index f03065d70c..d6ace0a063 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -243,9 +243,17 @@ function install_data() { } function install_connector() { - ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin ${install_main_dir}/connector + if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then + ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then + ${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 30e9fa51a7..d0eeffc86a 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -117,10 +117,18 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector - cp -r ${connector_dir}/nodejs ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note # cp ${script_dir}/release_note ${install_dir} diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 181536b7f1..b5649e019e 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -144,9 +144,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 36b1fe5bd8..e4d2d71b01 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -114,6 +114,25 @@ mkdir -p ${install_dir}/examples examples_dir="${top_dir}/tests/examples" cp -r ${examples_dir}/c ${install_dir}/examples if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then + rm -rf ${examples_dir}/JDBC/connectionPools/target + fi + if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then + rm -rf ${examples_dir}/JDBC/JDBCDemo/target + fi + if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then + rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target + fi + if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then + rm -rf ${examples_dir}/JDBC/springbootdemo/target + fi + if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then + rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target + fi + if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then + rm -rf ${examples_dir}/JDBC/taosdemo/target + fi + cp -r ${examples_dir}/JDBC ${install_dir}/examples cp -r ${examples_dir}/matlab ${install_dir}/examples cp -r ${examples_dir}/python ${install_dir}/examples @@ -131,9 +150,17 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 554e7884b1..1e43f775e2 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -166,9 +166,18 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ + + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python2/taos/cinterface.py sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/linux/python3/taos/cinterface.py diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..050667e5b4 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f From db03be64060bce26a4ab60b997c9416147501389 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 16:59:19 +0800 Subject: [PATCH 4/4] [TD-4296]: taosdemo sub keepProgress. (#6200) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f9913163b9..02d72fc445 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -391,7 +391,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; - int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int resubAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -4447,6 +4447,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(superQuery, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + ////goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume = 1; + } + // sqls cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); if (!subsqls) { @@ -4478,18 +4490,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - cJSON* superResubAfterConsume = - cJSON_GetObjectItem(sql, "resubAfterConsume"); - if (superResubAfterConsume - && superResubAfterConsume->type == cJSON_Number) { - g_queryInfo.superQueryInfo.resubAfterConsume[j] = - superResubAfterConsume->valueint; - } else if (!superResubAfterConsume) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; - } - cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -6688,6 +6688,8 @@ static void *superSubscribe(void *sarg) { } TAOS_RES* res = NULL; + uint64_t st = 0, et = 0; + while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { @@ -6696,7 +6698,12 @@ static void *superSubscribe(void *sarg) { continue; } + st = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et)); res = taos_consume(tsub[tsubSeq]); + et = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); + if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", @@ -6714,7 +6721,7 @@ static void *superSubscribe(void *sarg) { if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) && (consumed[tsubSeq] >= - g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + g_queryInfo.superQueryInfo.resubAfterConsume)) { printf("keepProgress:%d, resub super table query: %"PRIu64"\n", g_queryInfo.superQueryInfo.subscribeKeepProgress, pThreadInfo->querySeq);