From 114aef8e7eea8f7e7b844bd4d5af7b3764cc122d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 16:27:34 +0800 Subject: [PATCH 1/5] [TD-2295] feature: add test case to support hivemq extension. --- tests/pytest/hivemq-extension-test.py | 166 ++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 tests/pytest/hivemq-extension-test.py diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py new file mode 100644 index 0000000000..22773a6581 --- /dev/null +++ b/tests/pytest/hivemq-extension-test.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ +import sys +import os +import os.path +import time +import glob +import getopt +import subprocess +from shutil import which +from multipledispatch import dispatch + +@dispatch(str, str) +def v_print(msg: str, arg: str): + if verbose: + print(msg % arg) + + +@dispatch(str, int) +def v_print(msg: str, arg: int): + if verbose: + print(msg % int(arg)) + + +@dispatch(str, int, int) +def v_print(msg: str, arg1: int, arg2: int): + if verbose: + print(msg % (int(arg1), int(arg2))) + + +@dispatch(str, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3))) + + +@dispatch(str, int, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + +def isHiveMQInstalled(): + v_print("%s", "Check if HiveMQ installed") + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + if (len(hiveMQDir) == 0): + return False + else: + v_print("HiveMQ installed at %s", hiveMQDir[0]) + return True + +def isMosquittoInstalled(): + v_print("%s", "Check if mosquitto installed") + if not which('mosquitto_pub'): + v_print("%s", "mosquitto is not installed") + return False + else: + return True + +def installExtension(): + currentDir = os.getcwd() + os.chdir('../../src/connector/hivemq-tdengine-extension') + v_print("%s", "build extension..") + os.system('mvn clean package') + + tdExtensionZip = 'target/hivemq-tdengine-extension*.zip' + tdExtensionZipDir = glob.glob(tdExtensionZip) + + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + extPath = hiveMQDir[0] + '/extensions' + + tdExtDir = glob.glob(extPath + '/hivemq-tdengine-extension') + if len(tdExtDir): + v_print("%s", "delete exist extension..") + os.system('rm -rf %s' % tdExtDir[0]) + + v_print("%s", "unzip extension..") + os.system('unzip %s -d %s' % (tdExtensionZipDir[0], extPath)) + + os.chdir(currentDir) + +def stopHiveMQ(): + toBeKilled = "hivemq.jar" + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled + + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + print("hiveMQ processID: %s" % processID) + v_print("%s", "hivemq is NOT running") + +def runHiveMQ(): + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &' + os.system(runPath) + time.sleep(10) + v_print("%s", "hivemq is running") + +def runTDengine(): + pass + +def reCreateDatabase(): + os.system('taos -s "DROP DATABASE IF EXISTS hivemq"') + os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"') + +def sendMqttMsg(topic: str, payload: str): + testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) + os.system(testStr) + time.sleep(3) + +def checkTDengineData(topic: str, payload: str): + output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8') + if (topic in output) and (payload in output): + v_print("%s", output) + return True + else: + v_print("%s", "ERROR: mqtt topic or payload NOT found") + return False + + +if __name__ == "__main__": + verbose = True + testTopic = 'test' + testPayload = 'hello world' + + if not isHiveMQInstalled(): + sys.exit(1) + + if not isMosquittoInstalled(): + sys.exit(1) + + stopHiveMQ() + + installExtension() + + runTDengine() + + reCreateDatabase() + + runHiveMQ() + + sendMqttMsg(testTopic, testPayload) + + if not checkTDengineData(testTopic, testPayload): + sys.exit(1) + + sys.exit(0) From 2558e74bc6ad4c7d93cc1693cae98c0a6eb16f8d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 16:58:57 +0800 Subject: [PATCH 2/5] launch taosd from build directory instead of system. --- tests/pytest/hivemq-extension-test.py | 53 +++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index 22773a6581..daff763f72 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -91,9 +91,8 @@ def installExtension(): os.chdir(currentDir) -def stopHiveMQ(): - toBeKilled = "hivemq.jar" - psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled +def stopProgram(prog: str): + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") @@ -104,7 +103,10 @@ def stopHiveMQ(): time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - print("hiveMQ processID: %s" % processID) + pass + +def stopHiveMQ(): + stopProgram("hivemq.jar") v_print("%s", "hivemq is NOT running") def runHiveMQ(): @@ -115,12 +117,44 @@ def runHiveMQ(): time.sleep(10) v_print("%s", "hivemq is running") +def getBuildPath(): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + def runTDengine(): - pass + stopProgram("taosd") + + buildPath = getBuildPath() + + if (buildPath == ""): + v_print("%s", "taosd NOT found!") + os.exit(1) + else: + v_print("%s", "taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + os.system('%s > /dev/null &' % binPath) + time.sleep(10) def reCreateDatabase(): - os.system('taos -s "DROP DATABASE IF EXISTS hivemq"') - os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"') + buildPath = getBuildPath() + binPath = buildPath + "/build/bin/taos" + + os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath) + os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath) def sendMqttMsg(topic: str, payload: str): testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) @@ -128,7 +162,10 @@ def sendMqttMsg(topic: str, payload: str): time.sleep(3) def checkTDengineData(topic: str, payload: str): - output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8') + buildPath = getBuildPath() + binPath = buildPath + "/build/bin/taos" + + output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8') if (topic in output) and (payload in output): v_print("%s", output) return True From 773101a247d7548aa21f050b4491d8664a272de2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 17:50:09 +0800 Subject: [PATCH 3/5] change to python3 explicitly. --- tests/pytest/hivemq-extension-test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index daff763f72..eb27f7a7d9 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. @@ -55,6 +55,7 @@ def isHiveMQInstalled(): defaultHiveMQPath = "/opt/hivemq*" hiveMQDir = glob.glob(defaultHiveMQPath) if (len(hiveMQDir) == 0): + v_print("%s", "ERROR: hivemq not found!") return False else: v_print("HiveMQ installed at %s", hiveMQDir[0]) From 37d39c9e731cf75d0672ce7cd602e422ecd45a34 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Wed, 2 Dec 2020 03:50:02 +0000 Subject: [PATCH 4/5] [TD-2255]: fix incorrect log variable and sim test cases --- src/mnode/src/mnodeDb.c | 2 +- tests/script/general/db/alter_option.sim | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 39a0c03008..69e8f076e9 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -250,7 +250,7 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { } if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) { - mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep); + mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep2, TSDB_MIN_KEEP, pCfg->daysToKeep); return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index c8aa2480c5..1c3f543ffd 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -115,31 +115,31 @@ if $data7_db != 20,20,20 then return -1 endi -sql alter database db keep 10 -sql show databases -print keep $data7_db -if $data7_db != 20,20,10 then - return -1 -endi - sql alter database db keep 20 sql show databases print keep $data7_db -if $data7_db != 20,20,20 then +if $data7_db != 20,20,20 then return -1 endi sql alter database db keep 30 sql show databases print keep $data7_db -if $data7_db != 20,20,30 then +if $data7_db != 20,20,30 then + return -1 +endi + +sql alter database db keep 40 +sql show databases +print keep $data7_db +if $data7_db != 20,20,40 then return -1 endi sql alter database db keep 40 sql alter database db keep 30 sql alter database db keep 20 -sql alter database db keep 10 +sql_error alter database db keep 10 sql_error alter database db keep 9 sql_error alter database db keep 1 sql alter database db keep 0 @@ -277,4 +277,4 @@ sql_error alter database db prec 'us' print ============== step status sql_error alter database db status 'delete' -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From e9c5859e41703801e35b4dfcfaf8b6ddf9c05353 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 2 Dec 2020 15:48:09 +0800 Subject: [PATCH 5/5] auto pull submodule code and add more check. --- tests/pytest/hivemq-extension-test.py | 69 ++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index eb27f7a7d9..3d0b1ef83f 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -21,6 +21,7 @@ import subprocess from shutil import which from multipledispatch import dispatch + @dispatch(str, str) def v_print(msg: str, arg: str): if verbose: @@ -50,28 +51,34 @@ def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): if verbose: print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + def isHiveMQInstalled(): v_print("%s", "Check if HiveMQ installed") defaultHiveMQPath = "/opt/hivemq*" hiveMQDir = glob.glob(defaultHiveMQPath) if (len(hiveMQDir) == 0): - v_print("%s", "ERROR: hivemq not found!") + v_print("%s", "ERROR: HiveMQ NOT found") return False else: v_print("HiveMQ installed at %s", hiveMQDir[0]) return True + def isMosquittoInstalled(): v_print("%s", "Check if mosquitto installed") if not which('mosquitto_pub'): - v_print("%s", "mosquitto is not installed") + v_print("%s", "ERROR: mosquitto is NOT installed") return False else: return True + def installExtension(): currentDir = os.getcwd() - os.chdir('../../src/connector/hivemq-tdengine-extension') + extDir = 'src/connector/hivemq-tdengine-extension' + os.chdir('../..') + os.system('git submodule update --init -- %s' % extDir) + os.chdir(extDir) v_print("%s", "build extension..") os.system('mvn clean package') @@ -92,6 +99,7 @@ def installExtension(): os.chdir(currentDir) + def stopProgram(prog: str): psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog @@ -106,9 +114,24 @@ def stopProgram(prog: str): psCmd, shell=True).decode("utf-8") pass + def stopHiveMQ(): stopProgram("hivemq.jar") - v_print("%s", "hivemq is NOT running") + v_print("%s", "ERROR: HiveMQ is NOT running") + + +def checkProgramRunning(prog: str): + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog + + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + if not processID: + v_print("ERROR: %s is NOT running", prog) + return False + else: + return True + def runHiveMQ(): defaultHiveMQPath = "/opt/hivemq*" @@ -116,11 +139,19 @@ def runHiveMQ(): runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &' os.system(runPath) time.sleep(10) - v_print("%s", "hivemq is running") + + if not checkProgramRunning("hivemq.jar"): + return False + else: + v_print("%s", "hivemq is running") + return True + def getBuildPath(): selfPath = os.path.dirname(os.path.realpath(__file__)) + binPath = '' + if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] else: @@ -130,9 +161,10 @@ def getBuildPath(): if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + binPath = root[:len(root) - len("/build/bin")] break - return buildPath + return binPath + def runTDengine(): stopProgram("taosd") @@ -140,8 +172,8 @@ def runTDengine(): buildPath = getBuildPath() if (buildPath == ""): - v_print("%s", "taosd NOT found!") - os.exit(1) + v_print("%s", "ERROR: taosd NOT found!") + sys.exit(1) else: v_print("%s", "taosd found in %s" % buildPath) @@ -149,6 +181,13 @@ def runTDengine(): os.system('%s > /dev/null &' % binPath) time.sleep(10) + if not checkProgramRunning("taosd"): + return False + else: + v_print("%s", "TDengine is running") + return True + + def reCreateDatabase(): buildPath = getBuildPath() @@ -157,16 +196,20 @@ def reCreateDatabase(): os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath) os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath) + def sendMqttMsg(topic: str, payload: str): testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) os.system(testStr) time.sleep(3) + def checkTDengineData(topic: str, payload: str): buildPath = getBuildPath() binPath = buildPath + "/build/bin/taos" - output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8') + output = subprocess.check_output( + '%s -s "select * from hivemq.mqtt_payload"' % + binPath, shell=True).decode('utf-8') if (topic in output) and (payload in output): v_print("%s", output) return True @@ -190,11 +233,13 @@ if __name__ == "__main__": installExtension() - runTDengine() + if not runTDengine(): + sys.exit(1) reCreateDatabase() - runHiveMQ() + if not runHiveMQ(): + sys.exit(1) sendMqttMsg(testTopic, testPayload)