From 98a8751cb74c5989392aa36b58a29b785569bdd1 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 15 Jun 2020 15:18:22 +0800 Subject: [PATCH 1/6] continuously write tsdb --- tests/pytest/query/querytest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/pytest/query/querytest.py diff --git a/tests/pytest/query/querytest.py b/tests/pytest/query/querytest.py new file mode 100644 index 0000000000..d15e82ff9c --- /dev/null +++ b/tests/pytest/query/querytest.py @@ -0,0 +1,30 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import time + +conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") +c1 = conn.cursor() + +c1.execute('create database db') +c1.execute('use db') + +c1.execute('create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))') + +i = 1 + +while True: + c1.execute("insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" % (i % 10000, i % 100000)) + i += 1 + i = i % 32000000 \ No newline at end of file From c4812f91a91e53287f6e8f4f224d6c28a98c89f9 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 Jun 2020 15:16:43 +0800 Subject: [PATCH 2/6] [TD-632]: keep inserting data into database --- tests/pytest/insert/writeDBNonStop.py | 81 +++++++++++++++++++++++++++ tests/pytest/insert/writeDBNonStop.sh | 42 ++++++++++++++ tests/pytest/query/querytest.py | 57 ------------------- 3 files changed, 123 insertions(+), 57 deletions(-) create mode 100644 tests/pytest/insert/writeDBNonStop.py create mode 100644 tests/pytest/insert/writeDBNonStop.sh delete mode 100644 tests/pytest/query/querytest.py diff --git a/tests/pytest/insert/writeDBNonStop.py b/tests/pytest/insert/writeDBNonStop.py new file mode 100644 index 0000000000..0af21bff85 --- /dev/null +++ b/tests/pytest/insert/writeDBNonStop.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +from datetime import datetime +import csv + + +class DBWriteNonStop: + def __init__(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos" + + def connectDB(self): + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + self.cursor = self.conn.cursor() + + def createTable(self): + self.cursor.execute("drop database if exists dbwrite") + self.cursor.execute("create database dbwrite") + self.cursor.execute("use dbwrite") + self.cursor.execute( + "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))") + + def writeDataToCSVFile(self, data, duration): + csvFile = open('csvFile.csv', 'a', newline='') + writer = csv.writer(csvFile) + writer.writerow([data[0][0], data[0][1], data[0][2], + data[0][3], data[0][4], data[0][5], duration]) + csvFile.close() + + def insertData(self): + i = 1 + startTime = datetime.now() + while True: + self.cursor.execute( + "insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" % + (i % + 10000, + i % + 100000)) + i += 1 + i = i % 32000000 + endTime = datetime.now() + if (endTime - startTime).seconds >= 10 * 2: + startTime = endTime + start = datetime.now() + self.cursor.execute( + "select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st") + data = self.cursor.fetchall() + end = datetime.now() + self.writeDataToCSVFile(data, (end - start).seconds) + time.sleep(.001) + + def closeConn(self): + self.cursor.close() + self.conn.close() + +test = DBWriteNonStop() +test.connectDB() +test.createTable() +test.insertData() +test.closeConn() \ No newline at end of file diff --git a/tests/pytest/insert/writeDBNonStop.sh b/tests/pytest/insert/writeDBNonStop.sh new file mode 100644 index 0000000000..57e6a93efd --- /dev/null +++ b/tests/pytest/insert/writeDBNonStop.sh @@ -0,0 +1,42 @@ +#!/bin/bash +ulimit -c unlimited + +function buildTDengine { + cd /root/TDengine + + git remote update + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git pull + + LOCAL_COMMIT=`git rev-parse --short @` + cd debug + rm -rf * + cmake .. + make > /dev/null + make install + fi +} + +function restartTaosd { + systemctl stop taosd + pkill -KILL -x taosd + sleep 10 + + rm -rf /var/lib/taos/data/* + rm -rf /var/lib/taos/log/* + + taosd 2>&1 > /dev/null & + sleep 10 +} + +buildTDengine +restartTaosd +python3 insert/writeDBNonStop.py \ No newline at end of file diff --git a/tests/pytest/query/querytest.py b/tests/pytest/query/querytest.py deleted file mode 100644 index d506888448..0000000000 --- a/tests/pytest/query/querytest.py +++ /dev/null @@ -1,57 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import taos -import time -from datetime import datetime - -class DBWriteNonStop: - def __init__(self): - self.host = "127.0.0.1" - self.user = "root" - self.password = "taosdata" - self.config = "/etc/taos" - - def connectDB(self): - self.conn = taos.connect(self.host, self.user, self.password, self.config) - self.cursor = self.conn.cursor() - - def createTable(self): - self.cursor.execute("drop database if exists dbwrite") - self.cursor.execute("create database dbwrite") - self.cursor.execute("use dbwrite") - self.cursor.execute("create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))") - - def insertData(self): - i = 1 - startTime = datetime.now() - while True: - self.cursor.execute("insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" % (i % 10000, i % 100000)) - i += 1 - i = i % 32000000 - endTime = datetime.now() - if (endTime - startTime).seconds >= 5 * 2: - startTime = endTime - self.cursor.execute("select last(ts) from st >> output.txt") - self.cursor.execute("select count(*) from st >> output.txt") - time.sleep(.001) - - def closeConn(self): - self.cursor.close() - self.conn.close() - -test = DBWriteNonStop() -test.connectDB() -test.createTable() -test.insertData() -test.closeConn() \ No newline at end of file From 4d8fb3b91b2b161565a35eca3329b84df9d5ebe6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 18:41:50 +0800 Subject: [PATCH 3/6] add client/client.py to test client functions. [TD-725] --- tests/pytest/client/client.py | 38 ++++++++++++++++++++++++++++++++++ tests/pytest/fulltest.sh | 3 ++- tests/pytest/regressiontest.sh | 1 + tests/pytest/smoketest.sh | 4 ++++ tests/pytest/valgrind-test.sh | 5 +++++ 5 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/client/client.py diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py new file mode 100644 index 0000000000..b3a93bd363 --- /dev/null +++ b/tests/pytest/client/client.py @@ -0,0 +1,38 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + ret = tdSql.query('select database()') + + tdSql.checkData(0, 0, "db") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 0bacd63ec1..37f436ef78 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -1,6 +1,7 @@ #!/bin/bash ulimit -c unlimited +python3 ./test.py -f client/client.py python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py python3 ./test.py -f insert/float.py @@ -148,4 +149,4 @@ python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #alter table -python3 ./test.py -f alter/alter_table_crash.py \ No newline at end of file +python3 ./test.py -f alter/alter_table_crash.py diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 955d6aa2ef..0c248f3387 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -1,6 +1,7 @@ #!/bin/bash ulimit -c unlimited +python3 ./test.py -f client/client.py python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py python3 ./test.py -f insert/float.py diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index 1c51da397e..c56a4726dd 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -1,6 +1,10 @@ #!/bin/bash ulimit -c unlimited +# client +python3 ./test.py $1 -f client/client.py +python3 ./test.py $1 -s && sleep 1 + # insert python3 ./test.py $1 -f insert/basic.py python3 ./test.py $1 -s && sleep 1 diff --git a/tests/pytest/valgrind-test.sh b/tests/pytest/valgrind-test.sh index bf42cd59cd..9dc9de1b15 100755 --- a/tests/pytest/valgrind-test.sh +++ b/tests/pytest/valgrind-test.sh @@ -1,4 +1,9 @@ #!/bin/bash + +# client +PYTHONMALLOC=malloc python3 ./test.py -g -f client/client.py +PYTHONMALLOC=malloc python3 ./test.py -g -s && sleep 1 + # insert PYTHONMALLOC=malloc python3 ./test.py -g -f insert/basic.py PYTHONMALLOC=malloc python3 ./test.py -g -s && sleep 1 From 6e4545f0b7c1ae2faf3eac346b2b8ae798bc2d5c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 19:15:08 +0800 Subject: [PATCH 4/6] test more client functions. --- tests/pytest/client/client.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py index b3a93bd363..21df7e6a86 100644 --- a/tests/pytest/client/client.py +++ b/tests/pytest/client/client.py @@ -26,9 +26,20 @@ class TDTestCase: tdSql.prepare() ret = tdSql.query('select database()') - tdSql.checkData(0, 0, "db") + ret = tdSql.query('select server_version()') + tdSql.checkData(0, 0, "2.0.0.0") + + ret = tdSql.query('select client_version()') + tdSql.checkData(0, 0, "2.0.0.0") + + ret = tdSql.query('select server_status()') + tdSql.checkData(0, 0, 1) + + ret = tdSql.query('select server_status() as result') + tdSql.checkData(0, 0, 1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 3588f39307031e0d8024c99067acfd43cbe2725f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 Jun 2020 19:32:32 +0800 Subject: [PATCH 5/6] TD-728: add test case to reproduce select last crash --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/select_last_crash.py | 54 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/pytest/query/select_last_crash.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 0bacd63ec1..c042c457a1 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -142,6 +142,7 @@ python3 ./test.py -f query/filterFloatAndDouble.py python3 ./test.py -f query/filterOtherTypes.py python3 ./test.py -f query/querySort.py python3 ./test.py -f query/queryJoin.py +python3 ./test.py -f query/select_last_crash.py #stream python3 ./test.py -f stream/stream1.py diff --git a/tests/pytest/query/select_last_crash.py b/tests/pytest/query/select_last_crash.py new file mode 100644 index 0000000000..9aeb122f82 --- /dev/null +++ b/tests/pytest/query/select_last_crash.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + + self.rowNum = 5000 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute( + "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))") + tdSql.execute( + "create table t1 using st tags('dev_001')") + + for i in range(self.rowNum): + tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1)) + + tdSql.query("select last(*) from st") + tdSql.checkRows(1) + + print( + "======= Verify filter for %s type finished =========" % + curType) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b25c36c35efcb6175e837677ac357014570362bc Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 22 Jun 2020 19:46:03 +0800 Subject: [PATCH 6/6] [modify for covrity scan] --- src/kit/shell/src/shellLinux.c | 6 +++--- src/kit/taosdump/taosdump.c | 2 +- src/os/linux/src/linuxSysPara.c | 4 ++-- src/util/inc/tutil.h | 2 +- src/util/src/tnote.c | 9 +++++---- src/util/src/tutil.c | 2 ++ src/util/tests/stringTest.cpp | 2 ++ 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 9d4de06b0f..94f3901dd8 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -86,7 +86,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { wordfree(&full_path); return -1; } - strcpy(configDir, full_path.we_wordv[0]); + tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; case 's': @@ -100,7 +100,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - strcpy(arguments->file, full_path.we_wordv[0]); + tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; case 'D': @@ -108,7 +108,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - strcpy(arguments->dir, full_path.we_wordv[0]); + tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; case 'T': diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index dea05e1c2a..678de7daa7 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -655,7 +655,7 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { } close(fd); - remove(".table.tmp"); + (void)remove(".table.tmp"); return 0; } diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index 04fa6b1f56..a1d013fa72 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -553,7 +553,7 @@ void taosSetCoreDump() { struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uPrint("the old unlimited para: rlim_cur=%" PRIu64, ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + uPrint("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -565,7 +565,7 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uPrint("the new unlimited para: rlim_cur=%" PRIu64, ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + uPrint("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); } #ifndef _TD_ARM_ diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 94084b5638..23d8348cbe 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -135,7 +135,7 @@ char* strtolower(char *dst, const char *src); int64_t strnatoi(char *num, int32_t len); -char* strreplace(const char* str, const char* pattern, const char* rep); +//char* strreplace(const char* str, const char* pattern, const char* rep); char *strbetween(char *string, char *begin, char *end); diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index aa61898543..12a7fc2b9b 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -208,14 +208,15 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf } } - sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); + char noteName[NOTE_FILE_NAME_LEN * 2] = "\0"; + sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); pthread_mutex_init(&pNote->taosNoteMutex, NULL); umask(0); - pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (pNote->taosNoteFd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno)); + fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); return -1; } taosLockNote(pNote->taosNoteFd, pNote); @@ -223,7 +224,7 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf // only an estimate for number of lines struct stat filestat; if (fstat(pNote->taosNoteFd, &filestat) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", name, strerror(errno)); + fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); return -1; } size = (int)filestat.st_size; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index ccdf7111a7..2bfef63f84 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -314,6 +314,7 @@ int64_t strnatoi(char *num, int32_t len) { return ret; } +#if 0 FORCE_INLINE size_t getLen(size_t old, size_t size) { if (old == 1) { old = 2; @@ -401,6 +402,7 @@ char *strreplace(const char *str, const char *pattern, const char *rep) { return dest; } +#endif char *strbetween(char *string, char *begin, char *end) { char *result = NULL; diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp index 0dc1ee7a77..95fba0cd3e 100644 --- a/src/util/tests/stringTest.cpp +++ b/src/util/tests/stringTest.cpp @@ -26,6 +26,7 @@ TEST(testCase, string_dequote_test) { EXPECT_EQ(3, lx); } +#if 0 TEST(testCase, string_replace_test) { char t3[] = "abc01abc02abc"; char* ret = strreplace(t3, "abc", "7"); @@ -87,6 +88,7 @@ TEST(testCase, string_replace_test) { EXPECT_STREQ("abcdef", ret); free(ret); } +#endif TEST(testCase, string_tolower_test) { char t[1024] = {1};