Merge branch 'develop' into feature/2.0tsdb

src/kit/shell/src/shellLinux.c

@@ -86,7 +86,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         wordfree(&full_path);
         return -1;
       }
-      strcpy(configDir, full_path.we_wordv[0]);
+      tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 's':
@@ -100,7 +100,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->file, full_path.we_wordv[0]);
+      tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'D':
@@ -108,7 +108,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->dir, full_path.we_wordv[0]);
+      tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'T':

src/kit/taosdump/taosdump.c

@@ -655,7 +655,7 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
   }
 
   close(fd);
-  remove(".table.tmp");
+  (void)remove(".table.tmp");
 
   return 0;
 }

src/os/linux/src/linuxSysPara.c

@@ -553,7 +553,7 @@ void taosSetCoreDump() {
   struct rlimit rlim;
   struct rlimit rlim_new;
   if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uPrint("the old unlimited para: rlim_cur=%" PRIu64, ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+    uPrint("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
     rlim_new.rlim_cur = RLIM_INFINITY;
     rlim_new.rlim_max = RLIM_INFINITY;
     if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
@@ -565,7 +565,7 @@ void taosSetCoreDump() {
   }
 
   if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uPrint("the new unlimited para: rlim_cur=%" PRIu64, ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+    uPrint("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
   }
 
 #ifndef _TD_ARM_

src/util/inc/tutil.h

@@ -135,7 +135,7 @@ char* strtolower(char *dst, const char *src);
 
 int64_t strnatoi(char *num, int32_t len);
 
-char* strreplace(const char* str, const char* pattern, const char* rep);
+//char* strreplace(const char* str, const char* pattern, const char* rep);
 
 char *strbetween(char *string, char *begin, char *end);
 

src/util/src/tnote.c

@@ -208,14 +208,15 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
         }
     }
 
-    sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
+    char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
+    sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
     pthread_mutex_init(&pNote->taosNoteMutex, NULL);
 
     umask(0);
-    pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+    pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
     if (pNote->taosNoteFd < 0) {
-        fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno));
+        fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno));
         return -1;
     }
     taosLockNote(pNote->taosNoteFd, pNote);
@@ -223,7 +224,7 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
     // only an estimate for number of lines
     struct stat filestat;
     if (fstat(pNote->taosNoteFd, &filestat) < 0) {
-      fprintf(stderr, "failed to fstat note file:%s reason:%s\n", name, strerror(errno));
+      fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno));
       return -1;
     }    
     size = (int)filestat.st_size;

src/util/src/tutil.c

@@ -314,6 +314,7 @@ int64_t strnatoi(char *num, int32_t len) {
   return ret;
 }
 
+#if 0
 FORCE_INLINE size_t getLen(size_t old, size_t size) {
   if (old == 1) {
     old = 2;
@@ -401,6 +402,7 @@ char *strreplace(const char *str, const char *pattern, const char *rep) {
 
   return dest;
 }
+#endif
 
 char *strbetween(char *string, char *begin, char *end) {
   char *result = NULL;

src/util/tests/stringTest.cpp

@@ -26,6 +26,7 @@ TEST(testCase, string_dequote_test) {
   EXPECT_EQ(3, lx);
 }
 
+#if 0
 TEST(testCase, string_replace_test) {
   char  t3[] = "abc01abc02abc";
   char* ret = strreplace(t3, "abc", "7");
@@ -87,6 +88,7 @@ TEST(testCase, string_replace_test) {
   EXPECT_STREQ("abcdef", ret);
   free(ret);
 }
+#endif
 
 TEST(testCase, string_tolower_test) {
   char t[1024] = {1};

tests/perftest-scripts/perftest-consistent-inserting-data.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+ulimit -c unlimited
+
+function buildTDengine {
+	cd /root/TDengine
+
+	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+		git pull
+
+		LOCAL_COMMIT=`git rev-parse --short @`
+		cd debug
+		rm -rf *
+		cmake ..
+		make > /dev/null
+		make install
+	fi
+}
+
+function restartTaosd {
+	systemctl stop taosd
+	pkill -KILL -x taosd
+	sleep 10
+	
+	rm -rf /var/lib/taos/data/*
+	rm -rf /var/lib/taos/log/*
+	
+	taosd 2>&1 > /dev/null &
+	sleep 10
+}
+
+buildTDengine
+restartTaosd
+cd /root/TDengine/tests/pytest/insert
+python3 writeDBNonStop.py

tests/pytest/client/client.py

@@ -0,0 +1,49 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        ret = tdSql.query('select database()')
+        tdSql.checkData(0, 0, "db")
+
+        ret = tdSql.query('select server_version()')
+        tdSql.checkData(0, 0, "2.0.0.0")
+
+        ret = tdSql.query('select client_version()')
+        tdSql.checkData(0, 0, "2.0.0.0")
+
+        ret = tdSql.query('select server_status()')
+        tdSql.checkData(0, 0, 1)
+
+        ret = tdSql.query('select server_status() as result')
+        tdSql.checkData(0, 0, 1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/fulltest.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 ulimit -c unlimited
 
+python3 ./test.py -f client/client.py
 python3 ./test.py -f insert/basic.py
 python3 ./test.py -f insert/int.py
 python3 ./test.py -f insert/float.py
@@ -142,10 +143,11 @@ python3 ./test.py -f query/filterFloatAndDouble.py
 python3 ./test.py -f query/filterOtherTypes.py
 python3 ./test.py -f query/querySort.py
 python3 ./test.py -f query/queryJoin.py
+python3 ./test.py -f query/select_last_crash.py
 
 #stream
 python3 ./test.py -f stream/stream1.py
 python3 ./test.py -f stream/stream2.py
 
 #alter table
-python3 ./test.py -f alter/alter_table_crash.py
+python3 ./test.py -f alter/alter_table_crash.py

tests/pytest/insert/writeDBNonStop.py

@@ -0,0 +1,82 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+from datetime import datetime
+import csv
+
+
+class DBWriteNonStop:
+    def __init__(self):
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taos"
+
+    def connectDB(self):
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)
+        self.cursor = self.conn.cursor()
+
+    def createTable(self):
+        self.cursor.execute("drop database if exists dbwrite")
+        self.cursor.execute("create database dbwrite")
+        self.cursor.execute("use dbwrite")
+        self.cursor.execute(
+            "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
+
+    def writeDataToCSVFile(self, data, duration):
+        csvFile = open('/root/csvFile.csv', 'a', newline='')
+        writer = csv.writer(csvFile)
+        writer.writerow([data[0][0], data[0][1], data[0][2],
+                         data[0][3], data[0][4], data[0][5], duration])
+        csvFile.close()
+
+    def insertData(self):
+        print("===============inserting data===============")
+        i = 1
+        startTime = datetime.now()
+        while True:
+            self.cursor.execute(
+                "insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" %
+                (i %
+                 10000,
+                 i %
+                 100000))
+            i += 1
+            i = i % 32000000
+            endTime = datetime.now()
+            if (endTime - startTime).seconds >= 3600 * 1:
+                startTime = endTime
+                start = datetime.now()
+                self.cursor.execute(
+                    "select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st")
+                data = self.cursor.fetchall()
+                end = datetime.now()                
+                self.writeDataToCSVFile(data, (end - start).seconds)
+            time.sleep(.001)
+
+    def closeConn(self):
+        self.cursor.close()
+        self.conn.close()
+
+test = DBWriteNonStop()
+test.connectDB()
+test.createTable()
+test.insertData()
+test.closeConn()

tests/pytest/query/select_last_crash.py

@@ -0,0 +1,54 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        
+        self.rowNum = 5000
+        self.ts = 1537146000000
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
+        tdSql.execute(
+            "create table t1 using st tags('dev_001')")
+
+        for i in range(self.rowNum):
+            tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1))
+
+        tdSql.query("select last(*) from st")
+        tdSql.checkRows(1)
+
+        print(
+            "======= Verify filter for %s type finished =========" %
+            curType)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/regressiontest.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 ulimit -c unlimited
 
+python3 ./test.py -f client/client.py
 python3 ./test.py -f insert/basic.py
 python3 ./test.py -f insert/int.py
 python3 ./test.py -f insert/float.py

tests/pytest/smoketest.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 ulimit -c unlimited
 
+# client
+python3 ./test.py $1 -f client/client.py
+python3 ./test.py $1 -s && sleep 1
+
 # insert
 python3 ./test.py $1 -f insert/basic.py
 python3 ./test.py $1 -s && sleep 1

tests/pytest/valgrind-test.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
+
+# client
+PYTHONMALLOC=malloc python3 ./test.py -g -f client/client.py
+PYTHONMALLOC=malloc python3 ./test.py -g -s && sleep 1
+
 # insert
 PYTHONMALLOC=malloc python3 ./test.py -g -f insert/basic.py
 PYTHONMALLOC=malloc python3 ./test.py -g -s && sleep 1