Merge branch 'develop' into test

.travis.yml

@@ -135,92 +135,6 @@ matrix:
         #   https://scan.coverity.com/faq#frequency
         branch_pattern: coverity_scan
 
-  - os: linux
-    dist: bionic
-    language: c
-    compiler: gcc
-    env: ENV_COVER=true
-
-    git:
-      - depth: 1
-
-    addons:
-      apt:
-        packages:
-          - build-essential
-          - cmake
-          - net-tools
-          - python-pip
-          - python-setuptools
-          - python3-pip
-          - python3-setuptools
-          - lcov
-          - psmisc
-
-    before_script:
-      - cd ${TRAVIS_BUILD_DIR}
-      - mkdir debug
-      - cd debug
-
-    script:
-      - cmake -DCOVER=true .. > /dev/null
-      - make > /dev/null
-
-    after_success:
-      - |-
-        case $TRAVIS_OS_NAME in
-          linux)
-          cd ${TRAVIS_BUILD_DIR}/debug
-          make install > /dev/null || travis_terminate $?
-
-          pip install numpy
-          pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
-          pip3 install numpy
-          pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
-
-          cd ${TRAVIS_BUILD_DIR}/tests
-
-          ./test-all.sh smoke COVER
-
-          TEST_RESULT=$?
-
-          pkill taosd
-          sleep 1
-
-          cd ${TRAVIS_BUILD_DIR}
-          lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
-          lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info
-          lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $?
-
-          gem install coveralls-lcov
-
-          # Color setting
-          RED='\033[0;31m'
-          GREEN='\033[1;32m'
-          GREEN_DARK='\033[0;32m'
-          GREEN_UNDERLINE='\033[4;32m'
-          NC='\033[0m'
-
-          coveralls-lcov coverage.info
-          if [ "$?" -eq "0" ]; then
-            echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}"
-          else
-            echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} "
-          fi
-
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
-          if [ "$?" -eq "0" ]; then
-            echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
-          else
-            echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
-          fi
-
-          if [ "$TEST_RESULT" -ne "0" ]; then
-            travis_terminate $?
-          fi
-          ;;
-        esac
-
   - os: linux
     dist: trusty
     language: c

README.md

@@ -1,5 +1,6 @@
 [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine)
 [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
+[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
 
 [![TDengine](TDenginelogo.png)](https://www.taosdata.com)
 

src/client/src/tscParseInsert.c

@@ -46,18 +46,20 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
     return TK_ILLEGAL;
   }
   
-  int32_t radix = 10;
-  
-  int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin
-  if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
-    radix = radixList[pToken->type - TK_HEX];
-  }
-
   errno = 0;
-  *value = strtoll(pToken->z, endPtr, radix);
+  *value = strtoll(pToken->z, endPtr, 0);
+  if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') {
+    errno = 0;
+    double v = round(strtod(pToken->z, endPtr));
+    if (v > INT64_MAX || v <= INT64_MIN) {
+      errno = ERANGE;
+    } else {
+      *value = v;
+    }
+  }
   
   // not a valid integer number, return error
-  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+  if (*endPtr - pToken->z != pToken->n) {
     return TK_ILLEGAL;
   }
 
@@ -73,11 +75,11 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
   *value = strtod(pToken->z, endPtr);
   
   // not a valid integer number, return error
-  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+  if ((*endPtr - pToken->z) != pToken->n) {
     return TK_ILLEGAL;
-  } else {
-    return pToken->type;
   }
+
+  return pToken->type;
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -987,13 +989,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 }
 
 int validateTableName(char *tblName, int len, SSQLToken* psTblToken) {
-  char buf[TSDB_TABLE_ID_LEN] = {0};
-  tstrncpy(buf, tblName, sizeof(buf));
+  tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN);
 
   psTblToken->n    = len;
   psTblToken->type = TK_ID;
-  psTblToken->z    = buf;
-  tSQLGetToken(buf, &psTblToken->type);
+  tSQLGetToken(psTblToken->z, &psTblToken->type);
 
   return tscValidateName(psTblToken);
 }
@@ -1079,7 +1079,9 @@ int tsParseInsertSql(SSqlObj *pSql) {
     }
 
     pCmd->curSql = sToken.z;
+    char buf[TSDB_TABLE_ID_LEN];
     SSQLToken sTblToken;
+    sTblToken.z = buf;
     // Check if the table name available or not
     if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
       code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);

src/mnode/src/mnodeAcct.c

@@ -126,8 +126,8 @@ int32_t mnodeInitAccts() {
 }
 
 void mnodeCleanupAccts() {
-  sdbCloseTable(tsAcctSdb);
   acctCleanUp();
+  sdbCloseTable(tsAcctSdb);
 }
 
 void *mnodeGetAcct(char *name) {

src/mnode/src/mnodeVgroup.c

@@ -678,8 +678,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   SMnodeMsg *mnodeMsg = rpcMsg->handle;
   mnodeMsg->received++;
   if (rpcMsg->code == TSDB_CODE_SUCCESS) {
-    mnodeMsg->code = rpcMsg->code;
     mnodeMsg->successed++;
+  } else {
+    mnodeMsg->code = rpcMsg->code;
   }
 
   SVgObj *pVgroup = mnodeMsg->pVgroup;
@@ -702,7 +703,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
       code = TSDB_CODE_MND_SDB_ERROR;
     }
 
-    dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+    dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
   }
 }
 

src/os/linux/src/linuxPlatform.c

@@ -56,70 +56,8 @@ void taosMsleep(int mseconds) {
 
 bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
 
-void taosResetPthread(pthread_t *thread) { *thread = 0; }
-
 int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
 
-/*
-* Function to get the private ip address of current machine. If get IP
-* successfully, return 0, else, return -1. The return values is ip.
-*
-* Use:
-* if (taosGetPrivateIp(ip) != 0) {
-*     perror("Fail to get private IP address\n");
-*     exit(EXIT_FAILURE);
-* }
-*/
-int taosGetPrivateIp(char *const ip) {
-  bool hasLoCard = false;
-
-  struct ifaddrs *ifaddr, *ifa;
-  int             family, s;
-  char            host[NI_MAXHOST];
-
-  if (getifaddrs(&ifaddr) == -1) {
-    return -1;
-  }
-
-  /* Walk through linked list, maintaining head pointer so we can free list later */
-  int flag = 0;
-  for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-    if (ifa->ifa_addr == NULL) continue;
-
-    family = ifa->ifa_addr->sa_family;
-    if (strcmp("lo", ifa->ifa_name) == 0) {
-      hasLoCard = true;
-      continue;
-    }
-
-    if (family == AF_INET) {
-      /* printf("%-8s", ifa->ifa_name); */
-      s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6),
-                      host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-      if (s != 0) {
-        freeifaddrs(ifaddr);
-        return -1;
-      }
-
-      strcpy(ip, host);
-      flag = 1;
-      break;
-    }
-  }
-
-  freeifaddrs(ifaddr);
-  if (flag) {
-    return 0;
-  } else {
-    if (hasLoCard) {
-      uPrint("no net card was found, use lo:127.0.0.1 as default");
-      strcpy(ip, "127.0.0.1");
-      return 0;
-    }
-    return -1;
-  }
-}
-
 int taosSetNonblocking(int sock, int on) {
   int flags = 0;
   if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
@@ -294,21 +232,6 @@ ssize_t twrite(int fd, void *buf, size_t n) {
   return n;
 }
 
-bool taosSkipSocketCheck() {
-  struct utsname buf;
-  if (uname(&buf)) {
-    uPrint("can't fetch os info");
-    return false;
-  }
-
-  if (strstr(buf.release, "Microsoft") != 0) {
-    uPrint("using WSLv1");
-    return true;
-  }
-
-  return false;
-}
-
 void taosBlockSIGPIPE() {
   sigset_t signal_mask;
   sigemptyset(&signal_mask);

src/tsdb/src/tsdbMain.c

@@ -213,10 +213,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
       SFileGroup *pFGroup =
           taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
       if (pFGroup->fileId == fid) {
-        strcpy(fname, pFGroup->files[(*index) % 3].fname);
+        fname = strdup(pFGroup->files[(*index) % 3].fname);
       } else {
         if (pFGroup->fileId * 3 + 2 < eindex) {
-          strcpy(fname, pFGroup->files[0].fname);
+          fname = strdup(pFGroup->files[0].fname);
           *index = pFGroup->fileId * 3;
         } else {
           tfree(sdup);

src/tsdb/src/tsdbMemTable.c

@@ -99,6 +99,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
   if (tSkipListPut(pTableData->pData, pNode) == NULL) {
     tsdbFreeBytes(pRepo, (void *)pNode, bytes);
   } else {
+    if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
     if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
     if (pMemTable->keyLast < key) pMemTable->keyLast = key;
     pMemTable->numOfRows++;
@@ -586,6 +587,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
     goto _err;
   }
 
+  tfree(dataDir);
   tsdbCloseHelperFile(pHelper, 0);
 
   pthread_rwlock_wrlock(&(pFileH->fhlock));

tests/perftest-scripts/perftest-consistent-inserting-data.sh

@@ -39,4 +39,5 @@ function restartTaosd {
 
 buildTDengine
 restartTaosd
-python3 insert/writeDBNonStop.py
+cd /root/TDengine/tests/pytest/insert
+python3 writeDBNonStop.py

tests/pytest/insert/writeDBNonStop.py

@@ -41,13 +41,14 @@ class DBWriteNonStop:
             "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
 
     def writeDataToCSVFile(self, data, duration):
-        csvFile = open('csvFile.csv', 'a', newline='')
+        csvFile = open('/root/csvFile.csv', 'a', newline='')
         writer = csv.writer(csvFile)
         writer.writerow([data[0][0], data[0][1], data[0][2],
                          data[0][3], data[0][4], data[0][5], duration])
         csvFile.close()
 
     def insertData(self):
+        print("===============inserting data===============")
         i = 1
         startTime = datetime.now()
         while True:
@@ -60,7 +61,7 @@ class DBWriteNonStop:
             i += 1
             i = i % 32000000
             endTime = datetime.now()
-            if (endTime - startTime).seconds >= 10 * 2:
+            if (endTime - startTime).seconds >= 3600 * 1:
                 startTime = endTime
                 start = datetime.now()
                 self.cursor.execute(

tests/pytest/table/boundary.py

@@ -141,7 +141,7 @@ class TDTestCase:
         tdSql.prepare()
 
         # 8 bytes for timestamp
-        maxRowSize = 65535 - 8
+        maxRowSize = self.getLimitFromSourceCode('TSDB_MAX_BYTES_PER_ROW') - 8
         maxCols = self.getLimitFromSourceCode('TSDB_MAX_COLUMNS') - 1
 
         # for binary cols, 2 bytes are used for length

tests/script/general/stable/refcount.sim

@@ -0,0 +1,132 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/exec.sh -n dnode1 -s start
+
+sleep 3000
+sql connect
+
+print =============== step1
+sql create database d1;
+sql use d1;
+sql create table d1.t1 (ts timestamp, i int);
+sql create table d1.t2 (ts timestamp, i int);
+sql create table d1.t3 (ts timestamp, i int);
+sql insert into d1.t1 values(now, 1);
+sql insert into d1.t2 values(now, 1);
+sql drop table d1.t1;
+sql drop database d1;
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print =============== step2
+sql create database d2;
+sql use d2;
+sql create table d2.t1 (ts timestamp, i int);
+sql create table d2.t2 (ts timestamp, i int);
+sql create table d2.t3 (ts timestamp, i int);
+sql insert into d2.t1 values(now, 1);
+sql insert into d2.t2 values(now, 1);
+sql drop table d2.t1;
+sql drop table d2.t2;
+sql drop table d2.t3;
+
+sql show d2.tables;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show d2.vgroups;
+if $rows != 0 then 
+  return -1
+endi
+
+sql drop database d2;
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print =============== step3
+sql create database d3;
+sql use d3;
+sql create table d3.st (ts timestamp, i int) tags (j int);
+sql create table d3.t1 using d3.st tags(1);
+sql create table d3.t2 using d3.st tags(1);
+sql create table d3.t3 using d3.st tags(1);
+sql insert into d3.t1 values(now, 1);
+sql drop table d3.t1;
+sql drop table d3.t2;
+sql drop table d3.t3;
+
+sql show d3.tables;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show d3.vgroups;
+if $rows != 0 then 
+  return -1
+endi
+
+sql drop database d3;
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print =============== step4
+sql create database d4;
+sql use d4;
+sql create table d4.st (ts timestamp, i int) tags (j int);
+sql create table d4.t1 using d4.st tags(1);
+sql create table d4.t2 using d4.st tags(1);
+sql create table d4.t3 using d4.st tags(1);
+sql insert into d4.t1 values(now, 1);
+sql drop table d4.t1;
+sql drop table d4.st;
+
+sql show d4.tables;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show d4.stables;
+if $rows != 0 then 
+  return -1
+endi
+
+sql drop database d4;
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print =============== step5
+sql create database d5;
+sql use d5;
+sql create table d5.st (ts timestamp, i int) tags (j int);
+sql create table d5.t1 using d5.st tags(1);
+sql create table d5.t2 using d5.st tags(1);
+sql create table d5.t3 using d5.st tags(1);
+sql insert into d5.t1 values(now, 1);
+sql drop table d5.t1;
+
+sql drop database d5;
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print =============== step6
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/unique/cluster/balance2.sim

@@ -328,7 +328,7 @@ $x = 0
 show6: 
 	$x = $x + 1
 	sleep 2000
-	if $x == 30 then
+	if $x == 10 then
 	  return -1
 	endi
 sql show dnodes -x show6