From f829bebf68a98dab0a3e9581006f48366e69a3d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 01:40:23 +0000 Subject: [PATCH 01/10] update table lastkey while insert data --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b1de90bc29..7a360c25f9 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -99,6 +99,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pNode) == NULL) { tsdbFreeBytes(pRepo, (void *)pNode, bytes); } else { + if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; pMemTable->numOfRows++; From 540852b1f9f8a9f9a366a03b3b56ad28db7a254a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 22 Jun 2020 09:46:52 +0800 Subject: [PATCH 02/10] fix td-452 --- src/client/src/tscParseInsert.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 69bc4a3e20..4020d201ce 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -46,18 +46,20 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return TK_ILLEGAL; } - int32_t radix = 10; - - int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin - if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) { - radix = radixList[pToken->type - TK_HEX]; - } - errno = 0; - *value = strtoll(pToken->z, endPtr, radix); + *value = strtoll(pToken->z, endPtr, 0); + if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') { + errno = 0; + double v = round(strtod(pToken->z, endPtr)); + if (v > INT64_MAX || v <= INT64_MIN) { + errno = ERANGE; + } else { + *value = v; + } + } // not a valid integer number, return error - if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) { + if (*endPtr - pToken->z != pToken->n) { return TK_ILLEGAL; } @@ -73,11 +75,11 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { *value = strtod(pToken->z, endPtr); // not a valid integer number, return error - if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) { + if ((*endPtr - pToken->z) != pToken->n) { return TK_ILLEGAL; - } else { - return pToken->type; } + + return pToken->type; } int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { From 8f1cee4aada27f9c9e77b9779d7e8dba84865290 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Jun 2020 02:55:48 +0000 Subject: [PATCH 03/10] [TD-715] incorrect handling of error code when creating vnode failed --- src/mnode/src/mnodeVgroup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9553ade3aa..e6759e90db 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -678,8 +678,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->handle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { - mnodeMsg->code = rpcMsg->code; mnodeMsg->successed++; + } else { + mnodeMsg->code = rpcMsg->code; } SVgObj *pVgroup = mnodeMsg->pVgroup; @@ -702,7 +703,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { code = TSDB_CODE_MND_SDB_ERROR; } - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code); } } From e1740422338fc3166aa077b948d3b22dfe1d1e10 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 02:57:01 +0000 Subject: [PATCH 04/10] fix balance coredump error --- src/tsdb/src/tsdbMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ac334e8ed1..92a5895f21 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -213,10 +213,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - strcpy(fname, pFGroup->files[(*index) % 3].fname); + fname = strdup(pFGroup->files[(*index) % 3].fname); } else { if (pFGroup->fileId * 3 + 2 < eindex) { - strcpy(fname, pFGroup->files[0].fname); + fname = strdup(pFGroup->files[0].fname); *index = pFGroup->fileId * 3; } else { tfree(sdup); From 2f341dabf258b061dad8d17fd2dd2cd6cff1737f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 11:05:52 +0800 Subject: [PATCH 05/10] remove coverage test from CI. --- .travis.yml | 86 ----------------------------------------------------- 1 file changed, 86 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcb2e8f22f..bb9ee5d430 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,92 +135,6 @@ matrix: # https://scan.coverity.com/faq#frequency branch_pattern: coverity_scan - - os: linux - dist: bionic - language: c - compiler: gcc - env: ENV_COVER=true - - git: - - depth: 1 - - addons: - apt: - packages: - - build-essential - - cmake - - net-tools - - python-pip - - python-setuptools - - python3-pip - - python3-setuptools - - lcov - - psmisc - - before_script: - - cd ${TRAVIS_BUILD_DIR} - - mkdir debug - - cd debug - - script: - - cmake -DCOVER=true .. > /dev/null - - make > /dev/null - - after_success: - - |- - case $TRAVIS_OS_NAME in - linux) - cd ${TRAVIS_BUILD_DIR}/debug - make install > /dev/null || travis_terminate $? - - pip install numpy - pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ - pip3 install numpy - pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ - - cd ${TRAVIS_BUILD_DIR}/tests - - ./test-all.sh smoke COVER - - TEST_RESULT=$? - - pkill taosd - sleep 1 - - cd ${TRAVIS_BUILD_DIR} - lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info - lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info - lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $? - - gem install coveralls-lcov - - # Color setting - RED='\033[0;31m' - GREEN='\033[1;32m' - GREEN_DARK='\033[0;32m' - GREEN_UNDERLINE='\033[4;32m' - NC='\033[0m' - - coveralls-lcov coverage.info - if [ "$?" -eq "0" ]; then - echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}" - else - echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} " - fi - - bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info - if [ "$?" -eq "0" ]; then - echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} " - else - echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} " - fi - - if [ "$TEST_RESULT" -ne "0" ]; then - travis_terminate $? - fi - ;; - esac - - os: linux dist: trusty language: c From 9f8d29ce11469b8f2263e584f00d03af09cf2b26 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 11:30:34 +0800 Subject: [PATCH 06/10] Add coverage test badge to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c6d92d0bd2..e6da690dab 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine) [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) +[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) From 270e2bb96f48503d89c4c99a28af955fee746aa0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 05:26:24 +0000 Subject: [PATCH 07/10] fix TSDB memory leak --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7a360c25f9..1108340cf0 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -587,6 +587,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } + tfree(dataDir); tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); From 41c7f722a679b0e93143c42631a65395c2615de8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 13:36:24 +0800 Subject: [PATCH 08/10] remove unused os-specified functions. [TD-716] --- src/os/linux/src/linuxPlatform.c | 77 -------------------------------- 1 file changed, 77 deletions(-) diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c index 782a508b17..a8ecf43c46 100644 --- a/src/os/linux/src/linuxPlatform.c +++ b/src/os/linux/src/linuxPlatform.c @@ -56,70 +56,8 @@ void taosMsleep(int mseconds) { bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -void taosResetPthread(pthread_t *thread) { *thread = 0; } - int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } -/* -* Function to get the private ip address of current machine. If get IP -* successfully, return 0, else, return -1. The return values is ip. -* -* Use: -* if (taosGetPrivateIp(ip) != 0) { -* perror("Fail to get private IP address\n"); -* exit(EXIT_FAILURE); -* } -*/ -int taosGetPrivateIp(char *const ip) { - bool hasLoCard = false; - - struct ifaddrs *ifaddr, *ifa; - int family, s; - char host[NI_MAXHOST]; - - if (getifaddrs(&ifaddr) == -1) { - return -1; - } - - /* Walk through linked list, maintaining head pointer so we can free list later */ - int flag = 0; - for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == NULL) continue; - - family = ifa->ifa_addr->sa_family; - if (strcmp("lo", ifa->ifa_name) == 0) { - hasLoCard = true; - continue; - } - - if (family == AF_INET) { - /* printf("%-8s", ifa->ifa_name); */ - s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6), - host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST); - if (s != 0) { - freeifaddrs(ifaddr); - return -1; - } - - strcpy(ip, host); - flag = 1; - break; - } - } - - freeifaddrs(ifaddr); - if (flag) { - return 0; - } else { - if (hasLoCard) { - uPrint("no net card was found, use lo:127.0.0.1 as default"); - strcpy(ip, "127.0.0.1"); - return 0; - } - return -1; - } -} - int taosSetNonblocking(int sock, int on) { int flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { @@ -294,21 +232,6 @@ ssize_t twrite(int fd, void *buf, size_t n) { return n; } -bool taosSkipSocketCheck() { - struct utsname buf; - if (uname(&buf)) { - uPrint("can't fetch os info"); - return false; - } - - if (strstr(buf.release, "Microsoft") != 0) { - uPrint("using WSLv1"); - return true; - } - - return false; -} - void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); From 806f8f5b8cd127fe60e4038a575a915a60d4cccc Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 22 Jun 2020 14:16:46 +0800 Subject: [PATCH 09/10] fix bug: return buffer in stack update boundary testcase. --- src/client/src/tscParseInsert.c | 8 ++++---- tests/pytest/table/boundary.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 4020d201ce..6e922d566b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -989,13 +989,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { - char buf[TSDB_TABLE_ID_LEN] = {0}; - tstrncpy(buf, tblName, sizeof(buf)); + tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN); psTblToken->n = len; psTblToken->type = TK_ID; - psTblToken->z = buf; - tSQLGetToken(buf, &psTblToken->type); + tSQLGetToken(psTblToken->z, &psTblToken->type); return tscValidateName(psTblToken); } @@ -1081,7 +1079,9 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; + char buf[TSDB_TABLE_ID_LEN]; SSQLToken sTblToken; + sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z); diff --git a/tests/pytest/table/boundary.py b/tests/pytest/table/boundary.py index 50586b72ff..a1bf096499 100644 --- a/tests/pytest/table/boundary.py +++ b/tests/pytest/table/boundary.py @@ -141,7 +141,7 @@ class TDTestCase: tdSql.prepare() # 8 bytes for timestamp - maxRowSize = 65535 - 8 + maxRowSize = self.getLimitFromSourceCode('TSDB_MAX_BYTES_PER_ROW') - 8 maxCols = self.getLimitFromSourceCode('TSDB_MAX_COLUMNS') - 1 # for binary cols, 2 bytes are used for length From 10d09614f30102a0e54299ea8168b675458f7c3f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Jun 2020 06:47:13 +0000 Subject: [PATCH 10/10] [TD-437] [TD-718] invalid read while drop db --- src/mnode/src/mnodeAcct.c | 2 +- tests/script/general/stable/refcount.sim | 132 +++++++++++++++++++++++ tests/script/unique/cluster/balance2.sim | 2 +- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/script/general/stable/refcount.sim diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index ad52a4c3ac..419372e06d 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -126,8 +126,8 @@ int32_t mnodeInitAccts() { } void mnodeCleanupAccts() { - sdbCloseTable(tsAcctSdb); acctCleanUp(); + sdbCloseTable(tsAcctSdb); } void *mnodeGetAcct(char *name) { diff --git a/tests/script/general/stable/refcount.sim b/tests/script/general/stable/refcount.sim new file mode 100644 index 0000000000..99d943b4d9 --- /dev/null +++ b/tests/script/general/stable/refcount.sim @@ -0,0 +1,132 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print =============== step1 +sql create database d1; +sql use d1; +sql create table d1.t1 (ts timestamp, i int); +sql create table d1.t2 (ts timestamp, i int); +sql create table d1.t3 (ts timestamp, i int); +sql insert into d1.t1 values(now, 1); +sql insert into d1.t2 values(now, 1); +sql drop table d1.t1; +sql drop database d1; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step2 +sql create database d2; +sql use d2; +sql create table d2.t1 (ts timestamp, i int); +sql create table d2.t2 (ts timestamp, i int); +sql create table d2.t3 (ts timestamp, i int); +sql insert into d2.t1 values(now, 1); +sql insert into d2.t2 values(now, 1); +sql drop table d2.t1; +sql drop table d2.t2; +sql drop table d2.t3; + +sql show d2.tables; +if $rows != 0 then + return -1 +endi + +sql show d2.vgroups; +if $rows != 0 then + return -1 +endi + +sql drop database d2; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step3 +sql create database d3; +sql use d3; +sql create table d3.st (ts timestamp, i int) tags (j int); +sql create table d3.t1 using d3.st tags(1); +sql create table d3.t2 using d3.st tags(1); +sql create table d3.t3 using d3.st tags(1); +sql insert into d3.t1 values(now, 1); +sql drop table d3.t1; +sql drop table d3.t2; +sql drop table d3.t3; + +sql show d3.tables; +if $rows != 0 then + return -1 +endi + +sql show d3.vgroups; +if $rows != 0 then + return -1 +endi + +sql drop database d3; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step4 +sql create database d4; +sql use d4; +sql create table d4.st (ts timestamp, i int) tags (j int); +sql create table d4.t1 using d4.st tags(1); +sql create table d4.t2 using d4.st tags(1); +sql create table d4.t3 using d4.st tags(1); +sql insert into d4.t1 values(now, 1); +sql drop table d4.t1; +sql drop table d4.st; + +sql show d4.tables; +if $rows != 0 then + return -1 +endi + +sql show d4.stables; +if $rows != 0 then + return -1 +endi + +sql drop database d4; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step5 +sql create database d5; +sql use d5; +sql create table d5.st (ts timestamp, i int) tags (j int); +sql create table d5.t1 using d5.st tags(1); +sql create table d5.t2 using d5.st tags(1); +sql create table d5.t3 using d5.st tags(1); +sql insert into d5.t1 values(now, 1); +sql drop table d5.t1; + +sql drop database d5; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step6 +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index 0b3e1374ec..4c4339a78b 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -328,7 +328,7 @@ $x = 0 show6: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes -x show6