From b4f6e6f9412ce6846139f6b29594760ea5a59ff1 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 9 Jun 2021 18:46:15 +0800 Subject: [PATCH 01/14] [TD-4630]: add exception handling for restful insert --- tests/pytest/insert/restfulInsert.py | 40 +++++++++++++++++++--------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index 6489fd31ff..43dc4bd403 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -40,12 +40,13 @@ class RestfulInsert: if tableID + i >= self.numOfTables : break name = 'beijing' if (tableID + i) % 2 == 0 else 'shanghai' data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) - response = requests.post(self.url, data, headers = self.header) - if response.status_code != 200: - print(response.content) + try: + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + except Exception as e: + print(e) - - def insertData(self, threadID): print("thread %d started" % threadID) tablesPerThread = int (self.numOfTables / self.numOfThreads) @@ -90,10 +91,16 @@ class RestfulInsert: if len(data) > 1024*1024 : print ('batch size is larger than 1M') exit(-1) - response = requests.post(self.url, data, headers = self.header) - if response.status_code != 200: - print(response.content) - + try: + startTime = time.time() + response = requests.post(self.url, data, headers = self.header) + endTime = time.time() + if response.status_code != 200: + print(response.content) + else: + print("inserted %d records, %d seconds" % (bloop, endTime - startTime)) + except Exception as e: + print(e) def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) @@ -119,10 +126,17 @@ class RestfulInsert: else: random.shuffle(values) for k in range(len(values)): - data += values[k] - response = requests.post(self.url, data, headers = self.header) - if response.status_code != 200: - print(response.content) + data += values[k] + try: + startTime = time.time() + response = requests.post(self.url, data, headers = self.header) + endTime = time.time() + if response.status_code != 200: + print(response.content) + else: + print("inserted %d records, %d seconds" % (self.batchSize, endTime - startTime)) + except Exception as e: + print(e) def run(self): data = "create database if not exists %s" % self.dbname From 490caa967ba71b7af0a67675adadeceeb2cfeaa5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 9 Jun 2021 20:31:11 +0800 Subject: [PATCH 02/14] [TD-4599]: fix false dnode offline --- src/balance/src/bnMain.c | 8 ++++---- src/balance/src/bnThread.c | 12 ++++++------ src/mnode/inc/mnodeDef.h | 3 +-- src/mnode/inc/mnodeDnode.h | 2 +- src/mnode/src/mnodeDnode.c | 12 ++++++------ 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 3055f77e81..25f316cb5e 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -405,7 +405,7 @@ void bnReset() { if (pDnode == NULL) break; // while master change, should reset dnode to offline - mInfo("dnode:%d set access:%d to 0", pDnode->dnodeId, pDnode->lastAccess); + mInfo("dnode:%d set access:%" PRId64 " to 0", pDnode->dnodeId, pDnode->lastAccess); pDnode->lastAccess = 0; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; @@ -499,7 +499,7 @@ static bool bnMontiorDropping() { if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; if (mnodeGetDnodesNum() <= 1) continue; - mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId, + mLInfo("dnode:%d, set to removing state for it offline:%" PRId64 " seconds", pDnode->dnodeId, tsAccessSquence - pDnode->lastAccess); pDnode->status = TAOS_DN_STATUS_DROPPING; @@ -574,8 +574,8 @@ void bnCheckStatus() { if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; - mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, - pDnode->lastAccess, pDnode->status); + mInfo("dnode:%d, set to offline state, access seq:%" PRId64 " last seq:%" PRId64 " laststat:%d", pDnode->dnodeId, + tsAccessSquence, pDnode->lastAccess, pDnode->status); bnSetVgroupOffline(pDnode); bnStartTimer(3000); } diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index d07591ecd5..f39b82f2c6 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -101,13 +101,13 @@ static void bnProcessTimer(void *handle, void *tmrId) { if (!sdbIsMaster()) return; if (tsBnThread.stop) return; - tsBnThread.timer = NULL; - tsAccessSquence++; - - bnStartTimer(-1); - bnCheckStatus(); - if (handle == NULL) { + tsBnThread.timer = NULL; + ++tsAccessSquence; + + bnStartTimer(-1); + bnCheckStatus(); + if (tsAccessSquence % tsBalanceInterval == 0) { mDebug("balance function is scheduled by timer"); bnPostSignal(); diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index e052f34a33..c1f2ea7fd7 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -48,9 +48,8 @@ typedef struct SDnodeObj { int32_t dnodeId; int32_t openVnodes; int64_t createdTime; - int32_t resever0; // from dnode status msg, config information + int64_t lastAccess; int32_t customScore; // config by user - uint32_t lastAccess; uint16_t numOfCores; // from dnode status msg uint16_t dnodePort; char dnodeFqdn[TSDB_FQDN_LEN]; diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..d357cd65b8 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,7 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); -extern int32_t tsAccessSquence; +extern int64_t tsAccessSquence; #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 8a5d24c474..fb775d92d8 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,8 +39,8 @@ #include "mnodePeer.h" #include "mnodeCluster.h" -int32_t tsAccessSquence = 0; -int64_t tsDnodeRid = -1; +int64_t tsAccessSquence = 0; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; @@ -567,7 +567,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { mnodeGetClusterId()); return TSDB_CODE_MND_INVALID_CLUSTER_ID; } else { - mTrace("dnode:%d, status received, access times %d openVnodes:%d:%d", pDnode->dnodeId, pDnode->lastAccess, + mTrace("dnode:%d, status received, access times %" PRId64 " openVnodes:%d:%d", pDnode->dnodeId, pDnode->lastAccess, htons(pStatus->openVnodes), pDnode->openVnodes); } } @@ -629,9 +629,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { bnNotify(); } - if (!tsEnableBalance) { - int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes < tsNumOfMnodes) bnNotify(); + int32_t numOfMnodes = mnodeGetMnodesNum(); + if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum()) { + bnNotify(); } if (openVnodes != pDnode->openVnodes) { From 935d41b1f77caff075e2c14d0055e1eb872315f4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Jun 2021 09:53:51 +0800 Subject: [PATCH 03/14] bnThread: dummy commit to make CI happy --- src/balance/src/bnThread.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index f39b82f2c6..b5043c19bb 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -122,8 +122,7 @@ static void bnProcessTimer(void *handle, void *tmrId) { void bnStartTimer(int32_t mseconds) { if (tsBnThread.stop) return; - bool updateSoon = (mseconds != -1); - if (updateSoon) { + if (mseconds != -1) { mTrace("balance function will be called after %d ms", mseconds); taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { @@ -132,5 +131,5 @@ void bnStartTimer(int32_t mseconds) { } void bnNotify() { - bnStartTimer(500); + bnStartTimer(10); } From 96d69b632f3801a878dca81b96fb2eb8d5366c5b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Jun 2021 18:12:13 +0800 Subject: [PATCH 04/14] hotfix/TD-4675 --- src/mnode/src/mnodeSdb.c | 15 +++++++++++---- src/wal/src/walWrite.c | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index a65e29f1ee..d80a4a511d 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -663,10 +663,17 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbMgmt.version + 1) { - pthread_mutex_unlock(&tsSdbMgmt.mutex); - sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); - return TSDB_CODE_SYN_INVALID_VERSION; + if (qtype != TAOS_QTYPE_WAL) { + pthread_mutex_unlock(&tsSdbMgmt.mutex); + sdbError( + "vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, + pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); + return TSDB_CODE_SYN_INVALID_VERSION; + } else { + // If cksum is wrong when recovering wal, use this code + tsSdbMgmt.version = pHead->version; + } + } else { tsSdbMgmt.version = pHead->version; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index ca19a39f2b..ab3fa21938 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -346,7 +346,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } #if defined(WAL_CHECKSUM_WHOLE) - if (pHead->sver == 0 && !walValidateChecksum(pHead)) { + if ((pHead->sver == 0 && !walValidateChecksum(pHead)) || pHead->sver < 0 || pHead->sver > 1) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); From b93129fea59d17de9099dcfb817f749194eb9aa0 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 10 Jun 2021 19:06:30 +0800 Subject: [PATCH 05/14] [TD4610] --- tests/pytest/fulltest.sh | 13 +++- tests/pytest/insert/special_character_show.py | 59 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/insert/special_character_show.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7434d32b9d..ce80c68b25 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -20,6 +20,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +python3 ./test.py -f insert/special_character_show.py python3 ./test.py -f insert/before_1970.py python3 bug2265.py python3 ./test.py -f insert/bug3654.py @@ -38,6 +39,8 @@ python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py +#stable +python3 ./test.py -f stable/insert.py # tag python3 ./test.py -f tag_lite/filter.py @@ -227,7 +230,7 @@ python3 ./test.py -f query/queryFilterTswithDateUnit.py python3 ./test.py -f query/queryTscomputWithNow.py python3 ./test.py -f query/computeErrorinWhere.py python3 ./test.py -f query/queryTsisNull.py - +python3 ./test.py -f query/subqueryFilter.py #stream @@ -254,6 +257,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py @@ -281,6 +286,7 @@ python3 ./test.py -f topic/topicQuery.py python3 ./test.py -f update/merge_commit_data-0.py # wal python3 ./test.py -f wal/addOldWalTest.py +python3 ./test.py -f wal/sdbComp.py # function python3 ./test.py -f functions/all_null_value.py @@ -312,6 +318,7 @@ python3 ./test.py -f query/last_row_cache.py python3 ./test.py -f account/account_create.py python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py +python3 ./test.py -f functions/function_stateWindow.py python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py @@ -333,5 +340,9 @@ python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 ./test.py -f tag_lite/drop_auto_create.py +python3 test.py -f insert/insert_before_use_db.py +python3 test.py -f alter/alter_keep.py +python3 test.py -f alter/alter_cacheLastRow.py python3 test.py -f alter/alter_keep_exception.py +python3 ./test.py -f query/querySession.py #======================p4-end=============== diff --git a/tests/pytest/insert/special_character_show.py b/tests/pytest/insert/special_character_show.py new file mode 100644 index 0000000000..3b2df5c873 --- /dev/null +++ b/tests/pytest/insert/special_character_show.py @@ -0,0 +1,59 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-4584 + + #1 + tdLog.info('=============== step1,create stable') + tdLog.info('create table stb1 (ts timestamp, value double) tags (bin binary(128))') + tdSql.execute('create table stb1 (ts timestamp, value double) tags (bin binary(128))') + + tdLog.info('=============== step2,create table增加了转义字符') + tdLog.info('create table tb1 using stb1 tags("abc\\"def")') + #增加了转义字符\ + tdSql.execute('create table tb1 using stb1 tags("abc\\"def")') + + tdLog.info('=============== step3,insert data') + tdLog.info('insert into tb1 values(now,1.0)') + tdSql.execute('insert into tb1 values(now,1.0)') + + tdLog.info('=============== step4,select table') + tdLog.info('select * from stb1 ') + tdSql.query('select * from stb1 ') + + tdLog.info('=============== step5,check data') + tdSql.checkData(0,2,'abc"def') + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 6b428ce341c0d0baeee1cb8fb902ff142feece1c Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 10 Jun 2021 19:26:23 +0800 Subject: [PATCH 06/14] [TD-i4610] --- tests/pytest/fulltest.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index ce80c68b25..188cb6ad07 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -20,8 +20,8 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -python3 ./test.py -f insert/special_character_show.py python3 ./test.py -f insert/before_1970.py +python3 ./test.py -f insert/special_character_show.py python3 bug2265.py python3 ./test.py -f insert/bug3654.py python3 ./test.py -f insert/insertDynamicColBeforeVal.py @@ -39,8 +39,6 @@ python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py -#stable -python3 ./test.py -f stable/insert.py # tag python3 ./test.py -f tag_lite/filter.py @@ -230,7 +228,7 @@ python3 ./test.py -f query/queryFilterTswithDateUnit.py python3 ./test.py -f query/queryTscomputWithNow.py python3 ./test.py -f query/computeErrorinWhere.py python3 ./test.py -f query/queryTsisNull.py -python3 ./test.py -f query/subqueryFilter.py + #stream @@ -257,8 +255,6 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py @@ -286,7 +282,6 @@ python3 ./test.py -f topic/topicQuery.py python3 ./test.py -f update/merge_commit_data-0.py # wal python3 ./test.py -f wal/addOldWalTest.py -python3 ./test.py -f wal/sdbComp.py # function python3 ./test.py -f functions/all_null_value.py @@ -318,7 +313,6 @@ python3 ./test.py -f query/last_row_cache.py python3 ./test.py -f account/account_create.py python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py -python3 ./test.py -f functions/function_stateWindow.py python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py @@ -340,9 +334,5 @@ python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 ./test.py -f tag_lite/drop_auto_create.py -python3 test.py -f insert/insert_before_use_db.py -python3 test.py -f alter/alter_keep.py -python3 test.py -f alter/alter_cacheLastRow.py python3 test.py -f alter/alter_keep_exception.py -python3 ./test.py -f query/querySession.py #======================p4-end=============== From b0fb8f9dc6af18759eb23888e2ebdb8c5b88e1c1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Jun 2021 13:32:42 +0800 Subject: [PATCH 07/14] balance: remove unused pVgroup->lbTime checking --- src/balance/src/bnMain.c | 4 ---- src/balance/src/bnThread.c | 12 ++++++------ src/mnode/src/mnodeDnode.c | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 25f316cb5e..67741b1473 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -220,10 +220,6 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { } static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { - if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) { - return false; - } - int32_t rmVnodeVer = 0; for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVnode = pVgroup->vnodeGid + i; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index b5043c19bb..44cb24effa 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -101,12 +101,12 @@ static void bnProcessTimer(void *handle, void *tmrId) { if (!sdbIsMaster()) return; if (tsBnThread.stop) return; - if (handle == NULL) { - tsBnThread.timer = NULL; - ++tsAccessSquence; + tsBnThread.timer = NULL; + bnStartTimer(-1); + bnCheckStatus(); - bnStartTimer(-1); - bnCheckStatus(); + if (handle == NULL) { + ++tsAccessSquence; if (tsAccessSquence % tsBalanceInterval == 0) { mDebug("balance function is scheduled by timer"); @@ -131,5 +131,5 @@ void bnStartTimer(int32_t mseconds) { } void bnNotify() { - bnStartTimer(10); + bnStartTimer(500); } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index fb775d92d8..70a63517ca 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -630,7 +630,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum()) { + if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum() && !pDnode->isMgmt) { bnNotify(); } From f253b867cbb4ca2289de4c83ceb6c3eff395aae0 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 11 Jun 2021 16:03:29 +0800 Subject: [PATCH 08/14] [TD-4694]merge tsdbCompactImpl to community version --- src/tsdb/src/tsdbCompact.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 2e8e81b65e..c98e9175b8 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -14,8 +14,6 @@ */ #include "tsdbint.h" -#ifndef _TSDB_PLUGINS - typedef struct { STable * pTable; SBlockIdx * pBlkIdx; @@ -523,5 +521,3 @@ static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCol return 0; } - -#endif \ No newline at end of file From 8dfb92362fbe870a96f4ccc00a7abf086a605b6e Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Jun 2021 16:50:06 +0800 Subject: [PATCH 09/14] [TD-4663]: add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/functions/function_derivative.py | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 tests/pytest/functions/function_derivative.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 68d7bbefee..ee4b932b8c 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -317,6 +317,7 @@ python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py python3 ./test.py -f functions/function_session.py python3 ./test.py -f functions/function_stateWindow.py +python3 ./test.py -f functions/function_derivative.py python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py new file mode 100644 index 0000000000..4ddb9c309b --- /dev/null +++ b/tests/pytest/functions/function_derivative.py @@ -0,0 +1,139 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def insertAndCheckData(self): + types = ["tinyint", "tinyint unsigned", "smallint", "smallint unsigned", "int", "int unsigned", "bigint", "bigint unsigned", "float", "double", "bool", "binary(20)", "nchar(20)"] + + for type in types: + print("============== create table using %s type ================" % type) + tdSql.execute("drop table if exists stb") + tdSql.execute("create table stb(ts timestamp, col %s) tags (id int)" % type) + tdSql.execute("create table tb1 using stb tags(1)") + tdSql.execute("create table tb2 using stb tags(2)") + + if type == "tinyint" or type == "smallint" or type == "int" or type == "bigint": + tdSql.execute("insert into tb1 values(%d, 1)(%d, 11)(%d, 21)" % (self.ts, self.ts + 10000, self.ts + 20000)) + tdSql.execute("insert into tb1 values(%d, -1)(%d, -11)(%d, -21)" % (self.ts + 30000, self.ts + 40000, self.ts + 50000)) + tdSql.execute("insert into tb2 values(%d, 10)(%d, 20)(%d, 30)" % (self.ts + 60000, self.ts + 70000, self.ts + 80000)) + tdSql.execute("insert into tb2 values(%d, -10)(%d, -20)(%d, -30)" % (self.ts + 90000, self.ts + 1000000, self.ts + 1100000)) + + tdSql.execute("insert into tb3 using stb tags(3) values(%d, 10)" % (self.ts + 1200000)) + + tdSql.query("select derivative(col, 1s, 1) from stb group by tbname") + tdSql.checkRows(4) + + tdSql.query("select derivative(col, 10s, 1) from stb group by tbname") + tdSql.checkRows(4) + + tdSql.query("select derivative(col, 10s, 0) from stb group by tbname") + tdSql.checkRows(10) + + tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname") + + tdSql.query("select derivative(col, 10s, 1) from tb1") + tdSql.checkRows(2) + + tdSql.query("select derivative(col, 10s, 0) from tb1") + tdSql.checkRows(5) + + tdSql.query("select derivative(col, 10s, 1) from tb2") + tdSql.checkRows(2) + + tdSql.query("select derivative(col, 10s, 0) from tb2") + tdSql.checkRows(5) + + tdSql.query("select derivative(col, 10s, 0) from tb3") + tdSql.checkRows(0) + + elif type == "tinyint unsigned" or type == "smallint unsigned" or type == "int unsigned" or type == "bigint unsigned": + tdSql.execute("insert into tb1 values(%d, 1)(%d, 11)(%d, 21)" % (self.ts, self.ts + 10000, self.ts + 20000)) + tdSql.execute("insert into tb2 values(%d, 10)(%d, 20)(%d, 30)" % (self.ts + 60000, self.ts + 70000, self.ts + 80000)) + + tdSql.error("select derivative(col, 1s, 1) from tb1") + tdSql.error("select derivative(col, 10s, 0) from tb1") + tdSql.error("select derivative(col, 999ms, 0) from tb1") + tdSql.error("select derivative(col, 1s, 1) from tb2") + tdSql.error("select derivative(col, 10s, 0) from tb2") + tdSql.error("select derivative(col, 999ms, 0) from tb2") + + elif type == "float" or type == "double": + tdSql.execute("insert into tb1 values(%d, 1.0)(%d, 11.0)(%d, 21.0)" % (self.ts, self.ts + 10000, self.ts + 20000)) + tdSql.execute("insert into tb2 values(%d, 3.0)(%d, 4.0)(%d, 5.0)" % (self.ts + 60000, self.ts + 70000, self.ts + 80000)) + + tdSql.query("select derivative(col, 10s, 1) from tb1") + tdSql.checkRows(2) + + tdSql.query("select derivative(col, 10s, 0) from tb1") + tdSql.checkRows(2) + + tdSql.query("select derivative(col, 10s, 1) from tb2") + tdSql.checkRows(2) + + tdSql.query("select derivative(col, 10s, 0) from tb2") + tdSql.checkRows(2) + + elif type == "bool": + tdSql.execute("insert into tb1 values(%d, true)(%d, false)(%d, true)" % (self.ts, self.ts + 10000, self.ts + 20000)) + tdSql.execute("insert into tb2 values(%d, false)(%d, true)(%d, true)" % (self.ts + 60000, self.ts + 70000, self.ts + 80000)) + + tdSql.error("select derivative(col, 1s, 1) from tb1") + tdSql.error("select derivative(col, 10s, 0) from tb1") + tdSql.error("select derivative(col, 999ms, 0) from tb1") + tdSql.error("select derivative(col, 1s, 1) from tb2") + tdSql.error("select derivative(col, 10s, 0) from tb2") + tdSql.error("select derivative(col, 999ms, 0) from tb2") + + else: + tdSql.execute("insert into tb1 values(%d, 'test01')(%d, 'test01')(%d, 'test01')" % (self.ts, self.ts + 10000, self.ts + 20000)) + tdSql.execute("insert into tb2 values(%d, 'test01')(%d, 'test01')(%d, 'test01')" % (self.ts + 60000, self.ts + 70000, self.ts + 80000)) + + tdSql.error("select derivative(col, 1s, 1) from tb1") + tdSql.error("select derivative(col, 10s, 0) from tb1") + tdSql.error("select derivative(col, 999ms, 0) from tb1") + tdSql.error("select derivative(col, 1s, 1) from tb2") + tdSql.error("select derivative(col, 10s, 0) from tb2") + tdSql.error("select derivative(col, 999ms, 0) from tb2") + + tdSql.error("select derivative(col, 10s, 1) from stb") + tdSql.error("select derivative(col, 10s, 1) from stb group by col") + tdSql.error("select derivative(col, 10s, 1) from stb group by id") + tdSql.error("select derivative(col, 999ms, 1) from stb group by id") + tdSql.error("select derivative(col, 10s, 2) from stb group by id") + + def run(self): + tdSql.prepare() + self.insertAndCheckData() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b0f88f44ac6753907f7d681e2aa55cca774ef44f Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 11 Jun 2021 18:53:08 +0800 Subject: [PATCH 10/14] [TD-4694]merge tsdbCompactImpl to community version --- src/tsdb/src/tsdbCompact.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index c98e9175b8..0d2a2bb46f 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -521,3 +521,4 @@ static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCol return 0; } + \ No newline at end of file From 28e3f45c9b516b30b0de71c3a7f244bac354dd01 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 11 Jun 2021 20:00:15 +0800 Subject: [PATCH 11/14] [TD-4694]merge tsdbCompactImpl to community version --- src/tsdb/src/tsdbCompact.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 0d2a2bb46f..c98e9175b8 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -521,4 +521,3 @@ static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCol return 0; } - \ No newline at end of file From bf9171d531b212d2e031d2f3ab3af61c94914147 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 14 Jun 2021 16:33:26 +0800 Subject: [PATCH 12/14] fix doc --- documentation20/en/00.index/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/en/00.index/docs.md b/documentation20/en/00.index/docs.md index f8ebe8c35b..13f8a8565e 100644 --- a/documentation20/en/00.index/docs.md +++ b/documentation20/en/00.index/docs.md @@ -29,7 +29,7 @@ TDengine is a highly efficient platform to store, query, and analyze time-series ## [Data Modeling](/model) -- [Create a Library](/model#create-db): create a library for all data collection points with similar features +- [Create a Database](/model#create-db): create a database for all data collection points with similar features - [Create a Super Table(STable)](/model#create-stable): create a STable for all data collection points with the same type - [Create a Table](/model#create-table): use STable as the template, to create a table for each data collecting point From 3d765f3a32055d8cbe99bf2dc67004bf15b5af33 Mon Sep 17 00:00:00 2001 From: Baosheng Chang Date: Tue, 15 Jun 2021 09:47:01 +0800 Subject: [PATCH 13/14] Update fulltest.sh --- tests/pytest/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 1a7a0b64ee..15880987c8 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -257,8 +257,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py -python3 test.py -f client/change_time_1_1.py -python3 test.py -f client/change_time_1_2.py +#python3 test.py -f client/change_time_1_1.py +#python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py From b79839b5c949e2e8eb84ca773b473b0716c39047 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Mon, 14 Jun 2021 19:21:38 -0700 Subject: [PATCH 14/14] Change pyhon3.8 to python3 for compatability with 3.x (#6489) --- tests/pytest/smoketest.sh | 46 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index 7f7cb2a89e..cbe80882fb 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -2,38 +2,38 @@ ulimit -c unlimited # insert -python3.8 ./test.py $1 -f insert/basic.py -python3.8 ./test.py $1 -s && sleep 1 -python3.8 ./test.py $1 -f insert/bigint.py -python3.8 ./test.py $1 -s && sleep 1 -python3.8 ./test.py $1 -f insert/nchar.py -python3.8 ./test.py $1 -s && sleep 1 -python3.8 ./test.py $1 -f insert/multi.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/basic.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/nchar.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/multi.py +python3 ./test.py $1 -s && sleep 1 # table -python3.8 ./test.py $1 -f table/column_name.py -python3.8 ./test.py $1 -s && sleep 1 -python3.8 ./test.py $1 -f table/column_num.py -python3.8 ./test.py $1 -s && sleep 1 -python3.8 ./test.py $1 -f table/db_table.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f table/column_name.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f table/column_num.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f table/db_table.py +python3 ./test.py $1 -s && sleep 1 # import -python3.8 ./test.py $1 -f import_merge/importDataLastSub.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importDataLastSub.py +python3 ./test.py $1 -s && sleep 1 #tag -python3.8 ./test.py $1 -f tag_lite/filter.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f tag_lite/filter.py +python3 ./test.py $1 -s && sleep 1 #query -python3.8 ./test.py $1 -f query/filter.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f query/filter.py +python3 ./test.py $1 -s && sleep 1 # client -python3.8 ./test.py $1 -f client/client.py -python3.8 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f client/client.py +python3 ./test.py $1 -s && sleep 1 # connector -python3.8 ./test.py $1 -f connector/lua.py +python3 ./test.py $1 -f connector/lua.py