From 545be389931031c55b573130dbef624ee9cb1d45 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sat, 19 Dec 2020 13:29:52 +0800 Subject: [PATCH 1/3] [TD-965]: add test case for TWA --- tests/pytest/fulltest.sh | 2 +- tests/pytest/functions/function_twa.py | 62 +++------------- .../pytest/functions/function_twa_restart.py | 71 +++++-------------- tests/pytest/functions/function_twa_test2.py | 4 ++ 4 files changed, 32 insertions(+), 107 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 6d7f6f96ba..fbc4696e9f 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -215,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py diff --git a/tests/pytest/functions/function_twa.py b/tests/pytest/functions/function_twa.py index 1ce4c99b60..b9519a2abc 100644 --- a/tests/pytest/functions/function_twa.py +++ b/tests/pytest/functions/function_twa.py @@ -46,23 +46,17 @@ class TDTestCase: tdSql.error("select twa(ts) from test") tdSql.error("select twa(ts) from test1") - tdSql.error("select twa(col1) from test") - tdSql.error("select twa(col1) from test1") + tdSql.error("select twa(col1) from test") tdSql.error("select twa(col2) from test") - tdSql.error("select twa(col2) from test1") tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col3) from test1") - tdSql.error("select twa(col4) from test") - tdSql.error("select twa(col4) from test1") + tdSql.error("select twa(col4) from test") tdSql.error("select twa(col5) from test") - tdSql.error("select twa(col5) from test1") - tdSql.error("select twa(col6) from test") - tdSql.error("select twa(col6) from test1") + tdSql.error("select twa(col6) from test") tdSql.error("select twa(col7) from test") tdSql.error("select twa(col7) from test1") @@ -72,59 +66,23 @@ class TDTestCase: tdSql.error("select twa(col9) from test") tdSql.error("select twa(col9) from test1") - - tdSql.error("select twa(col1) from test where ts > %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): diff --git a/tests/pytest/functions/function_twa_restart.py b/tests/pytest/functions/function_twa_restart.py index 2025e3c9dd..56242c2953 100644 --- a/tests/pytest/functions/function_twa_restart.py +++ b/tests/pytest/functions/function_twa_restart.py @@ -28,12 +28,17 @@ class TDTestCase: self.ts = 1537146000000 def run(self): - tdSql.execute("use db") + tdSql.prepare() intData = [] floatData = [] + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -41,23 +46,17 @@ class TDTestCase: tdSql.error("select twa(ts) from test") tdSql.error("select twa(ts) from test1") - tdSql.error("select twa(col1) from test") - tdSql.error("select twa(col1) from test1") + tdSql.error("select twa(col1) from test") tdSql.error("select twa(col2) from test") - tdSql.error("select twa(col2) from test1") tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col3) from test1") - tdSql.error("select twa(col4) from test") - tdSql.error("select twa(col4) from test1") + tdSql.error("select twa(col4) from test") tdSql.error("select twa(col5) from test") - tdSql.error("select twa(col5) from test1") - tdSql.error("select twa(col6) from test") - tdSql.error("select twa(col6) from test1") + tdSql.error("select twa(col6) from test") tdSql.error("select twa(col7) from test") tdSql.error("select twa(col7) from test1") @@ -67,59 +66,23 @@ class TDTestCase: tdSql.error("select twa(col9) from test") tdSql.error("select twa(col9) from test1") - - tdSql.error("select twa(col1) from test where ts > %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): @@ -127,4 +90,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/functions/function_twa_test2.py b/tests/pytest/functions/function_twa_test2.py index b5cd24ce71..1ffaa3c628 100644 --- a/tests/pytest/functions/function_twa_test2.py +++ b/tests/pytest/functions/function_twa_test2.py @@ -39,6 +39,10 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 5.5) + tdSql.query("select twa(c) from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 5.5) + tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s)") tdSql.checkRows(10) tdSql.checkData(0, 1, 1.49995) From a55bec788f6ac84de52d19bbb85c9655844dc42d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 21 Dec 2020 15:51:23 +0800 Subject: [PATCH 2/3] TD-2509 --- src/sync/src/syncMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ac79b48606..b43626ce43 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1266,7 +1266,9 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - return TSDB_CODE_SYN_INVALID_VERSION; + if (pNode->replica != 1) { + return TSDB_CODE_SYN_INVALID_VERSION; + } } // always update version From e4821063399ca9bcff3b53931f76bd1bdbcccd0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Dec 2020 02:19:00 +0000 Subject: [PATCH 3/3] adjust more code --- src/tsdb/inc/tsdbMain.h | 19 +++++-------------- src/tsdb/src/tsdbMain.c | 5 ++--- src/tsdb/src/tsdbMemTable.c | 9 ++++----- src/tsdb/src/tsdbMeta.c | 10 ++-------- 4 files changed, 13 insertions(+), 30 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 45eb25ee15..49275541ad 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -66,10 +66,8 @@ typedef struct STable { SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void* eventHandler; // TODO void* streamHandler; // TODO - union { - TSKEY lastKey; - SDataRow lastRow; - }; + TSKEY lastKey; + SDataRow lastRow; char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions @@ -439,16 +437,9 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { } } -static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable, bool cacheLastRow) { - if (cacheLastRow) { - if (pTable->lastRow == NULL) { - return TSKEY_INITIAL_VAL; - } else { - return dataRowKey(pTable->lastRow); - } - } else { - return pTable->lastKey; - } +static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { + ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + return pTable->lastKey; } // ------------------ tsdbBuffer.c diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index cc1ea07554..b34b2fa9e6 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -722,8 +722,9 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; SCompIdx *pIdx = &(rhelper.curCompIdx); - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow); + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); if (pIdx->offset > 0 && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; if (pCfg->cacheLastRow) { // load the block of data if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err; @@ -745,8 +746,6 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, pCol->offset); } - } else { - pTable->lastKey = pIdx->maxKey; } } } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9582c0b49e..0c9d450824 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -664,7 +664,7 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void return -1; } - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow); + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); if (key > lastKey) { tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, REPO_ID(pRepo), key, lastKey); @@ -898,8 +898,9 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { STsdbCfg *pCfg = &pRepo->config; - if (tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow) < dataRowKey(row)) { - if (pCfg->cacheLastRow) { + if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + pTable->lastKey = dataRowKey(row); + if (pCfg->cacheLastRow || pTable->lastRow != NULL) { SDataRow nrow = pTable->lastRow; if (taosTSizeof(nrow) < dataRowLen(row)) { SDataRow orow = nrow; @@ -919,8 +920,6 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow dataRowCpy(nrow, row); TSDB_WUNLOCK_TABLE(pTable); } - } else { - pTable->lastKey = dataRowKey(row); } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 17f5e7052e..7b08178f49 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -663,7 +663,7 @@ static STable *tsdbNewTable() { return NULL; } - // pTable->lastKey = TSKEY_INITIAL_VAL; + pTable->lastKey = TSKEY_INITIAL_VAL; return pTable; } @@ -775,6 +775,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); + taosTZfree(pTable->lastRow); tfree(pTable->sql); free(pTable); } @@ -782,13 +783,6 @@ static void tsdbFreeTable(STable *pTable) { static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock) { STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = &(pRepo->config); - - if (pCfg->cacheLastRow) { - pTable->lastRow = NULL; - } else { - pTable->lastKey = TSKEY_INITIAL_VAL; - } if (lock && tsdbWLockRepoMeta(pRepo) < 0) { tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),