Merge branch 'feature/TD-2354' of https://github.com/taosdata/TDengine into feature/TD-2354

src/sync/src/syncMain.c

@@ -1266,8 +1266,10 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
       }
     }
 
+    if (pNode->replica != 1) {
       return TSDB_CODE_SYN_INVALID_VERSION;
     }
+  }
 
   // always update version
   sTrace("vgId:%d, update version, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,

src/tsdb/inc/tsdbMain.h

@@ -66,10 +66,8 @@ typedef struct STable {
   SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
   void*          eventHandler;   // TODO
   void*          streamHandler;  // TODO
-  union {
   TSKEY          lastKey;
   SDataRow       lastRow;
-  };
   char*          sql;
   void*          cqhandle;
   SRWLatch       latch;  // TODO: implementa latch functions
@@ -439,17 +437,10 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
   }
 }
 
-static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable, bool cacheLastRow) {
+static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
-  if (cacheLastRow) {
+  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
-    if (pTable->lastRow == NULL) {
-      return TSKEY_INITIAL_VAL;
-    } else {
-      return dataRowKey(pTable->lastRow);
-    }
-  } else {
   return pTable->lastKey;
 }
-}
 
 // ------------------ tsdbBuffer.c
 #define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold

src/tsdb/src/tsdbMain.c

@@ -722,8 +722,9 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO
       if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err;
       SCompIdx *pIdx = &(rhelper.curCompIdx);
 
-      TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow);
+      TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable);
       if (pIdx->offset > 0 && lastKey < pIdx->maxKey) {
+        pTable->lastKey = pIdx->maxKey;
         if (pCfg->cacheLastRow) { // load the block of data
           if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err;
 
@@ -745,8 +746,6 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO
             tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes,
                            pCol->offset);
           }
-        } else {
-          pTable->lastKey = pIdx->maxKey;
         }
       }
     }

src/tsdb/src/tsdbMemTable.c

@@ -664,7 +664,7 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void
       return -1;
     }
 
-    TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow);
+    TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable);
     if (key > lastKey) {
       tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
                 REPO_ID(pRepo), key, lastKey);
@@ -898,8 +898,9 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
 static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) {
   STsdbCfg *pCfg = &pRepo->config;
 
-  if (tsdbGetTableLastKeyImpl(pTable, pCfg->cacheLastRow) < dataRowKey(row)) {
+  if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) {
-    if (pCfg->cacheLastRow) {
+    pTable->lastKey = dataRowKey(row);
+    if (pCfg->cacheLastRow || pTable->lastRow != NULL) {
       SDataRow nrow = pTable->lastRow;
       if (taosTSizeof(nrow) < dataRowLen(row)) {
         SDataRow orow = nrow;
@@ -919,8 +920,6 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow
         dataRowCpy(nrow, row);
         TSDB_WUNLOCK_TABLE(pTable);
       }
-    } else {
-      pTable->lastKey = dataRowKey(row);
     }
   }
 

src/tsdb/src/tsdbMeta.c

@@ -663,7 +663,7 @@ static STable *tsdbNewTable() {
     return NULL;
   }
 
-  // pTable->lastKey = TSKEY_INITIAL_VAL;
+  pTable->lastKey = TSKEY_INITIAL_VAL;
 
   return pTable;
 }
@@ -775,6 +775,7 @@ static void tsdbFreeTable(STable *pTable) {
     kvRowFree(pTable->tagVal);
 
     tSkipListDestroy(pTable->pIndex);
+    taosTZfree(pTable->lastRow);
     tfree(pTable->sql);
     free(pTable);
   }
@@ -782,13 +783,6 @@ static void tsdbFreeTable(STable *pTable) {
 
 static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock) {
   STsdbMeta *pMeta = pRepo->tsdbMeta;
-  STsdbCfg * pCfg = &(pRepo->config);
-
-  if (pCfg->cacheLastRow) {
-    pTable->lastRow = NULL;
-  } else {
-    pTable->lastKey = TSKEY_INITIAL_VAL;
-  }
 
   if (lock && tsdbWLockRepoMeta(pRepo) < 0) {
     tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),

tests/pytest/fulltest.sh

@@ -215,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1
 python3 ./test.py -f functions/function_stddev.py -r 1
 python3 ./test.py -f functions/function_sum.py -r 1
 python3 ./test.py -f functions/function_top.py -r 1
-#python3 ./test.py -f functions/function_twa.py -r 1
+python3 ./test.py -f functions/function_twa.py -r 1
 python3 ./test.py -f functions/function_twa_test2.py
 python3 queryCount.py
 python3 ./test.py -f query/queryGroupbyWithInterval.py

tests/pytest/functions/function_twa.py

@@ -47,22 +47,16 @@ class TDTestCase:
         tdSql.error("select twa(ts) from test1")
 
         tdSql.error("select twa(col1) from test")        
-        tdSql.error("select twa(col1) from test1")
 
         tdSql.error("select twa(col2) from test")
-        tdSql.error("select twa(col2) from test1")
 
         tdSql.error("select twa(col3) from test")
-        tdSql.error("select twa(col3) from test1")
 
         tdSql.error("select twa(col4) from test")        
-        tdSql.error("select twa(col4) from test1")
 
         tdSql.error("select twa(col5) from test")
-        tdSql.error("select twa(col5) from test1")
 
         tdSql.error("select twa(col6) from test")                
-        tdSql.error("select twa(col6) from test1")        
 
         tdSql.error("select twa(col7) from test")
         tdSql.error("select twa(col7) from test1")
@@ -73,58 +67,22 @@ class TDTestCase:
         tdSql.error("select twa(col9) from test")
         tdSql.error("select twa(col9) from test1")
                 
-        tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
-        tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
-            
-        tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
-        
-        tdSql.query("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col1) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.error("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col2) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.error("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col3) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.error("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
         tdSql.query("select twa(col4) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.error("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
         tdSql.query("select twa(col5) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
+        tdSql.error("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
         tdSql.query("select twa(col6) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
     def stop(self):

tests/pytest/functions/function_twa_restart.py

@@ -28,12 +28,17 @@ class TDTestCase:
         self.ts = 1537146000000
         
     def run(self):
-        tdSql.execute("use db")
+        tdSql.prepare()
 
         intData = []        
         floatData = []
 
+        tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+                    col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        tdSql.execute("create table test1 using test tags('beijing')")
         for i in range(self.rowNum):
+            tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
             intData.append(i + 1)            
             floatData.append(i + 0.1)                        
 
@@ -42,22 +47,16 @@ class TDTestCase:
         tdSql.error("select twa(ts) from test1")
 
         tdSql.error("select twa(col1) from test")        
-        tdSql.error("select twa(col1) from test1")
 
         tdSql.error("select twa(col2) from test")
-        tdSql.error("select twa(col2) from test1")
 
         tdSql.error("select twa(col3) from test")
-        tdSql.error("select twa(col3) from test1")
 
         tdSql.error("select twa(col4) from test")        
-        tdSql.error("select twa(col4) from test1")
 
         tdSql.error("select twa(col5) from test")
-        tdSql.error("select twa(col5) from test1")
 
         tdSql.error("select twa(col6) from test")                
-        tdSql.error("select twa(col6) from test1")        
 
         tdSql.error("select twa(col7) from test")
         tdSql.error("select twa(col7) from test1")
@@ -68,58 +67,22 @@ class TDTestCase:
         tdSql.error("select twa(col9) from test")
         tdSql.error("select twa(col9) from test1")
                 
-        tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
-        tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
-            
-        tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
-        tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
-
-        tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
-
-        tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
-        tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
-        
-        tdSql.query("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col1) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.error("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col2) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.error("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
         tdSql.query("select twa(col3) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.error("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
         tdSql.query("select twa(col4) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.error("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
         tdSql.query("select twa(col5) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
-        tdSql.query("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
+        tdSql.error("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
         tdSql.query("select twa(col6) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
 
     def stop(self):

tests/pytest/functions/function_twa_test2.py

@@ -39,6 +39,10 @@ class TDTestCase:
         tdSql.checkRows(1)
         tdSql.checkData(0, 0, 5.5)
 
+        tdSql.query("select twa(c) from t1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 5.5)
+
         tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s)")
         tdSql.checkRows(10)
         tdSql.checkData(0, 1, 1.49995)