Merge branch '3.0' into 3.0test/jcy

source/dnode/vnode/src/inc/tsdb.h

@@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
 typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
 void    tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
 bool    tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
+bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
 int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
 int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p
   char    fname[TSDB_FILENAME_LEN];
 
   if (pFrom && pTo) {
-    if (pFrom != pTo) {
+    if (!tsdbDelFileIsSame(pFrom, pTo)) {
       tsdbDelFileName(pFS->pTsdb, pFrom, fname);
       if (taosRemoveFile(fname) < 0) {
         code = TAOS_SYSTEM_ERROR(errno);

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty
   }
 }
 
+bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
+
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
   int32_t code = 0;
   int64_t n;

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
 
   tsdbDelFileName(pTsdb, pFile, fname);
   pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
-  if (pDelFReader == NULL) {
+  if (pDelFReader->pReadH == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     taosMemoryFree(pDelFReader);
     goto _err;

tests/system-test/2-query/abs.py

@@ -554,6 +554,9 @@ class TDTestCase:
         tdSql.query("select t1 from stb1 where abs(c1+t1)=1")
         tdSql.checkRows(1)
         tdSql.checkData(0,0,0)
+        
+        tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)")
+        tdSql.checkRows(25)
 
         tdSql.query(
             "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")

tests/system-test/2-query/csum.py

@@ -435,8 +435,8 @@ class TDTestCase:
         tdSql.checkRows(40)
 
         # # bug need fix
-        # tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
+        tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
-        # tdSql.checkRows(4)
+        tdSql.checkRows(4)
         # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ")
 
 

tests/system-test/2-query/last_row.py

@@ -722,10 +722,18 @@ class TDTestCase:
         tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)")
         tdSql.checkRows(11)
 
+
+        tdSql.query("select max(c1) from stb1 interval(50s) sliding(30s)")
+        tdSql.checkRows(13)
+
+        tdSql.query("select unique(c1) from stb1 partition by tbname")
+
         # interval
+
         tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)")
         tdSql.checkRows(27)
 
+
         tdSql.query("select last_row(c1) from testdb.ct1 interval(50s) sliding(30s)")
         tdSql.checkRows(5)
         last_row_result = tdSql.queryResult

tests/system-test/2-query/max_partition.py

@@ -162,10 +162,45 @@ class TDTestCase:
         tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)")
         tdSql.checkRows(self.row_nums*2)
 
+        tdSql.query("select unique(c1) from stb partition  by tbname order by tbname")
+
         tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)")
         tdSql.checkData(0,0,'sub_stb_1')
         tdSql.checkData(0,1,self.row_nums)
 
+        tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1")
+        tdSql.checkRows(72)
+
+        tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1")
+        tdSql.checkRows(72)
+
+        tdSql.query("select c1 , csum(c1) from stb partition by c1")
+        tdSql.checkRows(80)
+
+        tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1")
+        tdSql.checkRows(21)
+        # bug need fix 
+        # tdSql.checkData(0,1,None)
+
+        tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0,1,0.000000000)
+
+        tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0,1,None)
+
+        tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1")
+        tdSql.checkRows(72)
+        # bug need fix 
+        # tdSql.checkData(0,1,None)
+
+
+
+
+
+
+
         # bug need fix 
         # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ")  
         # tdSql.checkRows(5)

tests/system-test/2-query/sample.py

@@ -870,7 +870,10 @@ class TDTestCase:
         tdSql.query("select sample(c1 ,1000) from st")
         tdSql.checkRows(1000)
 
-
+        # bug need fix 
+        tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ")
+        tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ")
+        # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ")
 
     def run(self):
         import traceback

tests/system-test/7-tmq/tmqCommon.py

@@ -205,6 +205,13 @@ class TMQCom:
         tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
         return    
 
+    def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0):
+        for _ in range(count):
+            create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};'
+            ctbStartIdx += 1
+            tdLog.info("drop ctb sql: %s"%create_ctable_sql)
+            tsql.execute(create_ctable_sql)
+
     # schema: (ts timestamp, c1 int, c2 binary(16))
     def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
         tdLog.debug("start to insert data ............")

tests/system-test/7-tmq/tmqUpdate-1ctb.py

@@ -116,7 +116,12 @@ class TDTestCase:
         # paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         consumerId     = 0
-        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2)
+        
+        if self.snapshot == 0:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2))
+        elif self.snapshot == 1:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
+            
         topicList      = topicFromStb1
         ifcheckdata    = 1
         ifManualCommit = 1
@@ -199,7 +204,11 @@ class TDTestCase:
         # paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         consumerId     = 1
-        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        if self.snapshot == 0:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2))
+        elif self.snapshot == 1:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
+        
         topicList      = topicFromStb1
         ifcheckdata    = 1
         ifManualCommit = 1

tests/system-test/fulltest.sh

@@ -185,7 +185,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
 python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
 #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
-#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
+python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
 python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py
 #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py