Merge pull request #14522 from taosdata/feature/TD-11274-3.0

test: add rsma persistence and recovery case

source/dnode/vnode/src/sma/smaCommit.c

@@ -83,8 +83,8 @@ int32_t smaBegin(SSma *pSma) {
 /**
  * @brief pre-commit for rollup sma.
  *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
- *  2) perform persist task for qTaskInfo
+ *  2) wait all triggered fetch tasks finished
- *  3) wait all triggered fetch tasks finished
+ *  3) perform persist task for qTaskInfo
  *
  * @param pSma
  * @return int32_t
@@ -102,10 +102,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
   // step 1: set persistence task paused
   atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
 
-  // step 2: perform persist task for qTaskInfo
+  // step 2: wait all triggered fetch tasks finished
-  tdRSmaPersistExecImpl(pRSmaStat);
-
-  // step 3: wait all triggered fetch tasks finished
   int32_t nLoops = 0;
   while (1) {
     if (T_REF_VAL_GET(pStat) == 0) {
@@ -121,6 +118,9 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
     }
   }
 
+  // step 3: perform persist task for qTaskInfo
+  tdRSmaPersistExecImpl(pRSmaStat);
+
   smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
 
   return TSDB_CODE_SUCCESS;

tests/script/jenkins/basic.txt

@@ -164,6 +164,7 @@
 ./test.sh -f tsim/sma/drop_sma.sim
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
+./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
 
 # --- valgrind
 ./test.sh -f tsim/valgrind/checkError.sim -v

tests/script/tsim/sma/rsmaPersistenceRecovery.sim

@@ -0,0 +1,237 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database with retentions
+sql create database d0 retentions 5s:7d,5m:21d,15m:365d;
+sql use d0
+
+print =============== create super table and register rsma
+sql create table if not exists stb (ts timestamp, c1 int, c2 float) tags (city binary(20),district binary(20)) rollup(max) max_delay 5s,5s watermark 2s,3s;
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags("BeiJing", "ChaoYang");
+
+sql show tables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== insert data and trigger rollup
+sql insert into ct1 values(now, 10, 10.0);
+sql insert into ct1 values(now+1s, 1, 1.0);
+sql insert into ct1 values(now+2s, 100, 100.0);
+
+print =============== wait maxdelay 5+1 seconds for results
+sleep 6000
+
+print =============== select * from retention level 2 from memory
+sql select * from ct1;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 2 file rows $rows > 2
+  return -1
+endi
+
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 2 file result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+print =============== select * from retention level 1 from memory
+sql select * from ct1 where ts > now-8d;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 1 file rows $rows > 2
+  return -1
+endi
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 1 file result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+print =============== select * from retention level 0 from memory
+sql select * from ct1 where ts > now-3d;
+print $data00 $data01 $data02
+print $data10 $data11 $data12
+print $data20 $data21 $data22
+
+if $rows < 1 then
+  print retention level 0 file rows $rows < 1
+  return -1
+endi
+
+if $data01 != 10 then
+  print retention level 0 file result $data01 != 10
+  return -1
+endi
+
+#===================================================================
+
+
+#==================== reboot to trigger commit data to file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+print =============== select * from retention level 2 from file
+sql select * from ct1;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 2 file rows $rows > 2
+  return -1
+endi
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 2 file result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+print =============== select * from retention level 1 from file
+sql select * from ct1 where ts > now-8d;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 1 file rows $rows > 2
+  return -1
+endi
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 1 file result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+print =============== select * from retention level 0 from file
+sql select * from ct1 where ts > now-3d;
+print $data00 $data01 $data02
+print $data10 $data11 $data12
+print $data20 $data21 $data22
+if $rows < 1 then
+  print retention level 0 file rows $rows < 1
+  return -1
+endi
+
+if $data01 != 10 then
+  print retention level 0 file result $data01 != 10
+  return -1
+endi
+
+print =============== insert after rsma qtaskinfo recovery
+sql insert into ct1 values(now, 50, 500.0);
+sql insert into ct1 values(now+1s, 40, 40.0);
+
+print =============== wait maxdelay 5+1 seconds for results
+sleep 6000
+
+print =============== select * from retention level 2 from file and memory after rsma qtaskinfo recovery
+sql select * from ct1;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 2 file/mem rows $rows > 2
+  return -1
+endi
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 2 file/mem result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+if $data02 != 500.00000 then
+  if $data02 != 100.00000 then
+    print retention level 1 file/mem result $data02 != 500.00000 or 100.00000
+  return -1
+  endi
+endi
+
+print =============== select * from retention level 1 from file and memory after rsma qtaskinfo recovery
+sql select * from ct1 where ts > now-8d;
+print $data00 $data01 $data02
+if $rows > 2 then
+  print retention level 1 file/mem rows $rows > 2
+  return -1
+endi
+
+if $data01 != 100 then
+  if $data01 != 10 then
+    print retention level 1 file/mem result $data01 != 100 or 10
+  return -1
+  endi
+endi
+
+if $data02 != 500.00000 then
+  if $data02 != 100.00000 then
+    print retention level 1 file/mem result $data02 != 500.00000 or 100.00000
+  return -1
+  endi
+endi
+
+
+print =============== select * from retention level 0 from file and memory after rsma qtaskinfo recovery
+sql select * from ct1 where ts > now-3d;
+print $data00 $data01 $data02
+print $data10 $data11 $data12
+print $data20 $data21 $data22
+print $data30 $data31 $data32
+print $data40 $data41 $data42
+
+if $rows < 1 then
+  print retention level 0 file/mem rows $rows < 1
+  return -1
+endi
+
+if $data01 != 10 then
+  print retention level 0 file/mem result $data01 != 10
+  return -1
+endi
+
+if $data11 != 1 then
+  print retention level 0 file/mem result $data11 != 1
+  return -1
+endi
+
+if $data21 != 100 then
+  print retention level 0 file/mem result $data21 != 100
+  return -1
+endi
+
+if $data31 != 50 then
+  print retention level 0 file/mem result $data31 != 50
+  return -1
+endi
+
+if $data32 != 500.00000 then
+  print retention level 0 file/mem result $data32 != 500.00000
+  return -1
+endi
+
+
+if $data41 != 40 then
+  print retention level 0 file/mem result $data41 != 40
+  return -1
+endi
+
+if $data42 != 40.00000 then
+  print retention level 0 file/mem result $data42 != 40.00000
+  return -1
+endi
+
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT