Feature/sangshuduo/td 3416 taosdemo interlace testcase (#5524)

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

check offset 0.

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

fix sample file import bug.

* [TD-3316] <fix>: add test case for limit and offset. fix sample data issue.

* [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file.

* [TD-3317] <feature>: make taosdemo support interlace mode.

json parameter rows_per_tbl support.

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode insertion.

refactor.

* [TD-3317] <feature>: support interlace mode insertion.

change json file.

* [TD-3317] <feature>: support interlace mode insertion.

fix multithread create table regression.

* [TD-3317] <feature>: support interlace mode insertion.

working but not perfect.

* [TD-3317] <feature>: support interlace mode insertion.

rename lowaTest with taosdemoTestWithJson

* [TD-3317] <feature>: support interlace mode insertion.

perfect

* [TD-3317] <feature>: support interlace mode insertion.

cleanup.

* [TD-3317] <feature>: support interlace mode insertion.

adjust algorithm of loop times.

* [TD-3317] <feature>: support interlace mode insertion.

fix delay time bug.

* [TD-3317] <feature>: support interlace mode insertion.

fix progressive timestamp bug.

* [TD-3317] <feature>: support interlace mode insertion.

add an option for performance print.

* [TD-3317] <feature>: support interlace mode insertion.

change json test case with less table for acceleration.

* [TD-3317] <feature>: support interlace mode insertion.

change progressive mode timestamp step and testcase.

* [TD-3197] <fix>: fix taosdemo coverity scan issues.

* [TD-3197] <fix>: fix taosdemo coverity scan issue.

fix subscribeTest pids uninitialized.

* [TD-3317] <feature>: support interlace mode insertion.

add time shift for no sleep time.

* [TD-3317] <feature>: support interlace insert.

rework timestamp.

* [TD-3416] <feature>: add taosdemo interlace testcase.

Co-authored-by: Shuduo Sang <sdsang@taosdata.com>

src/kit/taosdemo/taosdemo.c

@@ -4287,13 +4287,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
       if ((g_args.disorderRatio != 0)
                 && (rand_num < g_args.disorderRange)) {
              
-        int64_t d = startTime - taosRandom() % 1000000 + rand_num;
+        int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k 
+            - taosRandom() % 1000000 + rand_num;
         len = generateData(data, data_type,
                   ncols_per_record, d, lenOfBinary);
       } else {
             len = generateData(data, data_type,
                   ncols_per_record,
-                  startTime + DEFAULT_TIMESTAMP_STEP * startFrom,
+                  startTime + DEFAULT_TIMESTAMP_STEP * k,
                   lenOfBinary);
       }
 
@@ -4402,7 +4403,8 @@ static int generateDataBuffer(char *pTblName,
   int k;
   int dataLen;
   k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
-          g_args.num_of_RPR, pstr, insertRows, startFrom, startTime,
+          g_args.num_of_RPR, pstr, insertRows, startFrom,
+          startTime,
           pSamplePos, &dataLen);
   return k;
 }
@@ -4475,7 +4477,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
   int generatedRecPerTbl = 0;
   bool flagSleep = true;
   int sleepTimeTotal = 0;
-  int timeShift = 0;
   while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
     if ((flagSleep) && (insert_interval)) {
         st = taosGetTimestampUs();
@@ -4513,16 +4514,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
       generateDataTail(
         tableName, tableSeq, pThreadInfo, superTblInfo,
         batchPerTbl, pstr, insertRows, 0,
-        startTime + timeShift + sleepTimeTotal,
+        startTime,
         &(pThreadInfo->samplePos), &dataLen);
+
       pstr += dataLen;
       recOfBatch += batchPerTbl;
+      startTime += batchPerTbl * superTblInfo->timeStampStep;
       pThreadInfo->totalInsertRows += batchPerTbl;
+
       verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
                 pThreadInfo->threadID, __func__, __LINE__,
                 batchPerTbl, recOfBatch);
 
-      timeShift ++;
       tableSeq ++;
       if (insertMode == INTERLACE_INSERT_MODE) {
           if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
@@ -4668,13 +4671,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
 
       int generated = generateDataBuffer(
               tableName, tableSeq, pThreadInfo, buffer, insertRows,
-            i, start_time + pThreadInfo->totalInsertRows * timeStampStep,
+            i, start_time,
             &(pThreadInfo->samplePos));
       if (generated > 0)
         i += generated;
       else
         goto free_and_statistics_2;
 
+      start_time +=  generated * timeStampStep;
       pThreadInfo->totalInsertRows += generated;
 
       startTs = taosGetTimestampUs();

tests/pytest/fulltest.sh

@@ -238,6 +238,7 @@ python3 test.py -f tools/taosdemoTestLimitOffset.py
 python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/taosdemoTest2.py
 python3 test.py -f tools/taosdemoTestSampleData.py
+python3 test.py -f tools/taosdemoTestInterlace.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py

tests/pytest/tools/taosdemoTestInterlace.py

@@ -17,6 +17,7 @@ from util.log import *
 from util.cases import *
 from util.sql import *
 from util.dnodes import *
+import subprocess
 
 
 class TDTestCase:
@@ -39,7 +40,7 @@ class TDTestCase:
             if ("taosd" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
-                    buildPath = root[:len(root)-len("/build/bin")]
+                    buildPath = root[:len(root) - len("/build/bin")]
                     break
         return buildPath
 
@@ -50,14 +51,23 @@ class TDTestCase:
             tdLog.exit("taosd not found!")
         else:
             tdLog.info("taosd found in %s" % buildPath)
-        binPath = buildPath+ "/build/bin/"
+        binPath = buildPath + "/build/bin/"
-        os.system("%staosdemo -f tools/insert-interlace.json" % binPath)
+        taosdemoCmd = "%staosdemo -f tools/insert-interlace.json -pp 2>&1 | grep sleep | wc -l" % binPath
+        sleepTimes = subprocess.check_output(
+            taosdemoCmd, shell=True).decode("utf-8")
+        print("sleep times: %d" % int(sleepTimes))
+
+        if (int(sleepTimes) != 16):
+            caller = inspect.getframeinfo(inspect.stack()[0][0])
+            tdLog.exit(
+                "%s(%d) failed: expected sleep times 16, actual %d" %
+                (caller.filename, caller.lineno, int(sleepTimes)))
 
         tdSql.execute("use db")
         tdSql.query("select count(tbname) from db.stb")
-        tdSql.checkData(0, 0, 100)
+        tdSql.checkData(0, 0, 9)
         tdSql.query("select count(*) from db.stb")
-        tdSql.checkData(0, 0, 33000)
+        tdSql.checkData(0, 0, 2250)
 
     def stop(self):
         tdSql.close()