Feature/sangshuduo/td 3317 taosdemo interlace (#5477)

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

check offset 0.

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

fix sample file import bug.

* [TD-3316] <fix>: add test case for limit and offset. fix sample data issue.

* [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file.

* [TD-3317] <feature>: make taosdemo support interlace mode.

json parameter rows_per_tbl support.

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode insertion.

refactor.

* [TD-3317] <feature>: support interlace mode insertion.

change json file.

* [TD-3317] <feature>: support interlace mode insertion.

fix multithread create table regression.

* [TD-3317] <feature>: support interlace mode insertion.

working but not perfect.

* [TD-3317] <feature>: support interlace mode insertion.

rename lowaTest with taosdemoTestWithJson

* [TD-3317] <feature>: support interlace mode insertion.

perfect

Co-authored-by: Shuduo Sang <sdsang@taosdata.com>

src/kit/taosdemo/insert-interlace.json

@@ -9,8 +9,7 @@
     "thread_count_create_tbl": 4,
     "result_file": "./insert_res.txt",
     "confirm_parameter_prompt": "no",
-    "insert_interval": 5000,
+    "insert_interval": 1000,
-    "rows_per_tbl": 50,
     "num_of_records_per_req": 100,
     "max_sql_len": 1024000,
     "databases": [{
@@ -35,12 +34,12 @@
         "super_tables": [{
             "name": "stb",
             "child_table_exists":"no",
-            "childtable_count": 10,
+            "childtable_count": 100,
             "childtable_prefix": "stb_",
             "auto_create_table": "no",
             "data_source": "rand",
             "insert_mode": "taosc",
-            "insert_rows": 200,
+            "insert_rows": 1000,
             "multi_thread_write_one_tbl": "no",
             "rows_per_tbl": 20,
             "max_sql_len": 1024000,
@@ -51,8 +50,8 @@
             "sample_format": "csv",
             "sample_file": "./sample.csv",
             "tags_file": "",
-            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "columns": [{"type": "INT"}],
-            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+            "tags": [{"type": "TINYINT", "count":1}]
         }]
     }]
 }

src/kit/taosdemo/taosdemo.c

@@ -4444,27 +4444,17 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
   int64_t startTime = pThreadInfo->start_time;
 
   int batchPerTblTimes;
-  int prevBatchPerTbl, lastBatchPerTbl;
+  int batchPerTbl;
 
-  if (pThreadInfo->ntables == 1) {
+  if ((rowsPerTbl > 0) && (pThreadInfo->ntables > 1)) {
-    batchPerTblTimes = 1;
-    lastBatchPerTbl = rowsPerTbl;
-    prevBatchPerTbl = rowsPerTbl;
-  } else if (rowsPerTbl > 0) {
     batchPerTblTimes = g_args.num_of_RPR / rowsPerTbl;
-    lastBatchPerTbl = g_args.num_of_RPR % rowsPerTbl;
+    batchPerTbl = rowsPerTbl;
-
-    if (lastBatchPerTbl > 0)
-      batchPerTblTimes += 1;
-    else
-      lastBatchPerTbl = rowsPerTbl;
-    prevBatchPerTbl = rowsPerTbl;
   } else {
     batchPerTblTimes = 1;
-    prevBatchPerTbl = g_args.num_of_RPR;
+    batchPerTbl = g_args.num_of_RPR;
-    lastBatchPerTbl = g_args.num_of_RPR;
   }
 
+  int generatedRecPerTbl = 0;
   while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
     if (insert_interval) {
         st = taosGetTimestampUs();
@@ -4476,12 +4466,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
     int recGenerated = 0;
 
     for (int i = 0; i < batchPerTblTimes; i ++) {
-        if (insertMode == INTERLACE_INSERT_MODE) {
-          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
-            // turn to first table
-            tableSeq = pThreadInfo->start_table_from;
-          }
-        }
       getTableName(tableName, pThreadInfo, tableSeq);
 
       int headLen;
@@ -4500,15 +4484,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
       pstr += headLen;
       int dataLen = 0;
 
-        int batchPerTbl;
+      printf("%s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
-        if (i == batchPerTblTimes - 1) {
+                __func__, __LINE__, i, batchPerTblTimes, batchPerTbl);
-            batchPerTbl = lastBatchPerTbl;
-        } else {
-            batchPerTbl = prevBatchPerTbl;
-        }
-
-        verbosePrint("%s() LN%d batchPerTbl = %d\n",
-                __func__, __LINE__, batchPerTbl);
       int numOfRecGenerated = generateDataTail(
                 tableName, tableSeq, pThreadInfo, superTblInfo,
                 batchPerTbl, pstr, insertRows, 0,
@@ -4520,10 +4497,25 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
       recGenerated += numOfRecGenerated;
 
       tableSeq ++;
+      if (insertMode == INTERLACE_INSERT_MODE) {
+          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
+            // turn to first table
+            tableSeq = pThreadInfo->start_table_from;
+            generatedRecPerTbl += numOfRecGenerated;
+          }
+      }
+
+      int remainRows = insertRows - generatedRecPerTbl;
+      if (batchPerTbl > remainRows)
+        batchPerTbl = remainRows;
+
+      if ((g_args.num_of_RPR - recGenerated) < batchPerTbl)
+        break;
     }
-      verbosePrint("%s() LN%d buffer:\n%s\n",
-              __func__, __LINE__, buffer);
     pThreadInfo->totalInsertRows += recGenerated;
+    printf("%s() LN%d recGenerated=%d totalInsertRows=%"PRId64" buffer:\n%s\n",
+              __func__, __LINE__, recGenerated, 
+              pThreadInfo->totalInsertRows, buffer);
 
     int affectedRows = execInsert(pThreadInfo, buffer, recGenerated);
     if (affectedRows < 0)