Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5659)

* [TD-3607] <fix>: fix taosdemo buffer overflow. * [TD-3607] <fix>: taosdemo buffer overflow. add tmp buffer. * [TD-3607] <fix>: taosdemo buffer overflow. fix data generation. * [TD-3607] <fix>: taosdemo buffer overflow. fix normal table writting. * [TD-3607] <fix>: taosdemo buffer overflow. remove tail spaces. * [TD-3607] <fix>: taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] <fix>: taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] <fix>: taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] <fix>: taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] <fix>: taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] <fix>: fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] <fix>: taosdemo buffer overflow. adjust sample data test case. Co-authored-by: Shuduo Sang <sdsang@taosdata.com>
2021-04-02 19:22:04 +08:00 · 2021-04-02 19:22:04 +08:00 · e1739b86a9
parent 251f20164f
commit e1739b86a9
8 changed files with 224 additions and 90 deletions
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@ -2433,82 +2433,82 @@ static int createDatabasesAndStables() {
        taos_close(taos);
        return -1;
      }
-    }

-    int dataLen = 0;
-    dataLen += snprintf(command + dataLen,
-        BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
+      int dataLen = 0;
+      dataLen += snprintf(command + dataLen,
+          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);

-    if (g_Dbs.db[i].dbCfg.blocks > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
-    }
-    if (g_Dbs.db[i].dbCfg.cache > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
-    }
-    if (g_Dbs.db[i].dbCfg.days > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
-    }
-    if (g_Dbs.db[i].dbCfg.keep > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
-    }
-    if (g_Dbs.db[i].dbCfg.quorum > 1) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
-    }
-    if (g_Dbs.db[i].dbCfg.replica > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
-    }
-    if (g_Dbs.db[i].dbCfg.update > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
-    }
-    //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
-    //  dataLen += snprintf(command + dataLen,
-    //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
-    //}
-    if (g_Dbs.db[i].dbCfg.minRows > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
-    }
-    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
-    }
-    if (g_Dbs.db[i].dbCfg.comp > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
-    }
-    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
-    }
-    if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
-    }
-    if (g_Dbs.db[i].dbCfg.fsync > 0) {
-      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
-              " fsync %d", g_Dbs.db[i].dbCfg.fsync);
-    }
-    if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
-            || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
-                    "us", strlen("us")))) {
-      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
-              " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
-    }
+      if (g_Dbs.db[i].dbCfg.blocks > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
+      }
+      if (g_Dbs.db[i].dbCfg.cache > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
+      }
+      if (g_Dbs.db[i].dbCfg.days > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
+      }
+      if (g_Dbs.db[i].dbCfg.keep > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
+      }
+      if (g_Dbs.db[i].dbCfg.quorum > 1) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
+      }
+      if (g_Dbs.db[i].dbCfg.replica > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
+      }
+      if (g_Dbs.db[i].dbCfg.update > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
+      }
+      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
+      //  dataLen += snprintf(command + dataLen,
+      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
+      //}
+      if (g_Dbs.db[i].dbCfg.minRows > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
+      }
+      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
+      }
+      if (g_Dbs.db[i].dbCfg.comp > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
+      }
+      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
+      }
+      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
+      }
+      if (g_Dbs.db[i].dbCfg.fsync > 0) {
+        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
+                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
+      }
+      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
+              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
+                      "us", strlen("us")))) {
+        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
+                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
+      }

-    debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
-    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
-      taos_close(taos);
-      errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
-      return -1;
+      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
+      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
+        taos_close(taos);
+        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
+        return -1;
+      }
+      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
    }
-    printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);

    debugPrint("%s() %d supertbl count:%d\n",
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
@ -5132,16 +5132,14 @@ static void startMultiThreadInsertData(int threads, char* db_name,
  if (superTblInfo) {
    int limit, offset;

-    if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) {
-      printf("WARNING: specified offset >= child table count! \n");
-      if (!g_args.answer_yes) {
-        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
-        (void)getchar();
-      }
+    if ((superTblInfo->childTblExists == TBL_NO_EXISTS) &&
+            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) {
+      printf("WARNING: offset and limit will not be used since the child tables are not exists!\n");
    }

-    if (superTblInfo->childTblOffset >= 0) {
-      if (superTblInfo->childTblLimit <= 0) {
+    if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS)
+            && (superTblInfo->childTblOffset >= 0)) {
+      if (superTblInfo->childTblLimit < 0) {
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }
@ -5149,13 +5147,32 @@ static void startMultiThreadInsertData(int threads, char* db_name,
      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
    } else {
-        limit = superTblInfo->childTblCount;
-        offset = 0;
+      limit = superTblInfo->childTblCount;
+      offset = 0;
    }

    ntables = limit;
    startFrom = offset;

+    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
+        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
+            > superTblInfo->childTblCount)) {
+      printf("WARNING: specified offset + limit > child table count!\n");
+      if (!g_args.answer_yes) {
+        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
+        (void)getchar();
+      }
+    }
+
+    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
+            && (0 == superTblInfo->childTblLimit)) {
+      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
+      if (!g_args.answer_yes) {
+        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
+        (void)getchar();
+      }
+    }
+
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
--- a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json
+++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json
@ -0,0 +1,57 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 0,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}
--- a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json
+++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json
@ -0,0 +1,59 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"yes",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 1000,
+            "childtable_limit": 33,
+            "childtable_offset": 33,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}
--- a/tests/pytest/tools/insert-tblimit-tboffset0.json
+++ b/tests/pytest/tools/insert-tblimit-tboffset0.json
@ -15,7 +15,7 @@
    "databases": [{
        "dbinfo": {
            "name": "db",
-            "drop": "yes",
+            "drop": "no",
            "replica": 1,
            "days": 10,
            "cache": 16,
@ -33,7 +33,7 @@
        },
        "super_tables": [{
            "name": "stb",
-            "child_table_exists":"no",
+            "child_table_exists":"yes",
            "childtable_count": 100,
            "childtable_prefix": "stb_",
            "auto_create_table": "no",
--- a/tests/pytest/tools/insert-tblimit1-tboffset.json
+++ b/tests/pytest/tools/insert-tblimit1-tboffset.json
@ -15,7 +15,7 @@
    "databases": [{
        "dbinfo": {
            "name": "db",
-            "drop": "yes",
+            "drop": "no",
            "replica": 1,
            "days": 10,
            "cache": 16,
@ -33,7 +33,7 @@
        },
        "super_tables": [{
            "name": "stb",
-            "child_table_exists":"no",
+            "child_table_exists":"yes",
            "childtable_count": 100,
            "childtable_prefix": "stb_",
            "auto_create_table": "no",
--- a/tests/pytest/tools/taosdemo-sampledata.json
+++ b/tests/pytest/tools/taosdemo-sampledata.json
@ -16,8 +16,6 @@
 			"name": "stb",
 			"child_table_exists":"no",
 			"childtable_count": 20,
-			"childtable_limit": 10,
-			"childtable_offset": 0,
 			"childtable_prefix": "t_",
 			"auto_create_table": "no",
 			"data_source": "sample",
--- a/tests/pytest/tools/taosdemoTestLimitOffset.py
+++ b/tests/pytest/tools/taosdemoTestLimitOffset.py
@ -51,7 +51,8 @@ class TDTestCase:
        else:
            tdLog.info("taosd found in %s" % buildPath)
        binPath = buildPath+ "/build/bin/"
-        os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath)
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-insertrec.json" % binPath)

        tdSql.execute("use db")
        tdSql.query("select count(tbname) from db.stb")
@ -59,6 +60,7 @@ class TDTestCase:
        tdSql.query("select count(*) from db.stb")
        tdSql.checkData(0, 0, 33000)

+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
        os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath)

        tdSql.execute("reset query cache")
@ -68,6 +70,7 @@ class TDTestCase:
        tdSql.query("select count(*) from db.stb")
        tdSql.checkData(0, 0, 20000)

+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
        os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath)

        tdSql.execute("reset query cache")
--- a/tests/pytest/tools/taosdemoTestSampleData.py
+++ b/tests/pytest/tools/taosdemoTestSampleData.py
@ -57,7 +57,7 @@ class TDTestCase:
        tdSql.query("select count(tbname) from db.stb")
        tdSql.checkData(0, 0, 20)
        tdSql.query("select count(*) from db.stb")
-        tdSql.checkData(0, 0, 200)
+        tdSql.checkData(0, 0, 400)

    def stop(self):
        tdSql.close()