xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feature/sangshuduo/td 3317 taosdemo interlace (#5537) 

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

check offset 0.

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

fix sample file import bug.

* [TD-3316] <fix>: add test case for limit and offset. fix sample data issue.

* [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file.

* [TD-3317] <feature>: make taosdemo support interlace mode.

json parameter rows_per_tbl support.

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode insertion.

refactor.

* [TD-3317] <feature>: support interlace mode insertion.

change json file.

* [TD-3317] <feature>: support interlace mode insertion.

fix multithread create table regression.

* [TD-3317] <feature>: support interlace mode insertion.

working but not perfect.

* [TD-3317] <feature>: support interlace mode insertion.

rename lowaTest with taosdemoTestWithJson

* [TD-3317] <feature>: support interlace mode insertion.

perfect

* [TD-3317] <feature>: support interlace mode insertion.

cleanup.

* [TD-3317] <feature>: support interlace mode insertion.

adjust algorithm of loop times.

* [TD-3317] <feature>: support interlace mode insertion.

fix delay time bug.

* [TD-3317] <feature>: support interlace mode insertion.

fix progressive timestamp bug.

* [TD-3317] <feature>: support interlace mode insertion.

add an option for performance print.

* [TD-3317] <feature>: support interlace mode insertion.

change json test case with less table for acceleration.

* [TD-3317] <feature>: support interlace mode insertion.

change progressive mode timestamp step and testcase.

* [TD-3197] <fix>: fix taosdemo coverity scan issues.

* [TD-3197] <fix>: fix taosdemo coverity scan issue.

fix subscribeTest pids uninitialized.

* [TD-3317] <feature>: support interlace mode insertion.

add time shift for no sleep time.

* [TD-3317] <feature>: support interlace insert.

rework timestamp.

* [TD-3317] <feature>: support interlace mode insertion.

change rows_per_tbl to interlace_rows.

Co-authored-by: Shuduo Sang <sdsang@taosdata.com>
This commit is contained in:
Shuduo Sang 2021-03-23 15:14:50 +08:00 committed by GitHub
parent dfdeb7e0cc
commit 3b97bd716d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/kit/taosdemo/insert-interlace.json
View File

@ -41,7 +41,7 @@
"insert_mode": "taosc", "insert_mode": "taosc",
"insert_rows": 1000, "insert_rows": 1000,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"rows_per_tbl": 20, "interlace_rows": 20,
"max_sql_len": 1024000, "max_sql_len": 1024000,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,

2
src/kit/taosdemo/insert.json
View File

@ -41,7 +41,7 @@
"insert_mode": "taosc", "insert_mode": "taosc",
"insert_rows": 100000, "insert_rows": 100000,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"rows_per_tbl": 0, "interlace_rows": 0,
"max_sql_len": 1024000, "max_sql_len": 1024000,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,

21
src/kit/taosdemo/taosdemo.c
View File

@ -199,7 +199,7 @@ typedef struct SArguments_S {
int num_of_CPR; int num_of_CPR;
int num_of_threads; int num_of_threads;
int insert_interval; int insert_interval;
int rows_per_tbl; int interlace_rows;
int num_of_RPR; int num_of_RPR;
int max_sql_len; int max_sql_len;
int num_of_tables; int num_of_tables;
@ -547,7 +547,7 @@ SArguments g_args = {
10, // num_of_CPR 10, // num_of_CPR
10, // num_of_connections/thread 10, // num_of_connections/thread
0, // insert_interval 0, // insert_interval
0, // rows_per_tbl; 0, // interlace_rows;
100, // num_of_RPR 100, // num_of_RPR
TSDB_PAYLOAD_SIZE, // max_sql_len TSDB_PAYLOAD_SIZE, // max_sql_len
10000, // num_of_tables 10000, // num_of_tables
@ -682,7 +682,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
} else if (strcmp(argv[i], "-i") == 0) { } else if (strcmp(argv[i], "-i") == 0) {
arguments->insert_interval = atoi(argv[++i]); arguments->insert_interval = atoi(argv[++i]);
} else if (strcmp(argv[i], "-B") == 0) { } else if (strcmp(argv[i], "-B") == 0) {
arguments->rows_per_tbl = atoi(argv[++i]); arguments->interlace_rows = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) { } else if (strcmp(argv[i], "-r") == 0) {
arguments->num_of_RPR = atoi(argv[++i]); arguments->num_of_RPR = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0) { } else if (strcmp(argv[i], "-t") == 0) {
@ -3008,13 +3008,13 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "rows_per_tbl"); cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "interlace_rows");
if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
g_args.rows_per_tbl = rowsPerTbl->valueint; g_args.interlace_rows = rowsPerTbl->valueint;
} else if (!rowsPerTbl) { } else if (!rowsPerTbl) {
g_args.rows_per_tbl = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
} else { } else {
errorPrint("%s() LN%d, failed to read json, rows_per_tbl input mistake\n", __func__, __LINE__); errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__);
goto PARSE_OVER; goto PARSE_OVER;
} }
@ -3498,7 +3498,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "interlace_rows");
if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint;
} else if (!rowsPerTbl) { } else if (!rowsPerTbl) {
@ -4425,7 +4425,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
int insertMode; int insertMode;
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl; int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows;
if (rowsPerTbl > 0) { if (rowsPerTbl > 0) {
insertMode = INTERLACE_INSERT_MODE; insertMode = INTERLACE_INSERT_MODE;
@ -4518,6 +4518,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
pstr += dataLen; pstr += dataLen;
recOfBatch += batchPerTbl; recOfBatch += batchPerTbl;
startTime += batchPerTbl * superTblInfo->timeStampStep;
pThreadInfo->totalInsertRows += batchPerTbl; pThreadInfo->totalInsertRows += batchPerTbl;
verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
@ -4746,7 +4747,7 @@ static void* syncWrite(void *sarg) {
threadInfo *winfo = (threadInfo *)sarg; threadInfo *winfo = (threadInfo *)sarg;
SSuperTable* superTblInfo = winfo->superTblInfo; SSuperTable* superTblInfo = winfo->superTblInfo;
int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl; int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows;
if (rowsPerTbl > 0) { if (rowsPerTbl > 0) {
// interlace mode // interlace mode

2
tests/examples/JDBC/taosdemo/src/main/resources/insert.json
View File

@ -38,7 +38,7 @@
"insert_rows": 100, "insert_rows": 100,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 0, "number_of_tbl_in_one_sql": 0,
"rows_per_tbl": 3, "interlace_rows": 3,
"max_sql_len": 1024, "max_sql_len": 1024,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,

2
tests/pytest/cluster/clusterEnvSetup/insert.json
View File

@ -39,7 +39,7 @@
"insert_rows": 100000, "insert_rows": 100000,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 1, "number_of_tbl_in_one_sql": 1,
"rows_per_tbl": 100, "interlace_rows": 100,
"max_sql_len": 1024000, "max_sql_len": 1024000,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,

4
tests/pytest/tools/insert-interlace.json
View File

@ -10,7 +10,7 @@
"result_file": "./insert_res.txt", "result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"insert_interval": 5000, "insert_interval": 5000,
"rows_per_tbl": 50, "interlace_rows": 50,
"num_of_records_per_req": 100, "num_of_records_per_req": 100,
"max_sql_len": 1024000, "max_sql_len": 1024000,
"databases": [{ "databases": [{
@ -42,7 +42,7 @@
"insert_mode": "taosc", "insert_mode": "taosc",
"insert_rows": 250, "insert_rows": 250,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"rows_per_tbl": 80, "interlace_rows": 80,
"max_sql_len": 1024000, "max_sql_len": 1024000,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,