xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feature/sangshuduo/td 3317 taosdemo interlace (#5477) 

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

check offset 0.

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

fix sample file import bug.

* [TD-3316] <fix>: add test case for limit and offset. fix sample data issue.

* [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file.

* [TD-3317] <feature>: make taosdemo support interlace mode.

json parameter rows_per_tbl support.

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode insertion.

refactor.

* [TD-3317] <feature>: support interlace mode insertion.

change json file.

* [TD-3317] <feature>: support interlace mode insertion.

fix multithread create table regression.

* [TD-3317] <feature>: support interlace mode insertion.

working but not perfect.

* [TD-3317] <feature>: support interlace mode insertion.

rename lowaTest with taosdemoTestWithJson

* [TD-3317] <feature>: support interlace mode insertion.

perfect

Co-authored-by: Shuduo Sang <sdsang@taosdata.com>
This commit is contained in:
Shuduo Sang 2021-03-18 09:41:26 +08:00 committed by GitHub
parent 3f312f8bf4
commit 199e61ec54
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 67 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/kit/taosdemo/insert-interlace.json
View File

@ -9,8 +9,7 @@
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 5000,
"rows_per_tbl": 50,
"insert_interval": 1000,
"num_of_records_per_req": 100,
"max_sql_len": 1024000,
"databases": [{
@ -35,12 +34,12 @@
"super_tables": [{
"name": "stb",
"child_table_exists":"no",
"childtable_count": 10,
"childtable_count": 100,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 200,
"insert_rows": 1000,
"multi_thread_write_one_tbl": "no",
"rows_per_tbl": 20,
"max_sql_len": 1024000,
@ -51,8 +50,8 @@
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
"columns": [{"type": "INT"}],
"tags": [{"type": "TINYINT", "count":1}]
}]
}]
}

56
src/kit/taosdemo/taosdemo.c
View File

@ -4444,27 +4444,17 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
int64_t startTime = pThreadInfo->start_time;
int batchPerTblTimes;
int prevBatchPerTbl, lastBatchPerTbl;
int batchPerTbl;
if (pThreadInfo->ntables == 1) {
batchPerTblTimes = 1;
lastBatchPerTbl = rowsPerTbl;
prevBatchPerTbl = rowsPerTbl;
} else if (rowsPerTbl > 0) {
if ((rowsPerTbl > 0) && (pThreadInfo->ntables > 1)) {
batchPerTblTimes = g_args.num_of_RPR / rowsPerTbl;
lastBatchPerTbl = g_args.num_of_RPR % rowsPerTbl;
if (lastBatchPerTbl > 0)
batchPerTblTimes += 1;
else
lastBatchPerTbl = rowsPerTbl;
prevBatchPerTbl = rowsPerTbl;
batchPerTbl = rowsPerTbl;
} else {
batchPerTblTimes = 1;
prevBatchPerTbl = g_args.num_of_RPR;
lastBatchPerTbl = g_args.num_of_RPR;
batchPerTbl = g_args.num_of_RPR;
}
int generatedRecPerTbl = 0;
while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
if (insert_interval) {
st = taosGetTimestampUs();
@ -4476,12 +4466,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
int recGenerated = 0;
for (int i = 0; i < batchPerTblTimes; i ++) {
if (insertMode == INTERLACE_INSERT_MODE) {
if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
// turn to first table
tableSeq = pThreadInfo->start_table_from;
}
}
getTableName(tableName, pThreadInfo, tableSeq);
int headLen;
@ -4500,15 +4484,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
pstr += headLen;
int dataLen = 0;
int batchPerTbl;
if (i == batchPerTblTimes - 1) {
batchPerTbl = lastBatchPerTbl;
} else {
batchPerTbl = prevBatchPerTbl;
}
verbosePrint("%s() LN%d batchPerTbl = %d\n",
__func__, __LINE__, batchPerTbl);
printf("%s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
__func__, __LINE__, i, batchPerTblTimes, batchPerTbl);
int numOfRecGenerated = generateDataTail(
tableName, tableSeq, pThreadInfo, superTblInfo,
batchPerTbl, pstr, insertRows, 0,
@ -4520,10 +4497,25 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
recGenerated += numOfRecGenerated;
tableSeq ++;
if (insertMode == INTERLACE_INSERT_MODE) {
if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
// turn to first table
tableSeq = pThreadInfo->start_table_from;
generatedRecPerTbl += numOfRecGenerated;
}
}
int remainRows = insertRows - generatedRecPerTbl;
if (batchPerTbl > remainRows)
batchPerTbl = remainRows;
if ((g_args.num_of_RPR - recGenerated) < batchPerTbl)
break;
}
verbosePrint("%s() LN%d buffer:\n%s\n",
__func__, __LINE__, buffer);
pThreadInfo->totalInsertRows += recGenerated;
printf("%s() LN%d recGenerated=%d totalInsertRows=%"PRId64" buffer:\n%s\n",
__func__, __LINE__, recGenerated,
pThreadInfo->totalInsertRows, buffer);
int affectedRows = execInsert(pThreadInfo, buffer, recGenerated);
if (affectedRows < 0)