Feature/sangshuduo/td 3317 taosdemo interlace (#5922)
* [TD-3316] <fix>: add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] <fix>: add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] <fix>: add test case for limit and offset. fix sample data issue. * [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file. * [TD-3317] <feature>: make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] <feature>: support interlace mode. refactor * [TD-3317] <feature>: support interlace mode. refactor * [TD-3317] <feature>: support interlace mode insertion. refactor. * [TD-3317] <feature>: support interlace mode insertion. change json file. * [TD-3317] <feature>: support interlace mode insertion. fix multithread create table regression. * [TD-3317] <feature>: support interlace mode insertion. working but not perfect. * [TD-3317] <feature>: support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] <feature>: support interlace mode insertion. perfect * [TD-3317] <feature>: support interlace mode insertion. cleanup. * [TD-3317] <feature>: support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] <feature>: support interlace mode insertion. fix delay time bug. * [TD-3317] <feature>: support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] <feature>: support interlace mode insertion. add an option for performance print. * [TD-3317] <feature>: support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] <feature>: support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] <fix>: fix taosdemo coverity scan issues. * [TD-3197] <fix>: fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] <feature>: support interlace mode insertion. add time shift for no sleep time. * [TD-3317] <feature>: support interlace insert. rework timestamp. * [TD-3317] <feature>: support interlace mode insertion. change rows_per_tbl to interlace_rows. * [TD-3317] <feature>: taosdemo suppoert interlace mode. remove trailing spaces. * [TD-3317] <feature>: taosdemo support interlace insertion. prompt if interlace > num_of_records_per_req * fill insert-into early to buffer. * fix buffer overflow issue. * change rows_per_tbl to interlace_rows to align with taosdemo. * adjust remainder rows logic. * [TD-3317]<fix>: taosdemo support interlace mode. fix global and stable interlace rows logic. * [TD-3317]<fix>: taosdemo support interlace mode. fix 'interlaceRows' is used uninitialized Co-authored-by: Shuduo Sang <sdsang@taosdata.com>
This commit is contained in:
Shuduo Sang
GitHub
parent
cb220d6572
commit
2aa0f15ea2
|
|
@ -252,7 +252,7 @@ typedef struct SSuperTable_S {
|
|||
int maxSqlLen; //
|
||||
|
||||
int insertInterval; // insert interval, will override global insert interval
|
||||
int64_t insertRows; // 0: no limit
|
||||
int64_t insertRows;
|
||||
int timeStampStep;
|
||||
char startTimestamp[MAX_TB_NAME_SIZE];
|
||||
char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json
|
||||
|
|
@ -535,11 +535,11 @@ SArguments g_args = {
|
|||
"127.0.0.1", // host
|
||||
6030, // port
|
||||
"root", // user
|
||||
#ifdef _TD_POWER_
|
||||
#ifdef _TD_POWER_
|
||||
"powerdb", // password
|
||||
#else
|
||||
#else
|
||||
"taosdata", // password
|
||||
#endif
|
||||
#endif
|
||||
"test", // database
|
||||
1, // replica
|
||||
"t", // tb_prefix
|
||||
|
|
@ -3256,7 +3256,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(
|
|||
}
|
||||
ret = true;
|
||||
|
||||
PARSE_OVER:
|
||||
PARSE_OVER:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -3926,7 +3926,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
|
||||
ret = true;
|
||||
|
||||
PARSE_OVER:
|
||||
PARSE_OVER:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -4310,7 +4310,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
|
|||
|
||||
ret = true;
|
||||
|
||||
PARSE_OVER:
|
||||
PARSE_OVER:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -4371,7 +4371,7 @@ static bool getInfoFromJsonFile(char* file) {
|
|||
goto PARSE_OVER;
|
||||
}
|
||||
|
||||
PARSE_OVER:
|
||||
PARSE_OVER:
|
||||
free(content);
|
||||
cJSON_Delete(root);
|
||||
fclose(fp);
|
||||
|
|
@ -4868,6 +4868,8 @@ static int generateInterlaceDataBuffer(
|
|||
pstr += dataLen;
|
||||
*pRemainderBufLen -= dataLen;
|
||||
} else {
|
||||
debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %d\n",
|
||||
__func__, __LINE__, k, batchPerTbl);
|
||||
pstr -= headLen;
|
||||
pstr[0] = '\0';
|
||||
k = 0;
|
||||
|
|
@ -4925,10 +4927,24 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
debugPrint("[%d] %s() LN%d: ### interlace write\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__);
|
||||
|
||||
int64_t insertRows;
|
||||
int interlaceRows;
|
||||
|
||||
SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
|
||||
|
||||
int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
|
||||
int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;
|
||||
if (superTblInfo) {
|
||||
insertRows = superTblInfo->insertRows;
|
||||
|
||||
if ((superTblInfo->interlaceRows == 0)
|
||||
&& (g_args.interlace_rows > 0)) {
|
||||
interlaceRows = g_args.interlace_rows;
|
||||
} else {
|
||||
interlaceRows = superTblInfo->interlaceRows;
|
||||
}
|
||||
} else {
|
||||
insertRows = g_args.num_of_DPT;
|
||||
interlaceRows = g_args.interlace_rows;
|
||||
}
|
||||
|
||||
if (interlaceRows > insertRows)
|
||||
interlaceRows = insertRows;
|
||||
|
|
@ -5063,15 +5079,15 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
if (generatedRecPerTbl >= insertRows)
|
||||
break;
|
||||
|
||||
int remainRows = insertRows - generatedRecPerTbl;
|
||||
if ((remainRows > 0) && (batchPerTbl > remainRows))
|
||||
batchPerTbl = remainRows;
|
||||
|
||||
if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int remainRows = insertRows - generatedRecPerTbl;
|
||||
if ((remainRows > 0) && (batchPerTbl > remainRows))
|
||||
batchPerTbl = remainRows;
|
||||
|
||||
verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__,
|
||||
generatedRecPerTbl, insertRows);
|
||||
|
|
@ -5133,7 +5149,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
}
|
||||
}
|
||||
|
||||
free_and_statistics_interlace:
|
||||
free_and_statistics_interlace:
|
||||
tmfree(buffer);
|
||||
|
||||
printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n",
|
||||
|
|
@ -5279,7 +5295,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
|
|||
}
|
||||
} // tableSeq
|
||||
|
||||
free_and_statistics_2:
|
||||
free_and_statistics_2:
|
||||
tmfree(buffer);
|
||||
|
||||
printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n",
|
||||
|
|
@ -5294,7 +5310,18 @@ static void* syncWrite(void *sarg) {
|
|||
threadInfo *pThreadInfo = (threadInfo *)sarg;
|
||||
SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
|
||||
|
||||
int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;
|
||||
int interlaceRows;
|
||||
|
||||
if (superTblInfo) {
|
||||
if ((superTblInfo->interlaceRows == 0)
|
||||
&& (g_args.interlace_rows > 0)) {
|
||||
interlaceRows = g_args.interlace_rows;
|
||||
} else {
|
||||
interlaceRows = superTblInfo->interlaceRows;
|
||||
}
|
||||
} else {
|
||||
interlaceRows = g_args.interlace_rows;
|
||||
}
|
||||
|
||||
if (interlaceRows > 0) {
|
||||
// interlace mode
|
||||
|
|
@ -5981,7 +6008,6 @@ static void *specifiedTableQuery(void *sarg) {
|
|||
totalQueried ++;
|
||||
g_queryInfo.specifiedQueryInfo.totalQueried ++;
|
||||
|
||||
|
||||
et = taosGetTimestampMs();
|
||||
|
||||
int64_t currentPrintTime = taosGetTimestampMs();
|
||||
|
|
@ -5993,9 +6019,9 @@ static void *specifiedTableQuery(void *sarg) {
|
|||
pThreadInfo->threadID,
|
||||
totalQueried,
|
||||
(double)(totalQueried/((endTs-startTs)/1000.0)));
|
||||
}
|
||||
lastPrintTime = currentPrintTime;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -6079,10 +6105,10 @@ static void *superTableQuery(void *sarg) {
|
|||
pThreadInfo->threadID,
|
||||
totalQueried,
|
||||
(double)(totalQueried/((endTs-startTs)/1000.0)));
|
||||
}
|
||||
lastPrintTime = currentPrintTime;
|
||||
}
|
||||
}
|
||||
}
|
||||
et = taosGetTimestampMs();
|
||||
printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n",
|
||||
taosGetSelfPthreadId(),
|
||||
|
|
|
|||
Loading…
Reference in New Issue