Hotfix/sangshuduo/td 3733 for develop (#5768)
* [TD-3733]<fix>: taosdemo buffer processing refactor. * [TD-3733]<fix>: taosdemo generate SQL head buffer process. * [TD-3733]<fix>: taosdemo generate sql head buffer processing. fix compile issue for windows. * [TD-3733]<fix>: taosdemo generate sql head buffer processing. fix max_sql_len. generate data buffer refactor. Co-authored-by: Shuduo Sang <sdsang@taosdata.com>
This commit is contained in:
Shuduo Sang
GitHub
parent
0913d4553e
commit
fd5cebdf34
|
|
@ -2949,9 +2949,6 @@ static int readSampleFromCsvFileToMem(
|
|||
continue;
|
||||
}
|
||||
|
||||
verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen,
|
||||
superTblInfo->lenOfOneRow, getRows);
|
||||
|
||||
memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
|
||||
line, readLen);
|
||||
getRows++;
|
||||
|
|
@ -3214,9 +3211,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
|
||||
// rows per table need be less than insert batch
|
||||
if (g_args.interlace_rows > g_args.num_of_RPR) {
|
||||
printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n",
|
||||
printf("NOTICE: interlace rows value %d > num_of_records_per_req %d\n\n",
|
||||
g_args.interlace_rows, g_args.num_of_RPR);
|
||||
printf(" interlace rows value will be set to num_of_records_per_request %d\n\n",
|
||||
printf(" interlace rows value will be set to num_of_records_per_req %d\n\n",
|
||||
g_args.num_of_RPR);
|
||||
printf(" press Enter key to continue or Ctrl-C to stop.");
|
||||
(void)getchar();
|
||||
|
|
@ -3527,6 +3524,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
goto PARSE_OVER;
|
||||
}
|
||||
|
||||
/*
|
||||
cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
|
||||
if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
|
||||
g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
|
||||
|
|
@ -3536,6 +3534,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
|
||||
goto PARSE_OVER;
|
||||
}
|
||||
*/
|
||||
|
||||
cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
|
||||
if (childTblExists
|
||||
|
|
@ -3683,14 +3682,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
int32_t len = maxSqlLen->valueint;
|
||||
if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
|
||||
len = TSDB_MAX_ALLOWED_SQL_LEN;
|
||||
} else if (len < TSDB_MAX_SQL_LEN) {
|
||||
len = TSDB_MAX_SQL_LEN;
|
||||
} else if (len < 5) {
|
||||
len = 5;
|
||||
}
|
||||
g_Dbs.db[i].superTbls[j].maxSqlLen = len;
|
||||
} else if (!maxSqlLen) {
|
||||
g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
|
||||
} else {
|
||||
printf("ERROR: failed to read json, maxSqlLen not found\n");
|
||||
errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n",
|
||||
__func__, __LINE__);
|
||||
goto PARSE_OVER;
|
||||
}
|
||||
|
||||
|
|
@ -3716,9 +3716,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
|
|||
g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
|
||||
// rows per table need be less than insert batch
|
||||
if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
|
||||
printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n",
|
||||
printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_req %d\n\n",
|
||||
i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
|
||||
printf(" interlace rows value will be set to num_of_records_per_request %d\n\n",
|
||||
printf(" interlace rows value will be set to num_of_records_per_req %d\n\n",
|
||||
g_args.num_of_RPR);
|
||||
printf(" press Enter key to continue or Ctrl-C to stop.");
|
||||
(void)getchar();
|
||||
|
|
@ -4512,13 +4512,15 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
|
|||
}
|
||||
}
|
||||
|
||||
static int generateDataTail(char *tableName, int32_t tableSeq,
|
||||
threadInfo* pThreadInfo, SSuperTable* superTblInfo,
|
||||
static int generateDataTail(
|
||||
SSuperTable* superTblInfo,
|
||||
int batch, char* buffer, int remainderBufLen, int64_t insertRows,
|
||||
int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
|
||||
int len = 0;
|
||||
int ncols_per_record = 1; // count first col ts
|
||||
|
||||
char *pstr = buffer;
|
||||
|
||||
if (superTblInfo == NULL) {
|
||||
int datatypeSeq = 0;
|
||||
while(g_args.datatype[datatypeSeq]) {
|
||||
|
|
@ -4547,15 +4549,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
|
|||
pSamplePos);
|
||||
} else if (0 == strncasecmp(superTblInfo->dataSource,
|
||||
"rand", strlen("rand"))) {
|
||||
|
||||
int randTail = superTblInfo->timeStampStep * k;
|
||||
if (superTblInfo->disorderRatio > 0) {
|
||||
int rand_num = taosRandom() % 100;
|
||||
int randTail;
|
||||
if (0 != superTblInfo->disorderRatio
|
||||
&& rand_num < superTblInfo->disorderRatio) {
|
||||
randTail = (superTblInfo->timeStampStep * k
|
||||
+ (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
|
||||
if(rand_num < superTblInfo->disorderRatio) {
|
||||
randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
|
||||
debugPrint("rand data generated, back %d\n", randTail);
|
||||
} else {
|
||||
randTail = superTblInfo->timeStampStep * k;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t d = startTime
|
||||
|
|
@ -4570,7 +4571,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
|
|||
break;
|
||||
}
|
||||
|
||||
buffer += snprintf(buffer, retLen + 1, "%s", data);
|
||||
pstr += snprintf(pstr , retLen + 1, "%s", data);
|
||||
k++;
|
||||
len += retLen;
|
||||
remainderBufLen -= retLen;
|
||||
|
|
@ -4598,7 +4599,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
|
|||
if (len > remainderBufLen)
|
||||
break;
|
||||
|
||||
buffer += sprintf(buffer, " %s", data);
|
||||
pstr += sprintf(pstr, " %s", data);
|
||||
k++;
|
||||
len += retLen;
|
||||
remainderBufLen -= retLen;
|
||||
|
|
@ -4619,9 +4620,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
|
|||
}
|
||||
|
||||
static int generateSQLHead(char *tableName, int32_t tableSeq,
|
||||
threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer)
|
||||
threadInfo* pThreadInfo, SSuperTable* superTblInfo,
|
||||
char *buffer, int remainderBufLen)
|
||||
{
|
||||
int len;
|
||||
|
||||
#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into ..
|
||||
char headBuf[HEAD_BUFF_LEN];
|
||||
|
||||
if (superTblInfo) {
|
||||
if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
|
||||
char* tagsValBuf = NULL;
|
||||
|
|
@ -4638,8 +4644,9 @@ static int generateSQLHead(char *tableName, int32_t tableSeq,
|
|||
return -1;
|
||||
}
|
||||
|
||||
len = snprintf(buffer,
|
||||
superTblInfo->maxSqlLen,
|
||||
len = snprintf(
|
||||
headBuf,
|
||||
HEAD_BUFF_LEN,
|
||||
"insert into %s.%s using %s.%s tags %s values",
|
||||
pThreadInfo->db_name,
|
||||
tableName,
|
||||
|
|
@ -4648,30 +4655,93 @@ static int generateSQLHead(char *tableName, int32_t tableSeq,
|
|||
tagsValBuf);
|
||||
tmfree(tagsValBuf);
|
||||
} else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
|
||||
len = snprintf(buffer,
|
||||
superTblInfo->maxSqlLen,
|
||||
len = snprintf(
|
||||
headBuf,
|
||||
HEAD_BUFF_LEN,
|
||||
"insert into %s.%s values",
|
||||
pThreadInfo->db_name,
|
||||
tableName);
|
||||
} else {
|
||||
len = snprintf(buffer,
|
||||
superTblInfo->maxSqlLen,
|
||||
len = snprintf(
|
||||
headBuf,
|
||||
HEAD_BUFF_LEN,
|
||||
"insert into %s.%s values",
|
||||
pThreadInfo->db_name,
|
||||
tableName);
|
||||
}
|
||||
} else {
|
||||
len = snprintf(buffer,
|
||||
g_args.max_sql_len,
|
||||
len = snprintf(
|
||||
headBuf,
|
||||
HEAD_BUFF_LEN,
|
||||
"insert into %s.%s values",
|
||||
pThreadInfo->db_name,
|
||||
tableName);
|
||||
}
|
||||
|
||||
if (len > remainderBufLen)
|
||||
return -1;
|
||||
|
||||
tstrncpy(buffer, headBuf, len + 1);
|
||||
|
||||
return len;
|
||||
}
|
||||
|
||||
static int generateProgressiveDataBuffer(char *pTblName,
|
||||
static int generateInterlaceDataBuffer(
|
||||
char *tableName, int batchPerTbl, int i, int batchPerTblTimes,
|
||||
int32_t tableSeq,
|
||||
threadInfo *pThreadInfo, char *buffer,
|
||||
int64_t insertRows,
|
||||
int64_t startTime,
|
||||
int *pRemainderBufLen)
|
||||
{
|
||||
char *pstr = buffer;
|
||||
SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
|
||||
|
||||
int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
|
||||
superTblInfo, pstr, *pRemainderBufLen);
|
||||
|
||||
if (headLen <= 0) {
|
||||
return 0;
|
||||
}
|
||||
// generate data buffer
|
||||
verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__, i, buffer);
|
||||
|
||||
pstr += headLen;
|
||||
*pRemainderBufLen -= headLen;
|
||||
|
||||
int dataLen = 0;
|
||||
|
||||
verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__,
|
||||
i, batchPerTblTimes, batchPerTbl);
|
||||
|
||||
if (superTblInfo) {
|
||||
if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
|
||||
startTime = taosGetTimestamp(pThreadInfo->time_precision);
|
||||
}
|
||||
} else {
|
||||
startTime = 1500000000000;
|
||||
}
|
||||
int k = generateDataTail(
|
||||
superTblInfo,
|
||||
batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
|
||||
startTime,
|
||||
&(pThreadInfo->samplePos), &dataLen);
|
||||
|
||||
if (k > 0) {
|
||||
pstr += dataLen;
|
||||
*pRemainderBufLen -= dataLen;
|
||||
} else {
|
||||
pstr -= headLen;
|
||||
pstr[0] = '\0';
|
||||
}
|
||||
|
||||
return k;
|
||||
}
|
||||
|
||||
static int generateProgressiveDataBuffer(
|
||||
char *tableName,
|
||||
int32_t tableSeq,
|
||||
threadInfo *pThreadInfo, char *buffer,
|
||||
int64_t insertRows,
|
||||
|
|
@ -4691,6 +4761,7 @@ static int generateProgressiveDataBuffer(char *pTblName,
|
|||
|
||||
assert(buffer != NULL);
|
||||
|
||||
int k = 0;
|
||||
int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
|
||||
int remainderBufLen = maxSqlLen;
|
||||
|
||||
|
|
@ -4698,14 +4769,17 @@ static int generateProgressiveDataBuffer(char *pTblName,
|
|||
|
||||
char *pstr = buffer;
|
||||
|
||||
int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
|
||||
buffer);
|
||||
int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo,
|
||||
buffer, remainderBufLen);
|
||||
|
||||
if (headLen <= 0) {
|
||||
return 0;
|
||||
}
|
||||
pstr += headLen;
|
||||
remainderBufLen -= headLen;
|
||||
|
||||
int k;
|
||||
int dataLen;
|
||||
k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
|
||||
k = generateDataTail(superTblInfo,
|
||||
g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom,
|
||||
startTime,
|
||||
pSamplePos, &dataLen);
|
||||
|
|
@ -4722,8 +4796,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
|
||||
int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;
|
||||
|
||||
if (interlaceRows > insertRows)
|
||||
interlaceRows = insertRows;
|
||||
if (interlaceRows > g_args.num_of_RPR)
|
||||
interlaceRows = g_args.num_of_RPR;
|
||||
|
||||
int insertMode;
|
||||
|
||||
|
|
@ -4788,8 +4862,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
bool flagSleep = true;
|
||||
int sleepTimeTotal = 0;
|
||||
|
||||
int remainderBufLen;
|
||||
|
||||
while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
|
||||
if ((flagSleep) && (insert_interval)) {
|
||||
st = taosGetTimestampUs();
|
||||
|
|
@ -4797,7 +4869,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
}
|
||||
// generate data
|
||||
memset(buffer, 0, maxSqlLen);
|
||||
remainderBufLen = maxSqlLen;
|
||||
int remainderBufLen = maxSqlLen;
|
||||
|
||||
char *pstr = buffer;
|
||||
int recOfBatch = 0;
|
||||
|
|
@ -4811,50 +4883,23 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
int headLen;
|
||||
if (i == 0) {
|
||||
headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
|
||||
superTblInfo, pstr);
|
||||
} else {
|
||||
headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values",
|
||||
pThreadInfo->db_name,
|
||||
tableName);
|
||||
}
|
||||
|
||||
// generate data buffer
|
||||
verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__, i, buffer);
|
||||
|
||||
pstr += headLen;
|
||||
remainderBufLen -= headLen;
|
||||
|
||||
int dataLen = 0;
|
||||
|
||||
verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__,
|
||||
i, batchPerTblTimes, batchPerTbl);
|
||||
|
||||
if (superTblInfo) {
|
||||
if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
|
||||
startTime = taosGetTimestamp(pThreadInfo->time_precision);
|
||||
}
|
||||
} else {
|
||||
startTime = 1500000000000;
|
||||
}
|
||||
int generated = generateDataTail(
|
||||
tableName, tableSeq, pThreadInfo, superTblInfo,
|
||||
batchPerTbl, pstr, remainderBufLen, insertRows, 0,
|
||||
int generated = generateInterlaceDataBuffer(
|
||||
tableName, batchPerTbl, i, batchPerTblTimes,
|
||||
tableSeq,
|
||||
pThreadInfo, pstr,
|
||||
insertRows,
|
||||
startTime,
|
||||
&(pThreadInfo->samplePos), &dataLen);
|
||||
&remainderBufLen);
|
||||
|
||||
if (generated < 0) {
|
||||
debugPrint("[%d] %s() LN%d, generated data is %d\n",
|
||||
pThreadInfo->threadID, __func__, __LINE__, generated);
|
||||
goto free_and_statistics_interlace;
|
||||
} else if (generated == 0) {
|
||||
break;
|
||||
}
|
||||
pstr += dataLen;
|
||||
remainderBufLen -= dataLen;
|
||||
|
||||
tableSeq ++;
|
||||
recOfBatch += batchPerTbl;
|
||||
// startTime += batchPerTbl * superTblInfo->timeStampStep;
|
||||
pThreadInfo->totalInsertRows += batchPerTbl;
|
||||
|
|
@ -4862,7 +4907,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
|
|||
pThreadInfo->threadID, __func__, __LINE__,
|
||||
batchPerTbl, recOfBatch);
|
||||
|
||||
tableSeq ++;
|
||||
if (insertMode == INTERLACE_INSERT_MODE) {
|
||||
if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
|
||||
// turn to first table
|
||||
|
|
@ -5071,12 +5115,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
|
|||
*/
|
||||
} // num_of_DPT
|
||||
|
||||
if (g_args.verbose_print) {
|
||||
if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
|
||||
(0 == strncasecmp(
|
||||
superTblInfo->dataSource, "sample", strlen("sample")))) {
|
||||
printf("%s() LN%d samplePos=%d\n",
|
||||
verbosePrint("%s() LN%d samplePos=%d\n",
|
||||
__func__, __LINE__, pThreadInfo->samplePos);
|
||||
}
|
||||
}
|
||||
} // tableSeq
|
||||
|
||||
free_and_statistics_2:
|
||||
|
|
|
|||
Loading…
Reference in New Issue