xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-5136]<feature>: taosdemo simulate real senario. (#6833) 

* [TD-5136]<feature>: taosdemo simulate real senario.

* update test case according to taosdemo change
This commit is contained in:
Shuduo Sang 2021-07-12 17:08:54 +08:00 committed by GitHub
parent f3279fa698
commit 4486cae180
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 1755 additions and 1677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
src/kit/taosdemo/taosdemo.c
View File

@ -236,6 +236,7 @@ typedef struct SArguments_S {
char ** arg_list;
uint64_t totalInsertRows;
uint64_t totalAffectedRows;
bool demo_mode; // use default column name and semi-random data
} SArguments;
typedef struct SColumn_S {
@ -552,6 +553,8 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr,
uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
int disorderRatio, int disorderRange);
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
/* ************ Global variables ************ */
@ -578,7 +581,7 @@ SArguments g_args = {
#endif
"test", // database
1, // replica
"t", // tb_prefix
"d", // tb_prefix
NULL, // sqlFile
true, // use_metric
true, // drop_database
@ -590,10 +593,9 @@ SArguments g_args = {
"./output.txt", // output_file
0, // mode : sync or async
{
"FLOAT", // datatype
"INT", // datatype
"INT", // datatype
"INT", // datatype
"INT", // datatype
"FLOAT", // datatype
},
16, // len_of_binary
4, // num_of_CPR
@ -609,7 +611,10 @@ SArguments g_args = {
0, // disorderRatio
1000, // disorderRange
1, // method_of_delete
NULL // arg_list
NULL, // arg_list
0, // totalInsertRows;
0, // totalAffectedRows;
true, // demo_mode;
};
@ -709,7 +714,7 @@ static void printHelp() {
printf("%s%s%s%s\n", indent, "-a", indent,
"Set the replica parameters of the database, Default 1, min: 1, max: 3.");
printf("%s%s%s%s\n", indent, "-m", indent,
"Table prefix name. Default is 't'.");
"Table prefix name. Default is 'd'.");
printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
printf("%s%s%s%s\n", indent, "-o", indent,
@ -717,11 +722,11 @@ static void printHelp() {
printf("%s%s%s%s\n", indent, "-q", indent,
"Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
printf("%s%s%s%s\n", indent, "-b", indent,
"The data_type of columns, default: INT,INT,INT,INT.");
"The data_type of columns, default: FLOAT, INT, FLOAT.");
printf("%s%s%s%s\n", indent, "-w", indent,
"The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
printf("%s%s%s%s%d\n", indent, "-l", indent,
"The number of columns per record. Default is 4. Max values is ",
"The number of columns per record. Default is 3. Max values is ",
MAX_NUM_DATATYPE);
printf("%s%s%s%s\n", indent, "-T", indent,
"The number of threads. Default is 10.");
@ -733,6 +738,10 @@ static void printHelp() {
"The number of tables. Default is 10000.");
printf("%s%s%s%s\n", indent, "-n", indent,
"The number of records per table. Default is 10000.");
printf("%s%s%s%s\n", indent, "-M", indent,
"The value of records generated are totally random.");
printf("%s%s%s%s\n", indent, indent, indent,
" The default is to simulate power equipment senario.");
printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
printf("%s%s%s%s\n", indent, "-O", indent,
@ -746,7 +755,7 @@ static void printHelp() {
printf("%s%s%s%s\n", indent, "--help\t", indent,
"Print command line arguments list info.");
/* printf("%s%s%s%s\n", indent, "-D", indent,
"if elete database if exists. 0: no, 1: yes, default is 1");
"Delete database if exists. 0: no, 1: yes, default is 1");
*/
}
@ -769,6 +778,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-f") == 0) {
arguments->demo_mode = false;
arguments->metaFile = argv[++i];
} else if (strcmp(argv[i], "-c") == 0) {
if (argc == i+1) {
@ -911,6 +921,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
}
arguments->database = argv[++i];
} else if (strcmp(argv[i], "-l") == 0) {
arguments->demo_mode = false;
if (argc == i+1) {
if (!isStringNumber(argv[i+1])) {
printHelp();
@ -931,6 +942,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
}
} else if (strcmp(argv[i], "-b") == 0) {
arguments->demo_mode = false;
if (argc == i+1) {
printHelp();
errorPrint("%s", "\n\t-b need valid string following!\n");
@ -1000,6 +1012,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
arguments->tb_prefix = argv[++i];
} else if (strcmp(argv[i], "-N") == 0) {
arguments->use_metric = false;
} else if (strcmp(argv[i], "-M") == 0) {
arguments->demo_mode = false;
} else if (strcmp(argv[i], "-x") == 0) {
arguments->insert_only = false;
} else if (strcmp(argv[i], "-y") == 0) {
@ -1133,8 +1147,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
}
}
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
static void tmfclose(FILE *fp) {
if (NULL != fp) {
fclose(fp);
@ -1231,7 +1243,8 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
//printf("query result:%s\n", temp);
memcpy(databuf + totalLen, temp, len);
totalLen += len;
verbosePrint("%s() LN%d, totalLen: %"PRId64"\n", __func__, __LINE__, totalLen);
verbosePrint("%s() LN%d, totalLen: %"PRId64"\n",
__func__, __LINE__, totalLen);
}
verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
@ -1314,6 +1327,27 @@ static float rand_float(){
return randfloat[cursor];
}
static float demo_current_float(){
static int cursor;
cursor++;
cursor = cursor % MAX_PREPARED_RAND;
return (float)(5 + randfloat[cursor] / 1000000000);
}
static int32_t demo_voltage_int(){
static int cursor;
cursor++;
cursor = cursor % MAX_PREPARED_RAND;
return 210 + randint[cursor] % 20;
}
static float demo_phase_float(){
static int cursor;
cursor++;
cursor = cursor % MAX_PREPARED_RAND;
return (float)(120 + randfloat[cursor] / 1000000000);
}
#if 0
static const char charNum[] = "0123456789";
@ -1351,7 +1385,6 @@ static double rand_double() {
cursor++;
cursor = cursor % MAX_PREPARED_RAND;
return randdouble[cursor];
}
static void init_rand_data() {
@ -1386,6 +1419,11 @@ static void init_rand_data() {
static int printfInsertMeta() {
SHOW_PARSE_RESULT_START();
if (g_args.demo_mode)
printf("\ntaosdemo is simulating data generated by power equipments monitoring...\n\n");
else
printf("\ntaosdemo is simulating random data as you request..\n\n");
printf("interface: \033[33m%s\033[0m\n",
(g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
printf("host: \033[33m%s:%u\033[0m\n",
@ -1943,7 +1981,8 @@ static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
FILE* fp = fopen(fname, "at");
if (fp == NULL) {
errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
errorPrint("%s() LN%d, failed to open file: %s\n",
__func__, __LINE__, fname);
return -1;
}
@ -1966,7 +2005,8 @@ static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
if (i > 0) {
fputc(',', fp);
}
xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
xDumpFieldToFile(fp,
(const char*)row[i], fields +i, length[i], precision);
}
fputc('\n', fp);
@ -1988,7 +2028,8 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
int32_t code = taos_errno(res);
if (code != 0) {
errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
errorPrint( "failed to run <show databases>, reason: %s\n",
taos_errstr(res));
return -1;
}
@ -2318,8 +2359,10 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
int dataLen = 0;
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
for (int i = 0; i < stbInfo->tagCount; i++) {
if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
|| (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
if ((0 == strncasecmp(stbInfo->tags[i].dataType,
"binary", strlen("binary")))
|| (0 == strncasecmp(stbInfo->tags[i].dataType,
"nchar", strlen("nchar")))) {
if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
printf("binary or nchar length overflow, max size:%u\n",
(uint32_t)TSDB_MAX_BINARY_LEN);
@ -2664,13 +2707,20 @@ static int createSuperTable(
superTbl->columns[colIndex].dataLen);
lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
} else if (strcasecmp(dataType, "INT") == 0) {
if ((g_args.demo_mode) && (colIndex == 1)) {
len += snprintf(cols + len, STRING_LEN - len,
", VOLTAGE INT");
} else {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
}
lenOfOneRow += 11;
} else if (strcasecmp(dataType, "BIGINT") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
colIndex, "BIGINT");
lenOfOneRow += 21;
} else if (strcasecmp(dataType, "SMALLINT") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
colIndex, "SMALLINT");
lenOfOneRow += 6;
} else if (strcasecmp(dataType, "TINYINT") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
@ -2679,13 +2729,24 @@ static int createSuperTable(
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
lenOfOneRow += 6;
} else if (strcasecmp(dataType, "FLOAT") == 0) {
if (g_args.demo_mode) {
if (colIndex == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", CURRENT FLOAT");
} else if (colIndex == 2) {
len += snprintf(cols + len, STRING_LEN - len, ", PHASE FLOAT");
}
} else {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
}
lenOfOneRow += 22;
} else if (strcasecmp(dataType, "DOUBLE") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
colIndex, "DOUBLE");
lenOfOneRow += 42;
} else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
colIndex, "TIMESTAMP");
lenOfOneRow += 21;
} else {
taos_close(taos);
@ -2696,7 +2757,6 @@ static int createSuperTable(
}
superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
//printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
// save for creating child table
superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
@ -4746,7 +4806,8 @@ static void postFreeResource() {
static int getRowDataFromSample(
char* dataBuf, int64_t maxLen, int64_t timestamp,
SSuperTable* superTblInfo, int64_t* sampleUsePos) {
SSuperTable* superTblInfo, int64_t* sampleUsePos)
{
if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
/* int ret = readSampleFromCsvFileToMem(superTblInfo);
if (0 != ret) {
@ -4763,7 +4824,9 @@ static int getRowDataFromSample(
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
"(%" PRId64 ", ", timestamp);
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
"%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
"%s",
superTblInfo->sampleDataBuf
+ superTblInfo->lenOfOneRow * (*sampleUsePos));
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
(*sampleUsePos)++;
@ -4803,16 +4866,31 @@ static int64_t generateStbRowData(
tmfree(buf);
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"INT", strlen("INT"))) {
if ((g_args.demo_mode) && (i == 1)) {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%d,", demo_voltage_int());
} else {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%d,", rand_int());
}
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"BIGINT", strlen("BIGINT"))) {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%"PRId64",", rand_bigint());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"FLOAT", strlen("FLOAT"))) {
if (g_args.demo_mode) {
if (i == 0) {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%f,", demo_current_float());
} else {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%f,", demo_phase_float());
}
} else {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%f,", rand_float());
}
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"DOUBLE", strlen("DOUBLE"))) {
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,

2
tests/pytest/tools/taosdemoTest.py
View File

@ -51,7 +51,7 @@ class TDTestCase:
else:
tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
os.system("%staosdemo -y -t %d -n %d" %
os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT" %
(binPath, self.numberOfTables, self.numberOfRecords))
tdSql.execute("use test")

2
tests/pytest/tools/taosdemoTestTblAlt.py
View File

@ -54,7 +54,7 @@ class TDTestCase:
binPath = buildPath + "/build/bin/"
if(threadID == 0):
os.system("%staosdemo -y -t %d -n %d" %
os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" %
(binPath, self.numberOfTables, self.numberOfRecords))
if(threadID == 1):
time.sleep(2)

2
tests/pytest/tools/taosdemoTestWithoutMetric.py
View File

@ -60,7 +60,7 @@ class TDTestCase:
tdSql.execute("use test")
tdSql.query(
"select count(*) from test.t%d" % (self.numberOfTables -1))
"select count(*) from test.d%d" % (self.numberOfTables -1))
tdSql.checkData(0, 0, self.numberOfRecords)
def stop(self):