xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into xiaoping/add_test_case

This commit is contained in:
Ping Xiao 2021-04-03 22:22:24 +08:00
parent 8904a1644f a8d53d1f90
commit 58eac46929
17 changed files with 637 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
documentation20/cn/02.getting-started/docs.md
View File

@ -101,7 +101,7 @@ $ taos -h 192.168.0.1 -s "use db; show tables;"
### 运行SQL命令脚本 ### 运行SQL命令脚本
TDengine终端可以通过`source`命令来运行SQL命令脚本. TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本.
```mysql ```mysql
taos> source <filename>; taos> source <filename>;
@ -109,10 +109,10 @@ taos> source <filename>;
### Shell小技巧 ### Shell小技巧
- 可以使用上下光标键查看已经历史输入的命令 - 可以使用上下光标键查看历史输入的指令
- 修改用户密码。在shell中使用alter user命令 - 修改用户密码。在 shell 中使用 alter user 指令
- ctrl+c 中止正在进行中的查询 - ctrl+c 中止正在进行中的查询
- 执行`RESET QUERY CACHE`清空本地缓存的表的schema - 执行 `RESET QUERY CACHE` 清空本地缓存的表 schema
## <a class="anchor" id="demo"></a>TDengine 极速体验 ## <a class="anchor" id="demo"></a>TDengine 极速体验

2
src/dnode/src/dnodeVWrite.c
View File

@ -222,7 +222,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
} else { } else {
if (qtype == TAOS_QTYPE_FWD) { if (qtype == TAOS_QTYPE_FWD) {
vnodeConfirmForward(pVnode, pWrite->pHead.version, 0, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); vnodeConfirmForward(pVnode, pWrite->pHead.version, pWrite->code, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT);
} }
if (pWrite->rspRet.rsp) { if (pWrite->rspRet.rsp) {
rpcFreeCont(pWrite->rspRet.rsp); rpcFreeCont(pWrite->rspRet.rsp);

560
src/kit/taosdemo/taosdemo.c
View File

@ -211,8 +211,8 @@ typedef struct SArguments_S {
int num_of_tables; int num_of_tables;
int num_of_DPT; int num_of_DPT;
int abort; int abort;
int disorderRatio; int disorderRatio; // 0: no disorder, >0: x%
int disorderRange; int disorderRange; // ms or us by database precision
int method_of_delete; int method_of_delete;
char ** arg_list; char ** arg_list;
int64_t totalInsertRows; int64_t totalInsertRows;
@ -229,25 +229,25 @@ typedef struct SColumn_S {
typedef struct SSuperTable_S { typedef struct SSuperTable_S {
char sTblName[MAX_TB_NAME_SIZE+1]; char sTblName[MAX_TB_NAME_SIZE+1];
int childTblCount; int childTblCount;
bool childTblExists; // 0: no, 1: yes bool childTblExists; // 0: no, 1: yes
int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql
int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table
char childTblPrefix[MAX_TB_NAME_SIZE]; char childTblPrefix[MAX_TB_NAME_SIZE];
char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample
char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful
int childTblLimit; int childTblLimit;
int childTblOffset; int childTblOffset;
int multiThreadWriteOneTbl; // 0: no, 1: yes int multiThreadWriteOneTbl; // 0: no, 1: yes
int interlaceRows; // int interlaceRows; //
int disorderRatio; // 0: no disorder, >0: x% int disorderRatio; // 0: no disorder, >0: x%
int disorderRange; // ms or us by database precision int disorderRange; // ms or us by database precision
int maxSqlLen; // int maxSqlLen; //
int insertInterval; // insert interval, will override global insert interval int insertInterval; // insert interval, will override global insert interval
int64_t insertRows; // 0: no limit int64_t insertRows; // 0: no limit
int timeStampStep; int timeStampStep;
char startTimestamp[MAX_TB_NAME_SIZE]; // char startTimestamp[MAX_TB_NAME_SIZE];
char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json
char sampleFile[MAX_FILE_NAME_LEN+1]; char sampleFile[MAX_FILE_NAME_LEN+1];
char tagsFile[MAX_FILE_NAME_LEN+1]; char tagsFile[MAX_FILE_NAME_LEN+1];
@ -263,7 +263,6 @@ typedef struct SSuperTable_S {
int lenOfTagOfOneRow; int lenOfTagOfOneRow;
char* sampleDataBuf; char* sampleDataBuf;
int sampleDataBufSize;
//int sampleRowCount; //int sampleRowCount;
//int sampleUsePos; //int sampleUsePos;
@ -488,7 +487,7 @@ static int taosRandom()
return number; return number;
} }
#else #else // Not windows
static void setupForAnsiEscape(void) {} static void setupForAnsiEscape(void) {}
static void resetAfterAnsiEscape(void) { static void resetAfterAnsiEscape(void) {
@ -500,11 +499,15 @@ static void resetAfterAnsiEscape(void) {
static int taosRandom() static int taosRandom()
{ {
srand(time(NULL)); struct timeval tv;
gettimeofday(&tv, NULL);
srand(tv.tv_usec);
return rand(); return rand();
} }
#endif #endif // ifdef Windows
static int createDatabasesAndStables(); static int createDatabasesAndStables();
static void createChildTables(); static void createChildTables();
@ -677,7 +680,7 @@ static void printHelp() {
printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
printf("%s%s%s%s\n", indent, "-O", indent, printf("%s%s%s%s\n", indent, "-O", indent,
"Insert mode--0: In order, > 0: disorder ratio. Default is in order."); "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
printf("%s%s%s%s\n", indent, "-R", indent, printf("%s%s%s%s\n", indent, "-R", indent,
"Out of order data's range, ms, default is 1000."); "Out of order data's range, ms, default is 1000.");
printf("%s%s%s%s\n", indent, "-g", indent, printf("%s%s%s%s\n", indent, "-g", indent,
@ -801,20 +804,21 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
} else if (strcmp(argv[i], "-c") == 0) { } else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]); strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-O") == 0) { } else if (strcmp(argv[i], "-O") == 0) {
arguments->disorderRatio = atoi(argv[++i]); arguments->disorderRatio = atoi(argv[++i]);
if (arguments->disorderRatio > 1
|| arguments->disorderRatio < 0) { if (arguments->disorderRatio > 50)
arguments->disorderRatio = 50;
if (arguments->disorderRatio < 0)
arguments->disorderRatio = 0; arguments->disorderRatio = 0;
} else if (arguments->disorderRatio == 1) {
arguments->disorderRange = 10;
}
} else if (strcmp(argv[i], "-R") == 0) { } else if (strcmp(argv[i], "-R") == 0) {
arguments->disorderRange = atoi(argv[++i]); arguments->disorderRange = atoi(argv[++i]);
if (arguments->disorderRange == 1 if (arguments->disorderRange < 0)
&& (arguments->disorderRange > 50 arguments->disorderRange = 1000;
|| arguments->disorderRange <= 0)) {
arguments->disorderRange = 10;
}
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
arguments->replica = atoi(argv[++i]); arguments->replica = atoi(argv[++i]);
if (arguments->replica > 3 || arguments->replica < 1) { if (arguments->replica > 3 || arguments->replica < 1) {
@ -998,8 +1002,9 @@ static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName)
taos_free_result(res); taos_free_result(res);
} }
static double getCurrentTime() { static double getCurrentTimeUs() {
struct timeval tv; struct timeval tv;
if (gettimeofday(&tv, NULL) != 0) { if (gettimeofday(&tv, NULL) != 0) {
perror("Failed to get current time in ms"); perror("Failed to get current time in ms");
return 0.0; return 0.0;
@ -2279,7 +2284,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName,
} }
static int createSuperTable(TAOS * taos, char* dbName, static int createSuperTable(TAOS * taos, char* dbName,
SSuperTable* superTbls, bool use_metric) { SSuperTable* superTbl) {
char command[BUFFER_SIZE] = "\0"; char command[BUFFER_SIZE] = "\0";
char cols[STRING_LEN] = "\0"; char cols[STRING_LEN] = "\0";
@ -2287,19 +2292,26 @@ static int createSuperTable(TAOS * taos, char* dbName,
int len = 0; int len = 0;
int lenOfOneRow = 0; int lenOfOneRow = 0;
for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
char* dataType = superTbls->columns[colIndex].dataType; if (superTbl->columnCount == 0) {
errorPrint("%s() LN%d, super table column count is %d\n",
__func__, __LINE__, superTbl->columnCount);
return -1;
}
for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
char* dataType = superTbl->columns[colIndex].dataType;
if (strcasecmp(dataType, "BINARY") == 0) { if (strcasecmp(dataType, "BINARY") == 0) {
len += snprintf(cols + len, STRING_LEN - len, len += snprintf(cols + len, STRING_LEN - len,
", col%d %s(%d)", colIndex, "BINARY", ", col%d %s(%d)", colIndex, "BINARY",
superTbls->columns[colIndex].dataLen); superTbl->columns[colIndex].dataLen);
lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
} else if (strcasecmp(dataType, "NCHAR") == 0) { } else if (strcasecmp(dataType, "NCHAR") == 0) {
len += snprintf(cols + len, STRING_LEN - len, len += snprintf(cols + len, STRING_LEN - len,
", col%d %s(%d)", colIndex, "NCHAR", ", col%d %s(%d)", colIndex, "NCHAR",
superTbls->columns[colIndex].dataLen); superTbl->columns[colIndex].dataLen);
lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
} else if (strcasecmp(dataType, "INT") == 0) { } else if (strcasecmp(dataType, "INT") == 0) {
len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
lenOfOneRow += 11; lenOfOneRow += 11;
@ -2331,88 +2343,95 @@ static int createSuperTable(TAOS * taos, char* dbName,
} }
} }
superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
//printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
// save for creating child table // save for creating child table
superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1); superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
if (NULL == superTbls->colsOfCreateChildTable) { if (NULL == superTbl->colsOfCreateChildTable) {
printf("Failed when calloc, size:%d", len+1); errorPrint("%s() LN%d, Failed when calloc, size:%d",
__func__, __LINE__, len+1);
taos_close(taos); taos_close(taos);
exit(-1); exit(-1);
} }
snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable);
if (use_metric) { snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
char tags[STRING_LEN] = "\0"; verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbl->colsOfCreateChildTable);
int tagIndex;
len = 0;
int lenOfTagOfOneRow = 0; if (superTbl->tagCount == 0) {
len += snprintf(tags + len, STRING_LEN - len, "("); errorPrint("%s() LN%d, super table tag count is %d\n",
for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { __func__, __LINE__, superTbl->tagCount);
char* dataType = superTbls->tags[tagIndex].dataType; return -1;
if (strcasecmp(dataType, "BINARY") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
"BINARY", superTbls->tags[tagIndex].dataLen);
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
} else if (strcasecmp(dataType, "NCHAR") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
"NCHAR", superTbls->tags[tagIndex].dataLen);
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
} else if (strcasecmp(dataType, "INT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"INT");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
} else if (strcasecmp(dataType, "BIGINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"BIGINT");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
} else if (strcasecmp(dataType, "SMALLINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"SMALLINT");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
} else if (strcasecmp(dataType, "TINYINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"TINYINT");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
} else if (strcasecmp(dataType, "BOOL") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"BOOL");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
} else if (strcasecmp(dataType, "FLOAT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"FLOAT");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
} else if (strcasecmp(dataType, "DOUBLE") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"DOUBLE");
lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
} else {
taos_close(taos);
printf("config error tag type : %s\n", dataType);
exit(-1);
}
}
len -= 2;
len += snprintf(tags + len, STRING_LEN - len, ")");
superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
snprintf(command, BUFFER_SIZE,
"create table if not exists %s.%s (ts timestamp%s) tags %s",
dbName, superTbls->sTblName, cols, tags);
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);
if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
errorPrint( "create supertable %s failed!\n\n",
superTbls->sTblName);
return -1;
}
debugPrint("create supertable %s success!\n\n", superTbls->sTblName);
} }
char tags[STRING_LEN] = "\0";
int tagIndex;
len = 0;
int lenOfTagOfOneRow = 0;
len += snprintf(tags + len, STRING_LEN - len, "(");
for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
char* dataType = superTbl->tags[tagIndex].dataType;
if (strcasecmp(dataType, "BINARY") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
"BINARY", superTbl->tags[tagIndex].dataLen);
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
} else if (strcasecmp(dataType, "NCHAR") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
"NCHAR", superTbl->tags[tagIndex].dataLen);
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
} else if (strcasecmp(dataType, "INT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"INT");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
} else if (strcasecmp(dataType, "BIGINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"BIGINT");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
} else if (strcasecmp(dataType, "SMALLINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"SMALLINT");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
} else if (strcasecmp(dataType, "TINYINT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"TINYINT");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
} else if (strcasecmp(dataType, "BOOL") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"BOOL");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
} else if (strcasecmp(dataType, "FLOAT") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"FLOAT");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
} else if (strcasecmp(dataType, "DOUBLE") == 0) {
len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
"DOUBLE");
lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
} else {
taos_close(taos);
printf("config error tag type : %s\n", dataType);
exit(-1);
}
}
len -= 2;
len += snprintf(tags + len, STRING_LEN - len, ")");
superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;
snprintf(command, BUFFER_SIZE,
"create table if not exists %s.%s (ts timestamp%s) tags %s",
dbName, superTbl->sTblName, cols, tags);
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);
if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
errorPrint( "create supertable %s failed!\n\n",
superTbl->sTblName);
return -1;
}
debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
return 0; return 0;
} }
@ -2434,85 +2453,88 @@ static int createDatabasesAndStables() {
taos_close(taos); taos_close(taos);
return -1; return -1;
} }
}
int dataLen = 0; int dataLen = 0;
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
if (g_Dbs.db[i].dbCfg.blocks > 0) { if (g_Dbs.db[i].dbCfg.blocks > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
} }
if (g_Dbs.db[i].dbCfg.cache > 0) { if (g_Dbs.db[i].dbCfg.cache > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
} }
if (g_Dbs.db[i].dbCfg.days > 0) { if (g_Dbs.db[i].dbCfg.days > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
} }
if (g_Dbs.db[i].dbCfg.keep > 0) { if (g_Dbs.db[i].dbCfg.keep > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
} }
if (g_Dbs.db[i].dbCfg.quorum > 1) { if (g_Dbs.db[i].dbCfg.quorum > 1) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
} }
if (g_Dbs.db[i].dbCfg.replica > 0) { if (g_Dbs.db[i].dbCfg.replica > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
} }
if (g_Dbs.db[i].dbCfg.update > 0) { if (g_Dbs.db[i].dbCfg.update > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
} }
//if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
// dataLen += snprintf(command + dataLen, // dataLen += snprintf(command + dataLen,
// BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
//} //}
if (g_Dbs.db[i].dbCfg.minRows > 0) { if (g_Dbs.db[i].dbCfg.minRows > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
} }
if (g_Dbs.db[i].dbCfg.maxRows > 0) { if (g_Dbs.db[i].dbCfg.maxRows > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
} }
if (g_Dbs.db[i].dbCfg.comp > 0) { if (g_Dbs.db[i].dbCfg.comp > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
} }
if (g_Dbs.db[i].dbCfg.walLevel > 0) { if (g_Dbs.db[i].dbCfg.walLevel > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
} }
if (g_Dbs.db[i].dbCfg.cacheLast > 0) { if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
dataLen += snprintf(command + dataLen, dataLen += snprintf(command + dataLen,
BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
} }
if (g_Dbs.db[i].dbCfg.fsync > 0) { if (g_Dbs.db[i].dbCfg.fsync > 0) {
dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
" fsync %d", g_Dbs.db[i].dbCfg.fsync); " fsync %d", g_Dbs.db[i].dbCfg.fsync);
} }
if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
|| (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
"us", strlen("us")))) { "us", strlen("us")))) {
dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
" precision \'%s\';", g_Dbs.db[i].dbCfg.precision); " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
} }
debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
taos_close(taos); taos_close(taos);
errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
return -1; return -1;
}
printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
} }
printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
debugPrint("%s() %d supertbl count:%d\n", debugPrint("%s() %d supertbl count:%d\n",
__func__, __LINE__, g_Dbs.db[i].superTblCount); __func__, __LINE__, g_Dbs.db[i].superTblCount);
int validStbCount = 0;
for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
g_Dbs.db[i].superTbls[j].sTblName); g_Dbs.db[i].superTbls[j].sTblName);
@ -2522,12 +2544,11 @@ static int createDatabasesAndStables() {
if ((ret != 0) || (g_Dbs.db[i].drop)) { if ((ret != 0) || (g_Dbs.db[i].drop)) {
ret = createSuperTable(taos, g_Dbs.db[i].dbName, ret = createSuperTable(taos, g_Dbs.db[i].dbName,
&g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); &g_Dbs.db[i].superTbls[j]);
if (0 != ret) { if (0 != ret) {
errorPrint("\ncreate super table %d failed!\n\n", j); errorPrint("create super table %d failed!\n\n", j);
taos_close(taos); continue;
return -1;
} }
} }
@ -2536,10 +2557,13 @@ static int createDatabasesAndStables() {
if (0 != ret) { if (0 != ret) {
errorPrint("\nget super table %s.%s info failed!\n\n", errorPrint("\nget super table %s.%s info failed!\n\n",
g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
taos_close(taos); continue;
return -1;
} }
validStbCount ++;
} }
g_Dbs.db[i].superTblCount = validStbCount;
} }
taos_close(taos); taos_close(taos);
@ -2724,27 +2748,29 @@ static void createChildTables() {
int len; int len;
for (int i = 0; i < g_Dbs.dbCount; i++) { for (int i = 0; i < g_Dbs.dbCount; i++) {
if (g_Dbs.db[i].superTblCount > 0) { if (g_Dbs.use_metric) {
// with super table if (g_Dbs.db[i].superTblCount > 0) {
for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // with super table
if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
|| (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
continue; || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
continue;
}
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
int startFrom = 0;
g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
verbosePrint("%s() LN%d: create %d child tables from %d\n",
__func__, __LINE__, g_totalChildTables, startFrom);
startMultiThreadCreateChildTable(
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
g_Dbs.threadCountByCreateTbl,
startFrom,
g_Dbs.db[i].superTbls[j].childTblCount,
g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
} }
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
int startFrom = 0;
g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
verbosePrint("%s() LN%d: create %d child tables from %d\n",
__func__, __LINE__, g_totalChildTables, startFrom);
startMultiThreadCreateChildTable(
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
g_Dbs.threadCountByCreateTbl,
startFrom,
g_Dbs.db[i].superTbls[j].childTblCount,
g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
} }
} else { } else {
// normal table // normal table
@ -3552,19 +3578,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size");
if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) {
g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint;
if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) {
g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
}
} else if (!sampleDataBufSize) {
g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
} else {
printf("ERROR: failed to read json, sample_buf_size not found\n");
goto PARSE_OVER;
}
cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
if (sampleFormat && sampleFormat->type if (sampleFormat && sampleFormat->type
== cJSON_String && sampleFormat->valuestring != NULL) { == cJSON_String && sampleFormat->valuestring != NULL) {
@ -3662,6 +3675,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
if (disorderRatio && disorderRatio->type == cJSON_Number) { if (disorderRatio && disorderRatio->type == cJSON_Number) {
if (disorderRatio->valueint > 50)
disorderRatio->valueint = 50;
if (disorderRatio->valueint < 0)
disorderRatio->valueint = 0;
g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
} else if (!disorderRatio) { } else if (!disorderRatio) {
g_Dbs.db[i].superTbls[j].disorderRatio = 0; g_Dbs.db[i].superTbls[j].disorderRatio = 0;
@ -4322,6 +4341,8 @@ static int32_t generateData(char *recBuf, char **data_type,
pstr += sprintf(pstr, ")"); pstr += sprintf(pstr, ")");
verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);
return (int32_t)strlen(recBuf); return (int32_t)strlen(recBuf);
} }
@ -4433,7 +4454,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
pSamplePos); pSamplePos);
} else if (0 == strncasecmp(superTblInfo->dataSource, } else if (0 == strncasecmp(superTblInfo->dataSource,
"rand", strlen("rand"))) { "rand", strlen("rand"))) {
int rand_num = rand_tinyint() % 100; int rand_num = taosRandom() % 100;
if (0 != superTblInfo->disorderRatio if (0 != superTblInfo->disorderRatio
&& rand_num < superTblInfo->disorderRatio) { && rand_num < superTblInfo->disorderRatio) {
int64_t d = startTime int64_t d = startTime
@ -4460,15 +4481,16 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
len += retLen; len += retLen;
remainderBufLen -= retLen; remainderBufLen -= retLen;
} else { } else {
int rand_num = taosRandom() % 100;
char **data_type = g_args.datatype; char **data_type = g_args.datatype;
int lenOfBinary = g_args.len_of_binary; int lenOfBinary = g_args.len_of_binary;
int rand_num = taosRandom() % 100;
if ((g_args.disorderRatio != 0) if ((g_args.disorderRatio != 0)
&& (rand_num < g_args.disorderRange)) { && (rand_num < g_args.disorderRatio)) {
int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k
- taosRandom() % 1000000 + rand_num; - taosRandom() % g_args.disorderRange;
retLen = generateData(data, data_type, retLen = generateData(data, data_type,
ncols_per_record, d, lenOfBinary); ncols_per_record, d, lenOfBinary);
} else { } else {
@ -5017,7 +5039,7 @@ static void callBack(void *param, TAOS_RES *res, int code) {
int rand_num = taosRandom() % 100; int rand_num = taosRandom() % 100;
if (0 != winfo->superTblInfo->disorderRatio if (0 != winfo->superTblInfo->disorderRatio
&& rand_num < winfo->superTblInfo->disorderRatio) { && rand_num < winfo->superTblInfo->disorderRatio) {
int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num; int64_t d = winfo->lastTs - taosRandom() % winfo->superTblInfo->disorderRange;
generateRowData(data, d, winfo->superTblInfo); generateRowData(data, d, winfo->superTblInfo);
} else { } else {
generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo);
@ -5109,7 +5131,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
start_time = 1500000000000; start_time = 1500000000000;
} }
double start = getCurrentTime(); double start = getCurrentTimeUs();
// read sample data from file first // read sample data from file first
if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
@ -5146,16 +5168,14 @@ static void startMultiThreadInsertData(int threads, char* db_name,
if (superTblInfo) { if (superTblInfo) {
int limit, offset; int limit, offset;
if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) { if ((superTblInfo->childTblExists == TBL_NO_EXISTS) &&
printf("WARNING: specified offset >= child table count! \n"); ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) {
if (!g_args.answer_yes) { printf("WARNING: offset and limit will not be used since the child tables are not exists!\n");
printf(" Press enter key to continue or Ctrl-C to stop\n\n");
(void)getchar();
}
} }
if (superTblInfo->childTblOffset >= 0) { if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS)
if (superTblInfo->childTblLimit <= 0) { && (superTblInfo->childTblOffset >= 0)) {
if (superTblInfo->childTblLimit < 0) {
superTblInfo->childTblLimit = superTblInfo->childTblLimit =
superTblInfo->childTblCount - superTblInfo->childTblOffset; superTblInfo->childTblCount - superTblInfo->childTblOffset;
} }
@ -5163,13 +5183,32 @@ static void startMultiThreadInsertData(int threads, char* db_name,
offset = superTblInfo->childTblOffset; offset = superTblInfo->childTblOffset;
limit = superTblInfo->childTblLimit; limit = superTblInfo->childTblLimit;
} else { } else {
limit = superTblInfo->childTblCount; limit = superTblInfo->childTblCount;
offset = 0; offset = 0;
} }
ntables = limit; ntables = limit;
startFrom = offset; startFrom = offset;
if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
&& ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
> superTblInfo->childTblCount)) {
printf("WARNING: specified offset + limit > child table count!\n");
if (!g_args.answer_yes) {
printf(" Press enter key to continue or Ctrl-C to stop\n\n");
(void)getchar();
}
}
if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
&& (0 == superTblInfo->childTblLimit)) {
printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
if (!g_args.answer_yes) {
printf(" Press enter key to continue or Ctrl-C to stop\n\n");
(void)getchar();
}
}
superTblInfo->childTblName = (char*)calloc(1, superTblInfo->childTblName = (char*)calloc(1,
limit * TSDB_TABLE_NAME_LEN); limit * TSDB_TABLE_NAME_LEN);
if (superTblInfo->childTblName == NULL) { if (superTblInfo->childTblName == NULL) {
@ -5287,7 +5326,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
if (cntDelay == 0) cntDelay = 1; if (cntDelay == 0) cntDelay = 1;
avgDelay = (double)totalDelay / cntDelay; avgDelay = (double)totalDelay / cntDelay;
double end = getCurrentTime(); double end = getCurrentTimeUs();
double t = end - start; double t = end - start;
if (superTblInfo) { if (superTblInfo) {
@ -5366,7 +5405,7 @@ static void *readTable(void *sarg) {
sprintf(command, "select %s from %s%d where ts>= %" PRId64, sprintf(command, "select %s from %s%d where ts>= %" PRId64,
aggreFunc[j], tb_prefix, i, sTime); aggreFunc[j], tb_prefix, i, sTime);
double t = getCurrentTime(); double t = getCurrentTimeUs();
TAOS_RES *pSql = taos_query(taos, command); TAOS_RES *pSql = taos_query(taos, command);
int32_t code = taos_errno(pSql); int32_t code = taos_errno(pSql);
@ -5382,7 +5421,7 @@ static void *readTable(void *sarg) {
count++; count++;
} }
t = getCurrentTime() - t; t = getCurrentTimeUs() - t;
totalT += t; totalT += t;
taos_free_result(pSql); taos_free_result(pSql);
@ -5441,7 +5480,7 @@ static void *readMetric(void *sarg) {
printf("Where condition: %s\n", condition); printf("Where condition: %s\n", condition);
fprintf(fp, "%s\n", command); fprintf(fp, "%s\n", command);
double t = getCurrentTime(); double t = getCurrentTimeUs();
TAOS_RES *pSql = taos_query(taos, command); TAOS_RES *pSql = taos_query(taos, command);
int32_t code = taos_errno(pSql); int32_t code = taos_errno(pSql);
@ -5457,7 +5496,7 @@ static void *readMetric(void *sarg) {
while (taos_fetch_row(pSql) != NULL) { while (taos_fetch_row(pSql) != NULL) {
count++; count++;
} }
t = getCurrentTime() - t; t = getCurrentTimeUs() - t;
fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
num_of_tables * num_of_DPT / t, t * 1000); num_of_tables * num_of_DPT / t, t * 1000);
@ -5511,9 +5550,9 @@ static int insertTestProcess() {
double end; double end;
// create child tables // create child tables
start = getCurrentTime(); start = getCurrentTimeUs();
createChildTables(); createChildTables();
end = getCurrentTime(); end = getCurrentTimeUs();
if (g_totalChildTables > 0) { if (g_totalChildTables > 0) {
printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
@ -5525,20 +5564,23 @@ static int insertTestProcess() {
taosMsleep(1000); taosMsleep(1000);
// create sub threads for inserting data // create sub threads for inserting data
//start = getCurrentTime(); //start = getCurrentTimeUs();
for (int i = 0; i < g_Dbs.dbCount; i++) { for (int i = 0; i < g_Dbs.dbCount; i++) {
if (g_Dbs.db[i].superTblCount > 0) { if (g_Dbs.use_metric) {
for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { if (g_Dbs.db[i].superTblCount > 0) {
SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
if (0 == g_Dbs.db[i].superTbls[j].insertRows) {
continue; SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];
}
startMultiThreadInsertData( if (superTblInfo && (superTblInfo->insertRows > 0)) {
g_Dbs.threadCount, startMultiThreadInsertData(
g_Dbs.db[i].dbName, g_Dbs.threadCount,
g_Dbs.db[i].dbCfg.precision, g_Dbs.db[i].dbName,
superTblInfo); g_Dbs.db[i].dbCfg.precision,
superTblInfo);
}
} }
}
} else { } else {
startMultiThreadInsertData( startMultiThreadInsertData(
g_Dbs.threadCount, g_Dbs.threadCount,
@ -5547,7 +5589,7 @@ static int insertTestProcess() {
NULL); NULL);
} }
} }
//end = getCurrentTime(); //end = getCurrentTimeUs();
//int64_t totalInsertRows = 0; //int64_t totalInsertRows = 0;
//int64_t totalAffectedRows = 0; //int64_t totalAffectedRows = 0;
@ -6368,7 +6410,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile)
char * line = NULL; char * line = NULL;
size_t line_len = 0; size_t line_len = 0;
double t = getCurrentTime(); double t = getCurrentTimeUs();
while ((read_len = tgetline(&line, &line_len, fp)) != -1) { while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
if (read_len >= MAX_SQL_SIZE) continue; if (read_len >= MAX_SQL_SIZE) continue;
@ -6399,7 +6441,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile)
cmd_len = 0; cmd_len = 0;
} }
t = getCurrentTime() - t; t = getCurrentTimeUs() - t;
printf("run %s took %.6f second(s)\n\n", sqlFile, t); printf("run %s took %.6f second(s)\n\n", sqlFile, t);
tmfree(cmd); tmfree(cmd);

305
src/kit/taosdump/taosdump.c
View File

@ -39,6 +39,22 @@ typedef struct {
int8_t type; int8_t type;
} SOColInfo; } SOColInfo;
#define debugPrint(fmt, ...) \
do { if (g_args.debug_print || g_args.verbose_print) \
fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
#define verbosePrint(fmt, ...) \
do { if (g_args.verbose_print) \
fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
#define performancePrint(fmt, ...) \
do { if (g_args.performance_print) \
fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
#define errorPrint(fmt, ...) \
do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)
// -------------------------- SHOW DATABASE INTERFACE----------------------- // -------------------------- SHOW DATABASE INTERFACE-----------------------
enum _show_db_index { enum _show_db_index {
TSDB_SHOW_DB_NAME_INDEX, TSDB_SHOW_DB_NAME_INDEX,
@ -46,7 +62,7 @@ enum _show_db_index {
TSDB_SHOW_DB_NTABLES_INDEX, TSDB_SHOW_DB_NTABLES_INDEX,
TSDB_SHOW_DB_VGROUPS_INDEX, TSDB_SHOW_DB_VGROUPS_INDEX,
TSDB_SHOW_DB_REPLICA_INDEX, TSDB_SHOW_DB_REPLICA_INDEX,
TSDB_SHOW_DB_QUORUM_INDEX, TSDB_SHOW_DB_QUORUM_INDEX,
TSDB_SHOW_DB_DAYS_INDEX, TSDB_SHOW_DB_DAYS_INDEX,
TSDB_SHOW_DB_KEEP_INDEX, TSDB_SHOW_DB_KEEP_INDEX,
TSDB_SHOW_DB_CACHE_INDEX, TSDB_SHOW_DB_CACHE_INDEX,
@ -101,10 +117,10 @@ typedef struct {
char name[TSDB_DB_NAME_LEN + 1]; char name[TSDB_DB_NAME_LEN + 1];
char create_time[32]; char create_time[32];
int32_t ntables; int32_t ntables;
int32_t vgroups; int32_t vgroups;
int16_t replica; int16_t replica;
int16_t quorum; int16_t quorum;
int16_t days; int16_t days;
char keeplist[32]; char keeplist[32];
//int16_t daysToKeep; //int16_t daysToKeep;
//int16_t daysToKeep1; //int16_t daysToKeep1;
@ -172,48 +188,50 @@ static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-dat
/* The options we understand. */ /* The options we understand. */
static struct argp_option options[] = { static struct argp_option options[] = {
// connection option // connection option
{"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0},
{"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0},
#ifdef _TD_POWER_ #ifdef _TD_POWER_
{"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0},
#else #else
{"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0},
#endif #endif
{"port", 'P', "PORT", 0, "Port to connect", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0},
{"cversion", 'v', "CVERION", 0, "client version", 0}, {"cversion", 'v', "CVERION", 0, "client version", 0},
{"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0},
// input/output file // input/output file
{"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1},
{"inpath", 'i', "INPATH", 0, "Input file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1},
{"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1},
#ifdef _TD_POWER_ #ifdef _TD_POWER_
{"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1},
#else #else
{"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1},
#endif #endif
{"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1},
// dump unit options // dump unit options
{"all-databases", 'A', 0, 0, "Dump all databases.", 2}, {"all-databases", 'A', 0, 0, "Dump all databases.", 2},
{"databases", 'B', 0, 0, "Dump assigned databases", 2}, {"databases", 'B', 0, 0, "Dump assigned databases", 2},
// dump format options // dump format options
{"schemaonly", 's', 0, 0, "Only dump schema.", 3}, {"schemaonly", 's', 0, 0, "Only dump schema.", 3},
{"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, {"with-property", 'M', 0, 0, "Dump schema with properties.", 3},
{"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3},
{"end-time", 'E', "END_TIME", 0, "End time to dump.", 3}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Epoch or ISO8601/RFC3339 format is acceptable. For example: 2017-10-01T18:00:00+0800", 3},
{"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3},
{"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3},
{"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3},
{"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3},
{"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3},
{"debug", 'g', 0, 0, "Print debug info.", 1},
{"verbose", 'v', 0, 0, "Print verbose debug info.", 1},
{0}}; {0}};
/* Used by main to communicate with parse_opt. */ /* Used by main to communicate with parse_opt. */
struct arguments { typedef struct arguments {
// connection option // connection option
char *host; char *host;
char *user; char *user;
char *password; char *password;
uint16_t port; uint16_t port;
char cversion[12]; char cversion[12];
uint16_t mysqlFlag; uint16_t mysqlFlag;
// output file // output file
@ -238,9 +256,12 @@ struct arguments {
int32_t thread_num; int32_t thread_num;
int abort; int abort;
char **arg_list; char **arg_list;
int arg_list_len; int arg_list_len;
bool isDumpIn; bool isDumpIn;
}; bool debug_print;
bool verbose_print;
bool performance_print;
} SArguments;
/* Parse a single option. */ /* Parse a single option. */
static error_t parse_opt(int key, char *arg, struct argp_state *state) { static error_t parse_opt(int key, char *arg, struct argp_state *state) {
@ -286,6 +307,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path); wordfree(&full_path);
break; break;
case 'g':
arguments->debug_print = true;
break;
case 'i': case 'i':
arguments->isDumpIn = true; arguments->isDumpIn = true;
if (wordexp(arg, &full_path, 0) != 0) { if (wordexp(arg, &full_path, 0) != 0) {
@ -387,7 +411,7 @@ int taosCheckParam(struct arguments *arguments);
void taosFreeDbInfos(); void taosFreeDbInfos();
static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName);
struct arguments tsArguments = { struct arguments g_args = {
// connection option // connection option
NULL, NULL,
"root", "root",
@ -400,18 +424,18 @@ struct arguments tsArguments = {
"", "",
0, 0,
// outpath and inpath // outpath and inpath
"", "",
"", "",
"./dump_result.txt", "./dump_result.txt",
NULL, NULL,
// dump unit option // dump unit option
false, false,
false, false,
// dump format option // dump format option
false, false,
false, false,
0, 0,
INT64_MAX, INT64_MAX,
1, 1,
TSDB_MAX_SQL_LEN, TSDB_MAX_SQL_LEN,
1, 1,
@ -419,11 +443,14 @@ struct arguments tsArguments = {
// other options // other options
5, 5,
0, 0,
NULL, NULL,
0, 0,
false false,
false, // debug_print
false, // verbose_print
false // performance_print
}; };
static int queryDbImpl(TAOS *taos, char *command) { static int queryDbImpl(TAOS *taos, char *command) {
int i; int i;
TAOS_RES *res = NULL; TAOS_RES *res = NULL;
@ -434,7 +461,7 @@ static int queryDbImpl(TAOS *taos, char *command) {
taos_free_result(res); taos_free_result(res);
res = NULL; res = NULL;
} }
res = taos_query(taos, command); res = taos_query(taos, command);
code = taos_errno(res); code = taos_errno(res);
if (0 == code) { if (0 == code) {
@ -453,13 +480,40 @@ static int queryDbImpl(TAOS *taos, char *command) {
return 0; return 0;
} }
static void parse_args(int argc, char *argv[], SArguments *arguments) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-E") == 0) {
char *tmp = argv[++i];
int64_t tmpEpoch;
if (strchr(tmp, ':') && strchr(tmp, '-')) {
if (TSDB_CODE_SUCCESS != taosParseTime(
tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) {
fprintf(stderr, "Input end time error!\n");
return;
}
} else {
tmpEpoch = atoll(tmp);
}
sprintf(argv[i], "%"PRId64"", tmpEpoch);
debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n",
__func__, __LINE__, tmp, i, argv[i]);
} else if (strcmp(argv[i], "-g") == 0) {
arguments->debug_print = true;
}
}
}
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
/* Parse our arguments; every option seen by parse_opt will be /* Parse our arguments; every option seen by parse_opt will be
reflected in arguments. */ reflected in arguments. */
argp_parse(&argp, argc, argv, 0, 0, &tsArguments); parse_args(argc, argv, &g_args);
if (tsArguments.abort) { argp_parse(&argp, argc, argv, 0, 0, &g_args);
if (g_args.abort) {
#ifndef _ALPINE #ifndef _ALPINE
error(10, 0, "ABORTED"); error(10, 0, "ABORTED");
#else #else
@ -469,81 +523,82 @@ int main(int argc, char *argv[]) {
printf("====== arguments config ======\n"); printf("====== arguments config ======\n");
{ {
printf("host: %s\n", tsArguments.host); printf("host: %s\n", g_args.host);
printf("user: %s\n", tsArguments.user); printf("user: %s\n", g_args.user);
printf("password: %s\n", tsArguments.password); printf("password: %s\n", g_args.password);
printf("port: %u\n", tsArguments.port); printf("port: %u\n", g_args.port);
printf("cversion: %s\n", tsArguments.cversion); printf("cversion: %s\n", g_args.cversion);
printf("mysqlFlag: %d\n", tsArguments.mysqlFlag); printf("mysqlFlag: %d\n", g_args.mysqlFlag);
printf("outpath: %s\n", tsArguments.outpath); printf("outpath: %s\n", g_args.outpath);
printf("inpath: %s\n", tsArguments.inpath); printf("inpath: %s\n", g_args.inpath);
printf("resultFile: %s\n", tsArguments.resultFile); printf("resultFile: %s\n", g_args.resultFile);
printf("encode: %s\n", tsArguments.encode); printf("encode: %s\n", g_args.encode);
printf("all_databases: %d\n", tsArguments.all_databases); printf("all_databases: %d\n", g_args.all_databases);
printf("databases: %d\n", tsArguments.databases); printf("databases: %d\n", g_args.databases);
printf("schemaonly: %d\n", tsArguments.schemaonly); printf("schemaonly: %d\n", g_args.schemaonly);
printf("with_property: %d\n", tsArguments.with_property); printf("with_property: %d\n", g_args.with_property);
printf("start_time: %" PRId64 "\n", tsArguments.start_time); printf("start_time: %" PRId64 "\n", g_args.start_time);
printf("end_time: %" PRId64 "\n", tsArguments.end_time); printf("end_time: %" PRId64 "\n", g_args.end_time);
printf("data_batch: %d\n", tsArguments.data_batch); printf("data_batch: %d\n", g_args.data_batch);
printf("max_sql_len: %d\n", tsArguments.max_sql_len); printf("max_sql_len: %d\n", g_args.max_sql_len);
printf("table_batch: %d\n", tsArguments.table_batch); printf("table_batch: %d\n", g_args.table_batch);
printf("thread_num: %d\n", tsArguments.thread_num); printf("thread_num: %d\n", g_args.thread_num);
printf("allow_sys: %d\n", tsArguments.allow_sys); printf("allow_sys: %d\n", g_args.allow_sys);
printf("abort: %d\n", tsArguments.abort); printf("abort: %d\n", g_args.abort);
printf("isDumpIn: %d\n", tsArguments.isDumpIn); printf("isDumpIn: %d\n", g_args.isDumpIn);
printf("arg_list_len: %d\n", tsArguments.arg_list_len); printf("arg_list_len: %d\n", g_args.arg_list_len);
printf("debug_print: %d\n", g_args.debug_print);
for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { for (int32_t i = 0; i < g_args.arg_list_len; i++) {
printf("arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]);
} }
} }
printf("==============================\n"); printf("==============================\n");
if (tsArguments.cversion[0] != 0){ if (g_args.cversion[0] != 0){
tstrncpy(version, tsArguments.cversion, 11); tstrncpy(version, g_args.cversion, 11);
} }
if (taosCheckParam(&tsArguments) < 0) { if (taosCheckParam(&g_args) < 0) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
g_fpOfResult = fopen(tsArguments.resultFile, "a"); g_fpOfResult = fopen(g_args.resultFile, "a");
if (NULL == g_fpOfResult) { if (NULL == g_fpOfResult) {
fprintf(stderr, "Failed to open %s for save result\n", tsArguments.resultFile); fprintf(stderr, "Failed to open %s for save result\n", g_args.resultFile);
return 1; return 1;
}; };
fprintf(g_fpOfResult, "#############################################################################\n"); fprintf(g_fpOfResult, "#############################################################################\n");
fprintf(g_fpOfResult, "============================== arguments config =============================\n"); fprintf(g_fpOfResult, "============================== arguments config =============================\n");
{ {
fprintf(g_fpOfResult, "host: %s\n", tsArguments.host); fprintf(g_fpOfResult, "host: %s\n", g_args.host);
fprintf(g_fpOfResult, "user: %s\n", tsArguments.user); fprintf(g_fpOfResult, "user: %s\n", g_args.user);
fprintf(g_fpOfResult, "password: %s\n", tsArguments.password); fprintf(g_fpOfResult, "password: %s\n", g_args.password);
fprintf(g_fpOfResult, "port: %u\n", tsArguments.port); fprintf(g_fpOfResult, "port: %u\n", g_args.port);
fprintf(g_fpOfResult, "cversion: %s\n", tsArguments.cversion); fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion);
fprintf(g_fpOfResult, "mysqlFlag: %d\n", tsArguments.mysqlFlag); fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag);
fprintf(g_fpOfResult, "outpath: %s\n", tsArguments.outpath); fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath);
fprintf(g_fpOfResult, "inpath: %s\n", tsArguments.inpath); fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath);
fprintf(g_fpOfResult, "resultFile: %s\n", tsArguments.resultFile); fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile);
fprintf(g_fpOfResult, "encode: %s\n", tsArguments.encode); fprintf(g_fpOfResult, "encode: %s\n", g_args.encode);
fprintf(g_fpOfResult, "all_databases: %d\n", tsArguments.all_databases); fprintf(g_fpOfResult, "all_databases: %d\n", g_args.all_databases);
fprintf(g_fpOfResult, "databases: %d\n", tsArguments.databases); fprintf(g_fpOfResult, "databases: %d\n", g_args.databases);
fprintf(g_fpOfResult, "schemaonly: %d\n", tsArguments.schemaonly); fprintf(g_fpOfResult, "schemaonly: %d\n", g_args.schemaonly);
fprintf(g_fpOfResult, "with_property: %d\n", tsArguments.with_property); fprintf(g_fpOfResult, "with_property: %d\n", g_args.with_property);
fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", tsArguments.start_time); fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time);
fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", tsArguments.end_time); fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time);
fprintf(g_fpOfResult, "data_batch: %d\n", tsArguments.data_batch); fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch);
fprintf(g_fpOfResult, "max_sql_len: %d\n", tsArguments.max_sql_len); fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len);
fprintf(g_fpOfResult, "table_batch: %d\n", tsArguments.table_batch); fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch);
fprintf(g_fpOfResult, "thread_num: %d\n", tsArguments.thread_num); fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num);
fprintf(g_fpOfResult, "allow_sys: %d\n", tsArguments.allow_sys); fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys);
fprintf(g_fpOfResult, "abort: %d\n", tsArguments.abort); fprintf(g_fpOfResult, "abort: %d\n", g_args.abort);
fprintf(g_fpOfResult, "isDumpIn: %d\n", tsArguments.isDumpIn); fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn);
fprintf(g_fpOfResult, "arg_list_len: %d\n", tsArguments.arg_list_len); fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len);
for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { for (int32_t i = 0; i < g_args.arg_list_len; i++) {
fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]);
} }
} }
@ -552,11 +607,11 @@ int main(int argc, char *argv[]) {
time_t tTime = time(NULL); time_t tTime = time(NULL);
struct tm tm = *localtime(&tTime); struct tm tm = *localtime(&tTime);
if (tsArguments.isDumpIn) { if (g_args.isDumpIn) {
fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n");
fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
if (taosDumpIn(&tsArguments) < 0) { if (taosDumpIn(&g_args) < 0) {
fprintf(g_fpOfResult, "\n"); fprintf(g_fpOfResult, "\n");
fclose(g_fpOfResult); fclose(g_fpOfResult);
return -1; return -1;
@ -565,7 +620,7 @@ int main(int argc, char *argv[]) {
fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n");
fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
if (taosDumpOut(&tsArguments) < 0) { if (taosDumpOut(&g_args) < 0) {
fprintf(g_fpOfResult, "\n"); fprintf(g_fpOfResult, "\n");
fclose(g_fpOfResult); fclose(g_fpOfResult);
return -1; return -1;
@ -573,9 +628,9 @@ int main(int argc, char *argv[]) {
fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n");
fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut);
fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut);
fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut);
fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut);
} }
fprintf(g_fpOfResult, "\n"); fprintf(g_fpOfResult, "\n");
@ -1236,8 +1291,8 @@ void* taosDumpOutWorkThreadFp(void *arg)
FILE *fp = NULL; FILE *fp = NULL;
memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128);
if (tsArguments.outpath[0] != 0) { if (g_args.outpath[0] != 0) {
sprintf(tmpBuf, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); sprintf(tmpBuf, "%s/%s.tables.%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex);
} else { } else {
sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex);
} }
@ -1270,7 +1325,7 @@ void* taosDumpOutWorkThreadFp(void *arg)
ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
if (readLen <= 0) break; if (readLen <= 0) break;
int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon, pThread->dbName); int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &g_args, fp, pThread->taosCon, pThread->dbName);
if (ret >= 0) { if (ret >= 0) {
// TODO: sum table count and table rows by self // TODO: sum table count and table rows by self
pThread->tablesOfDumpOut++; pThread->tablesOfDumpOut++;
@ -1282,13 +1337,13 @@ void* taosDumpOutWorkThreadFp(void *arg)
} }
tablesInOneFile++; tablesInOneFile++;
if (tablesInOneFile >= tsArguments.table_batch) { if (tablesInOneFile >= g_args.table_batch) {
fclose(fp); fclose(fp);
tablesInOneFile = 0; tablesInOneFile = 0;
memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128);
if (tsArguments.outpath[0] != 0) { if (g_args.outpath[0] != 0) {
sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex);
} else { } else {
sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex);
} }
@ -1491,14 +1546,14 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
taos_free_result(res); taos_free_result(res);
lseek(fd, 0, SEEK_SET); lseek(fd, 0, SEEK_SET);
int maxThreads = tsArguments.thread_num; int maxThreads = g_args.thread_num;
int tableOfPerFile ; int tableOfPerFile ;
if (numOfTable <= tsArguments.thread_num) { if (numOfTable <= g_args.thread_num) {
tableOfPerFile = 1; tableOfPerFile = 1;
maxThreads = numOfTable; maxThreads = numOfTable;
} else { } else {
tableOfPerFile = numOfTable / tsArguments.thread_num; tableOfPerFile = numOfTable / g_args.thread_num;
if (0 != numOfTable % tsArguments.thread_num) { if (0 != numOfTable % g_args.thread_num) {
tableOfPerFile += 1; tableOfPerFile += 1;
} }
} }
@ -1806,9 +1861,9 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
//} //}
} }
fprintf(fp, "\n"); fprintf(fp, "\n");
atomic_add_fetch_64(&totalDumpOutRows, totalRows); atomic_add_fetch_64(&totalDumpOutRows, totalRows);
taos_free_result(tmpResult); taos_free_result(tmpResult);
free(tmpBuffer); free(tmpBuffer);
return totalRows; return totalRows;
@ -1824,7 +1879,7 @@ int taosCheckParam(struct arguments *arguments) {
fprintf(stderr, "start time is larger than end time\n"); fprintf(stderr, "start time is larger than end time\n");
return -1; return -1;
} }
if (arguments->arg_list_len == 0) { if (arguments->arg_list_len == 0) {
if ((!arguments->all_databases) && (!arguments->isDumpIn)) { if ((!arguments->all_databases) && (!arguments->isDumpIn)) {
fprintf(stderr, "taosdump requires parameters\n"); fprintf(stderr, "taosdump requires parameters\n");
@ -2214,7 +2269,7 @@ void* taosDumpInWorkThreadFp(void *arg)
continue; continue;
} }
fprintf(stderr, "Success Open input file: %s\n", SQLFileName); fprintf(stderr, "Success Open input file: %s\n", SQLFileName);
taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, tsArguments.encode, SQLFileName); taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, g_args.encode, SQLFileName);
} }
} }

3
src/query/inc/qExecutor.h
View File

@ -86,7 +86,8 @@ typedef struct SResultRow {
bool closed; // this result status: closed or opened bool closed; // this result status: closed or opened
uint32_t numOfRows; // number of rows of current time window uint32_t numOfRows; // number of rows of current time window
SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo
union {STimeWindow win; char* key;}; // start key of current result row STimeWindow win;
char* key; // start key of current result row
} SResultRow; } SResultRow;
typedef struct SGroupResInfo { typedef struct SGroupResInfo {

13
src/query/src/qExecutor.c
View File

@ -1876,14 +1876,15 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
taosHashCleanup(pRuntimeEnv->pResultRowHashTable); taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
pRuntimeEnv->pResultRowHashTable = NULL; pRuntimeEnv->pResultRowHashTable = NULL;
pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool);
taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult);
pRuntimeEnv->prevResult = NULL;
taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap);
pRuntimeEnv->pTableRetrieveTsMap = NULL; pRuntimeEnv->pTableRetrieveTsMap = NULL;
destroyOperatorInfo(pRuntimeEnv->proot); destroyOperatorInfo(pRuntimeEnv->proot);
pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool);
taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult);
pRuntimeEnv->prevResult = NULL;
} }
static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) {
@ -6463,6 +6464,9 @@ void freeQInfo(SQInfo *pQInfo) {
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables);
doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo);
teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
SQuery *pQuery = pQInfo->runtimeEnv.pQuery; SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
@ -6498,7 +6502,6 @@ void freeQInfo(SQInfo *pQInfo) {
} }
} }
doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo);
tfree(pQInfo->pBuf); tfree(pQInfo->pBuf);
tfree(pQInfo->sql); tfree(pQInfo->sql);

11
src/query/src/qUtil.c
View File

@ -66,8 +66,8 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
return; return;
} }
if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) { for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
for(int32_t i = 0; i < pResultRowInfo->size; ++i) { if (pResultRowInfo->pResult[i]) {
tfree(pResultRowInfo->pResult[i]->key); tfree(pResultRowInfo->pResult[i]->key);
} }
} }
@ -153,11 +153,8 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16
pResultRow->offset = -1; pResultRow->offset = -1;
pResultRow->closed = false; pResultRow->closed = false;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tfree(pResultRow->key);
tfree(pResultRow->key); pResultRow->win = TSWINDOW_INITIALIZER;
} else {
pResultRow->win = TSWINDOW_INITIALIZER;
}
} }
// TODO refactor: use macro // TODO refactor: use macro

2
src/sync/inc/syncInt.h
View File

@ -35,7 +35,7 @@ extern "C" {
#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
#define SYNC_RECV_BUFFER_SIZE (5*1024*1024) #define SYNC_RECV_BUFFER_SIZE (5*1024*1024)
#define SYNC_MAX_FWDS 512 #define SYNC_MAX_FWDS 1024
#define SYNC_FWD_TIMER 300 #define SYNC_FWD_TIMER 300
#define SYNC_ROLE_TIMER 15000 // ms #define SYNC_ROLE_TIMER 15000 // ms
#define SYNC_CHECK_INTERVAL 1000 // ms #define SYNC_CHECK_INTERVAL 1000 // ms

7
src/sync/src/syncMain.c
View File

@ -1459,7 +1459,12 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
if ((pNode->quorum > 1 || force) && code == 0) { if ((pNode->quorum > 1 || force) && code == 0) {
code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
if (code >= 0) code = 1; if (code >= 0) {
code = 1;
} else {
pthread_mutex_unlock(&pNode->mutex);
return code;
}
} }
int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen);

6
src/vnode/src/vnodeWrite.c
View File

@ -91,13 +91,17 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
int32_t syncCode = 0; int32_t syncCode = 0;
bool force = (pWrite == NULL ? false : pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); bool force = (pWrite == NULL ? false : pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT);
syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force); syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force);
if (syncCode < 0) return syncCode; if (syncCode < 0) {
pHead->version = 0;
return syncCode;
}
// write into WAL // write into WAL
code = walWrite(pVnode->wal, pHead); code = walWrite(pVnode->wal, pHead);
if (code < 0) { if (code < 0) {
if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code);
pHead->version = 0;
return code; return code;
} }

57
tests/pytest/tools/insert-tblimit-tboffset-createdb.json Normal file
View File

@ -0,0 +1,57 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"num_of_records_per_req": 100,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db",
"drop": "yes",
"replica": 1,
"days": 10,
"cache": 16,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb",
"child_table_exists":"no",
"childtable_count": 100,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 0,
"multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
}]
}]
}

59
tests/pytest/tools/insert-tblimit-tboffset-insertrec.json Normal file
View File

@ -0,0 +1,59 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"num_of_records_per_req": 100,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db",
"drop": "no",
"replica": 1,
"days": 10,
"cache": 16,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb",
"child_table_exists":"yes",
"childtable_count": 100,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 1000,
"childtable_limit": 33,
"childtable_offset": 33,
"multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
}]
}]
}

4
tests/pytest/tools/insert-tblimit-tboffset0.json
View File

@ -15,7 +15,7 @@
"databases": [{ "databases": [{
"dbinfo": { "dbinfo": {
"name": "db", "name": "db",
"drop": "yes", "drop": "no",
"replica": 1, "replica": 1,
"days": 10, "days": 10,
"cache": 16, "cache": 16,
@ -33,7 +33,7 @@
}, },
"super_tables": [{ "super_tables": [{
"name": "stb", "name": "stb",
"child_table_exists":"no", "child_table_exists":"yes",
"childtable_count": 100, "childtable_count": 100,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"auto_create_table": "no", "auto_create_table": "no",

4
tests/pytest/tools/insert-tblimit1-tboffset.json
View File

@ -15,7 +15,7 @@
"databases": [{ "databases": [{
"dbinfo": { "dbinfo": {
"name": "db", "name": "db",
"drop": "yes", "drop": "no",
"replica": 1, "replica": 1,
"days": 10, "days": 10,
"cache": 16, "cache": 16,
@ -33,7 +33,7 @@
}, },
"super_tables": [{ "super_tables": [{
"name": "stb", "name": "stb",
"child_table_exists":"no", "child_table_exists":"yes",
"childtable_count": 100, "childtable_count": 100,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"auto_create_table": "no", "auto_create_table": "no",

2
tests/pytest/tools/taosdemo-sampledata.json
View File

@ -16,8 +16,6 @@
"name": "stb", "name": "stb",
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 20, "childtable_count": 20,
"childtable_limit": 10,
"childtable_offset": 0,
"childtable_prefix": "t_", "childtable_prefix": "t_",
"auto_create_table": "no", "auto_create_table": "no",
"data_source": "sample", "data_source": "sample",

5
tests/pytest/tools/taosdemoTestLimitOffset.py
View File

@ -51,7 +51,8 @@ class TDTestCase:
else: else:
tdLog.info("taosd found in %s" % buildPath) tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath+ "/build/bin/" binPath = buildPath+ "/build/bin/"
os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
os.system("%staosdemo -f tools/insert-tblimit-tboffset-insertrec.json" % binPath)
tdSql.execute("use db") tdSql.execute("use db")
tdSql.query("select count(tbname) from db.stb") tdSql.query("select count(tbname) from db.stb")
@ -59,6 +60,7 @@ class TDTestCase:
tdSql.query("select count(*) from db.stb") tdSql.query("select count(*) from db.stb")
tdSql.checkData(0, 0, 33000) tdSql.checkData(0, 0, 33000)
os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath)
tdSql.execute("reset query cache") tdSql.execute("reset query cache")
@ -68,6 +70,7 @@ class TDTestCase:
tdSql.query("select count(*) from db.stb") tdSql.query("select count(*) from db.stb")
tdSql.checkData(0, 0, 20000) tdSql.checkData(0, 0, 20000)
os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath)
tdSql.execute("reset query cache") tdSql.execute("reset query cache")

2
tests/pytest/tools/taosdemoTestSampleData.py
View File

@ -57,7 +57,7 @@ class TDTestCase:
tdSql.query("select count(tbname) from db.stb") tdSql.query("select count(tbname) from db.stb")
tdSql.checkData(0, 0, 20) tdSql.checkData(0, 0, 20)
tdSql.query("select count(*) from db.stb") tdSql.query("select count(*) from db.stb")
tdSql.checkData(0, 0, 200) tdSql.checkData(0, 0, 400)
def stop(self): def stop(self):
tdSql.close() tdSql.close()