Merge pull request #18489 from taosdata/fix/TD-20318
test: add asan case and fix tmqSim error
This commit is contained in:
Shengliang Guan
GitHub
commit
791f0bd276
|
|
@ -687,9 +687,9 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData1.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqShow.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqAlterSchema.py
|
||||
,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py
|
||||
|
|
|
|||
|
|
@ -128,12 +128,12 @@ class TMQCom:
|
|||
os.system(shellCmd)
|
||||
|
||||
def stopTmqSimProcess(self, processorName):
|
||||
psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName)
|
||||
psCmd = "unset LD_PRELOAD; ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName)
|
||||
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
|
||||
onlyKillOnceWindows = 0
|
||||
while(processID):
|
||||
if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'):
|
||||
killCmd = "kill -INT %s > /dev/null 2>&1" % processID
|
||||
killCmd = "unset LD_PRELOAD; kill -INT %s > /dev/null 2>&1" % processID
|
||||
os.system(killCmd)
|
||||
onlyKillOnceWindows = 1
|
||||
time.sleep(0.2)
|
||||
|
|
|
|||
|
|
@ -14,13 +14,13 @@
|
|||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <time.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "taos.h"
|
||||
#include "taosdef.h"
|
||||
|
|
@ -45,11 +45,7 @@ typedef enum {
|
|||
NOTIFY_CMD_ID_BUTT,
|
||||
} NOTIFY_CMD_ID;
|
||||
|
||||
typedef enum enumQUERY_TYPE {
|
||||
NO_INSERT_TYPE,
|
||||
INSERT_TYPE,
|
||||
QUERY_TYPE_BUT
|
||||
} QUERY_TYPE;
|
||||
typedef enum enumQUERY_TYPE { NO_INSERT_TYPE, INSERT_TYPE, QUERY_TYPE_BUT } QUERY_TYPE;
|
||||
|
||||
typedef struct {
|
||||
TdThread thread;
|
||||
|
|
@ -153,7 +149,6 @@ static void printHelp() {
|
|||
printf("%s%s\n", indent, "-x");
|
||||
printf("%s%s%s\n", indent, indent, "consume thread number, default is 1");
|
||||
|
||||
|
||||
printf("%s%s\n", indent, "-l");
|
||||
printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes);
|
||||
printf("%s%s\n", indent, "-p");
|
||||
|
|
@ -165,7 +160,6 @@ static void printHelp() {
|
|||
printf("%s%s\n", indent, "-n");
|
||||
printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000");
|
||||
|
||||
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
|
|
@ -379,7 +373,6 @@ int queryDB(TAOS* taos, char* command) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
|
||||
void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) {
|
||||
int32_t i;
|
||||
for (i = 0; i < pInfo->numOfVgroups; i++) {
|
||||
|
|
@ -418,7 +411,6 @@ TAOS* createNewTaosConnect() {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) {
|
||||
char sqlStr[1100] = {0};
|
||||
|
||||
|
|
@ -548,8 +540,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f
|
|||
break;
|
||||
case TSDB_DATA_TYPE_BINARY:
|
||||
case TSDB_DATA_TYPE_NCHAR:
|
||||
case TSDB_DATA_TYPE_JSON:
|
||||
{
|
||||
case TSDB_DATA_TYPE_JSON: {
|
||||
int32_t bufIndex = 0;
|
||||
for (int32_t i = 0; i < length; i++) {
|
||||
buf[bufIndex] = val[i];
|
||||
|
|
@ -562,8 +553,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f
|
|||
buf[bufIndex] = 0;
|
||||
|
||||
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
|
||||
}
|
||||
break;
|
||||
} break;
|
||||
case TSDB_DATA_TYPE_TIMESTAMP:
|
||||
shellFormatTimestamp(buf, *(int64_t*)val, precision);
|
||||
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
|
||||
|
|
@ -635,7 +625,6 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
|
|||
return totalRows;
|
||||
}
|
||||
|
||||
|
||||
static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) {
|
||||
char buf[1024];
|
||||
int32_t totalRows = 0;
|
||||
|
|
@ -683,8 +672,8 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
|
|||
char sqlStr[1024] = {0};
|
||||
|
||||
// schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
|
||||
sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId,
|
||||
pInfo->consumerId);
|
||||
sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName,
|
||||
atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId, pInfo->consumerId);
|
||||
|
||||
taos_query_a(pInfo->taos, sqlStr, appNothing, NULL);
|
||||
|
||||
|
|
@ -865,6 +854,9 @@ void loop_consume(SThreadInfo* pInfo) {
|
|||
|
||||
taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n",
|
||||
pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt);
|
||||
|
||||
taosFsyncFile(pInfo->pConsumeRowsFile);
|
||||
taosCloseFile(&pInfo->pConsumeRowsFile);
|
||||
}
|
||||
|
||||
void* consumeThreadFunc(void* param) {
|
||||
|
|
@ -944,7 +936,8 @@ void parseConsumeInfo() {
|
|||
token = strtok(g_stConfInfo.stThreads[i].topicString, delim);
|
||||
while (token != NULL) {
|
||||
// printf("%s\n", token );
|
||||
tstrncpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token, sizeof(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]));
|
||||
tstrncpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token,
|
||||
sizeof(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]));
|
||||
ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]);
|
||||
// printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
|
||||
g_stConfInfo.stThreads[i].numOfTopic++;
|
||||
|
|
@ -960,7 +953,8 @@ void parseConsumeInfo() {
|
|||
ltrim(pstr);
|
||||
char* ret = strchr(pstr, ch);
|
||||
memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr);
|
||||
tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey]));
|
||||
tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1,
|
||||
sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey]));
|
||||
// printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
|
||||
// g_stConfInfo.value[g_stConfInfo.numOfKey]);
|
||||
g_stConfInfo.stThreads[i].numOfKey++;
|
||||
|
|
@ -1037,7 +1031,6 @@ int32_t getConsumeInfo() {
|
|||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) {
|
||||
char buf[16 * 1024];
|
||||
int32_t totalRows = 0;
|
||||
|
|
@ -1082,8 +1075,7 @@ void omb_loop_consume(SThreadInfo* pInfo) {
|
|||
char tmpString[128];
|
||||
taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString),
|
||||
pInfo->consumerId);
|
||||
printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString),
|
||||
pInfo->consumerId);
|
||||
printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), pInfo->consumerId);
|
||||
|
||||
pInfo->ts = taosGetTimestampMs();
|
||||
|
||||
|
|
@ -1106,14 +1098,16 @@ void omb_loop_consume(SThreadInfo* pInfo) {
|
|||
if (currentPrintTime - lastPrintTime > 10 * 1000) {
|
||||
int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg;
|
||||
int64_t deltaTime = currentPrintTime - lastPrintTime;
|
||||
printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalRows, totalMsgs,
|
||||
(totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime,
|
||||
printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64
|
||||
", rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalRows, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime,
|
||||
currentLenOfMsg * 1000.0 / (1024 * 1024) / deltaTime);
|
||||
|
||||
taosFprintfFile(
|
||||
g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime);
|
||||
taosFprintfFile(g_fp,
|
||||
"consumer id %d has currently poll total msgs: %" PRId64
|
||||
", period cons rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime,
|
||||
currentLenOfMsg * 1000.0 / deltaTime);
|
||||
lastPrintTime = currentPrintTime;
|
||||
lastTotalMsgs = totalMsgs;
|
||||
lastTotalLenOfMsg = totalLenOfMsg;
|
||||
|
|
@ -1125,9 +1119,9 @@ void omb_loop_consume(SThreadInfo* pInfo) {
|
|||
int64_t currentPrintTime = taosGetTimestampMs();
|
||||
int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg;
|
||||
int64_t deltaTime = currentPrintTime - lastPrintTime;
|
||||
printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalRows, totalMsgs,
|
||||
(totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime,
|
||||
printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64
|
||||
", rate: %.3f msgs/s, %.1f MB/s\n",
|
||||
pInfo->consumerId, totalRows, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime,
|
||||
currentLenOfMsg * 1000.0 / (1024 * 1024) / deltaTime);
|
||||
break;
|
||||
}
|
||||
|
|
@ -1136,10 +1130,8 @@ void omb_loop_consume(SThreadInfo* pInfo) {
|
|||
pInfo->consumeMsgCnt = totalMsgs;
|
||||
pInfo->consumeRowCnt = totalRows;
|
||||
pInfo->consumeLen = totalLenOfMsg;
|
||||
|
||||
}
|
||||
|
||||
|
||||
void* ombConsumeThreadFunc(void* param) {
|
||||
SThreadInfo* pInfo = (SThreadInfo*)param;
|
||||
|
||||
|
|
@ -1206,8 +1198,6 @@ void* ombConsumeThreadFunc(void* param) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static int queryDbExec(TAOS* taos, char* command, QUERY_TYPE type) {
|
||||
TAOS_RES* res = taos_query(taos, command);
|
||||
int32_t code = taos_errno(res);
|
||||
|
|
@ -1240,7 +1230,8 @@ void* ombProduceThreadFunc(void* param) {
|
|||
int64_t affectedRowsTotal = 0;
|
||||
int64_t sendMsgs = 0;
|
||||
|
||||
uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms
|
||||
uint32_t totalSendLoopTimes =
|
||||
g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms
|
||||
uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize;
|
||||
uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize;
|
||||
if (remainder) {
|
||||
|
|
@ -1255,7 +1246,8 @@ void* ombProduceThreadFunc(void* param) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate);
|
||||
printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes,
|
||||
batchPerTblTimes, pInfo->producerRate);
|
||||
|
||||
char ctbName[128] = {0};
|
||||
sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId);
|
||||
|
|
@ -1310,9 +1302,7 @@ void* ombProduceThreadFunc(void* param) {
|
|||
if (delta > 10 * 1000 * 1000) {
|
||||
printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n",
|
||||
pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes);
|
||||
printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n",
|
||||
pInfo->consumerId,
|
||||
sendMsgs * 1000.0 * 1000 / delta,
|
||||
printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", pInfo->consumerId, sendMsgs * 1000.0 * 1000 / delta,
|
||||
(totalMsgLen / 1024.0) / (delta / (1000 * 1000)));
|
||||
lastPrintTime = currentTs;
|
||||
sendMsgs = 0;
|
||||
|
|
@ -1327,7 +1317,6 @@ void* ombProduceThreadFunc(void* param) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
void printProduceInfo(int64_t start) {
|
||||
int64_t totalMsgs = 0;
|
||||
int64_t totalLenOfMsgs = 0;
|
||||
|
|
@ -1344,15 +1333,12 @@ void printProduceInfo(int64_t start) {
|
|||
double tInMs = (double)t / 1000000.0;
|
||||
printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs);
|
||||
|
||||
|
||||
printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n",
|
||||
tInMs, totalMsgs, g_stConfInfo.producers,
|
||||
(double)totalMsgs / tInMs,
|
||||
tInMs, totalMsgs, g_stConfInfo.producers, (double)totalMsgs / tInMs,
|
||||
(double)totalLenOfMsgs / (1024.0 * 1024) / tInMs);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
void startOmbConsume() {
|
||||
TdThreadAttr thattr;
|
||||
taosThreadAttrInit(&thattr);
|
||||
|
|
@ -1373,16 +1359,19 @@ void startOmbConsume() {
|
|||
printf("SQL: %s\n", sql);
|
||||
queryDbExec(taos, sql, NO_INSERT_TYPE);
|
||||
|
||||
sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers);
|
||||
sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName,
|
||||
g_stConfInfo.producers);
|
||||
printf("SQL: %s\n", sql);
|
||||
queryDbExec(taos, sql, NO_INSERT_TYPE);
|
||||
|
||||
sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen);
|
||||
sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName,
|
||||
stbName, g_stConfInfo.payloadLen);
|
||||
printf("SQL: %s\n", sql);
|
||||
queryDbExec(taos, sql, NO_INSERT_TYPE);
|
||||
|
||||
for (int i = 0; i < g_stConfInfo.producers; i++) {
|
||||
sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i);
|
||||
sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i,
|
||||
g_stConfInfo.dbName, i);
|
||||
printf("SQL: %s\n", sql);
|
||||
queryDbExec(taos, sql, NO_INSERT_TYPE);
|
||||
}
|
||||
|
|
@ -1392,7 +1381,6 @@ void startOmbConsume() {
|
|||
printf("SQL: %s\n", sql);
|
||||
queryDbExec(taos, sql, NO_INSERT_TYPE);
|
||||
|
||||
|
||||
int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers);
|
||||
|
||||
printf("==== create %d produce thread ====\n", g_stConfInfo.producers);
|
||||
|
|
@ -1451,15 +1439,14 @@ void startOmbConsume() {
|
|||
if (0 == t) t = 1;
|
||||
|
||||
double tInMs = (double)t / 1000000.0;
|
||||
taosFprintfFile(g_fp,
|
||||
"Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n",
|
||||
tInMs, totalMsgs, g_stConfInfo.numOfThread,
|
||||
(double)(totalMsgs / tInMs),
|
||||
taosFprintfFile(
|
||||
g_fp, "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n",
|
||||
tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs),
|
||||
(double)totalLenOfMsgs / (1024 * 1024) / tInMs);
|
||||
|
||||
printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n",
|
||||
tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread,
|
||||
(double)(totalMsgs / tInMs),
|
||||
printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64
|
||||
" with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n",
|
||||
tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs),
|
||||
(double)totalLenOfMsgs / (1024 * 1024) / tInMs);
|
||||
|
||||
taosFprintfFile(g_fp, "==== close tmqlog ====\n");
|
||||
|
|
@ -1468,7 +1455,6 @@ void startOmbConsume() {
|
|||
return;
|
||||
}
|
||||
|
||||
|
||||
int main(int32_t argc, char* argv[]) {
|
||||
parseArgument(argc, argv);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue