From b4eee354813f7626fb3ce8ec1fb474d03868eced Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 Aug 2021 17:31:20 +0800 Subject: [PATCH 01/49] [TD-6011]: where clause including 'bool' Keyword causes core dump --- src/common/src/tvariant.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 9e0f7ffc74..42ece19588 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -31,12 +31,12 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { switch (token->type) { case TSDB_DATA_TYPE_BOOL: { - int32_t k = strncasecmp(token->z, "true", 4); - if (k == 0) { + if (strncasecmp(token->z, "true", 4) == 0) { pVar->i64 = TSDB_TRUE; - } else { - assert(strncasecmp(token->z, "false", 5) == 0); + } else if (strncasecmp(token->z, "false", 5) == 0) { pVar->i64 = TSDB_FALSE; + } else { + return; } break; From ca68c37dffb243c35a58b7bca670497654dde9fe Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 Aug 2021 17:31:20 +0800 Subject: [PATCH 02/49] [TD-6011]: where clause including 'bool' Keyword causes core dump --- src/common/src/tvariant.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 33dab51633..3f9e219983 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -31,12 +31,12 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { switch (token->type) { case TSDB_DATA_TYPE_BOOL: { - int32_t k = strncasecmp(token->z, "true", 4); - if (k == 0) { + if (strncasecmp(token->z, "true", 4) == 0) { pVar->i64 = TSDB_TRUE; - } else { - assert(strncasecmp(token->z, "false", 5) == 0); + } else if (strncasecmp(token->z, "false", 5) == 0) { pVar->i64 = TSDB_FALSE; + } else { + return; } break; From 2d0092876e899c2120dc753c179258bdba6aba99 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Aug 2021 15:30:52 +0000 Subject: [PATCH 03/49] [TD-6214] improve insert performance --- src/client/src/tscServer.c | 212 +++++++++++++++++++++++++++++++------ 1 file changed, 178 insertions(+), 34 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c0723e210a..045fd5ca5c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -337,16 +337,189 @@ int tscSendMsgToServer(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } -static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { - SRpcMsg* rpcMsg = pSchedMsg->ahandle; - SRpcEpSet* pEpSet = pSchedMsg->thandle; +//static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { +// SRpcMsg* rpcMsg = pSchedMsg->ahandle; +// SRpcEpSet* pEpSet = pSchedMsg->thandle; +// +// TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; +// SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); +// if (pSql == NULL) { +// rpcFreeCont(rpcMsg->pCont); +// free(rpcMsg); +// free(pEpSet); +// return; +// } +// +// assert(pSql->self == handle); +// +// STscObj *pObj = pSql->pTscObj; +// SSqlRes *pRes = &pSql->res; +// SSqlCmd *pCmd = &pSql->cmd; +// +// pSql->rpcRid = -1; +// +// if (pObj->signature != pObj) { +// tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); +// +// taosRemoveRef(tscObjRef, handle); +// taosReleaseRef(tscObjRef, handle); +// rpcFreeCont(rpcMsg->pCont); +// free(rpcMsg); +// free(pEpSet); +// return; +// } +// +// SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); +// if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { +// tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", +// pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); +// +// taosRemoveRef(tscObjRef, handle); +// taosReleaseRef(tscObjRef, handle); +// rpcFreeCont(rpcMsg->pCont); +// free(rpcMsg); +// free(pEpSet); +// return; +// } +// +// if (pEpSet) { +// if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { +// if (pCmd->command < TSDB_SQL_MGMT) { +// tscUpdateVgroupInfo(pSql, pEpSet); +// } else { +// tscUpdateMgmtEpSet(pSql, pEpSet); +// } +// } +// } +// +// int32_t cmd = pCmd->command; +// +// // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema +// if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { +// pSql->cmd.insertParam.schemaAttached = 1; +// } +// +// // single table query error need to be handled here. +// if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && +// (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || +// rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { +// +// // 1. super table subquery +// // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer +// if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | +// TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && +// !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || +// (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { +// // do nothing in case of super table subquery +// } else { +// pSql->retry += 1; +// tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); +// +// pSql->res.code = rpcMsg->code; // keep the previous error code +// if (pSql->retry > pSql->maxRetry) { +// tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); +// } else { +// // wait for a little bit moment and then retry +// // todo do not sleep in rpc callback thread, add this process into queue to process +// if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { +// int32_t duration = getWaitingTimeInterval(pSql->retry); +// taosMsleep(duration); +// } +// +// pSql->retryReason = rpcMsg->code; +// rpcMsg->code = tscRenewTableMeta(pSql, 0); +// // if there is an error occurring, proceed to the following error handling procedure. +// if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { +// taosReleaseRef(tscObjRef, handle); +// rpcFreeCont(rpcMsg->pCont); +// free(rpcMsg); +// free(pEpSet); +// return; +// } +// } +// } +// } +// +// pRes->rspLen = 0; +// +// if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { +// tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); +// } else { +// pRes->code = rpcMsg->code; +// } +// +// if (pRes->code == TSDB_CODE_SUCCESS) { +// tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); +// pSql->retry = 0; +// } +// +// if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { +// assert(rpcMsg->msgType == pCmd->msgType + 1); +// pRes->code = rpcMsg->code; +// pRes->rspType = rpcMsg->msgType; +// pRes->rspLen = rpcMsg->contLen; +// +// if (pRes->rspLen > 0 && rpcMsg->pCont) { +// char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); +// if (tmp == NULL) { +// pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; +// } else { +// pRes->pRsp = tmp; +// memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); +// } +// } else { +// tfree(pRes->pRsp); +// } +// +// /* +// * There is not response callback function for submit response. +// * The actual inserted number of points is the first number. +// */ +// if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { +// SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; +// pMsg->code = htonl(pMsg->code); +// pMsg->numOfRows = htonl(pMsg->numOfRows); +// pMsg->affectedRows = htonl(pMsg->affectedRows); +// pMsg->failedRows = htonl(pMsg->failedRows); +// pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); +// +// pRes->numOfRows += pMsg->affectedRows; +// tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], +// tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); +// } else { +// tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); +// } +// } +// +// if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { +// rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); +// } +// +// bool shouldFree = tscShouldBeFreed(pSql); +// if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { +// if (rpcMsg->code != TSDB_CODE_SUCCESS) { +// pRes->code = rpcMsg->code; +// } +// rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; +// (*pSql->fp)(pSql->param, pSql, rpcMsg->code); +// } +// +// if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it +// tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); +// taosRemoveRef(tscObjRef, handle); +// } +// +// taosReleaseRef(tscObjRef, handle); +// rpcFreeCont(rpcMsg->pCont); +// free(rpcMsg); +// free(pEpSet); +//} +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); if (pSql == NULL) { rpcFreeCont(rpcMsg->pCont); - free(rpcMsg); - free(pEpSet); return; } @@ -364,8 +537,6 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { taosRemoveRef(tscObjRef, handle); taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); - free(rpcMsg); - free(pEpSet); return; } @@ -377,8 +548,6 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { taosRemoveRef(tscObjRef, handle); taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); - free(rpcMsg); - free(pEpSet); return; } @@ -432,8 +601,6 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); - free(rpcMsg); - free(pEpSet); return; } } @@ -511,29 +678,6 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); - free(rpcMsg); - free(pEpSet); -} - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - SSchedMsg schedMsg = {0}; - - schedMsg.fp = doProcessMsgFromServer; - - SRpcMsg* rpcMsgCopy = calloc(1, sizeof(SRpcMsg)); - memcpy(rpcMsgCopy, rpcMsg, sizeof(struct SRpcMsg)); - schedMsg.ahandle = (void*)rpcMsgCopy; - - SRpcEpSet* pEpSetCopy = NULL; - if (pEpSet != NULL) { - pEpSetCopy = calloc(1, sizeof(SRpcEpSet)); - memcpy(pEpSetCopy, pEpSet, sizeof(SRpcEpSet)); - } - - schedMsg.thandle = (void*)pEpSetCopy; - schedMsg.msg = NULL; - - taosScheduleTask(tscQhandle, &schedMsg); } int doBuildAndSendMsg(SSqlObj *pSql) { From cd9e338795a33eb5e3f5ce295365e980f79b271e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Aug 2021 19:47:20 +0000 Subject: [PATCH 04/49] [TD-6214] improve insert performance --- src/client/src/tscServer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 045fd5ca5c..06308f9e0c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -530,7 +530,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pSql->cmd; pSql->rpcRid = -1; - if (pObj->signature != pObj) { tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); From 27dab670f4790ef84e7ae79aa6ab63550efa8634 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 19 Aug 2021 09:36:09 +0800 Subject: [PATCH 05/49] fix illegal sql issue --- src/client/src/tscPrepare.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 06a9505086..2418e7dbdb 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -206,6 +206,8 @@ static int normalStmtPrepare(STscStmt* stmt) { return code; } start = i + token.n; + } else if (token.type == TK_ILLEGAL) { + return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid sql"); } i += token.n; From 9449c97ff5c0e145ae899c8237edde143ea60262 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Thu, 19 Aug 2021 09:52:53 +0800 Subject: [PATCH 06/49] [TD-6167] stmt api test --- tests/script/api/makefile | 5 +- tests/script/api/stmt_function.c | 383 +++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 tests/script/api/stmt_function.c diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 7595594cbf..92d0a89b0f 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -6,7 +6,8 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ + -fsanitize=address all: $(TARGET) @@ -14,8 +15,10 @@ exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) gcc $(CFLAGS) ./stmtTest.c -o $(ROOT)stmtTest $(LFLAGS) + gcc $(CFLAGS) ./stmt_function.c -o $(ROOT)stmt_function $(LFLAGS) clean: rm $(ROOT)batchprepare rm $(ROOT)stmtBatchTest rm $(ROOT)stmtTest + rm $(ROOT)stmt_function diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c new file mode 100644 index 0000000000..13ae90c211 --- /dev/null +++ b/tests/script/api/stmt_function.c @@ -0,0 +1,383 @@ +#include +#include +#include +#include "taos.h" +#include +#include +#include +#include + +void execute_simple_sql(void *taos, char *sql) { + TAOS_RES *result = taos_query(taos, sql); + if ( result == NULL || taos_errno(result) != 0) { + printf( "failed to %s, Reason: %s\n" , sql, taos_errstr(result)); + taos_free_result(result); + exit(EXIT_FAILURE); + } + taos_free_result(result); +} + +void taos_stmt_init_test() { + printf("start taos_stmt_init test \n"); + void *taos = NULL; + TAOS_STMT *stmt = NULL; + stmt = taos_stmt_init(taos); + assert(stmt == NULL); + // ASM ERROR + // assert(taos_stmt_close(stmt) != 0); + taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_init test\n"); +} +void taos_stmt_preprare_test() { + printf("start taos_stmt_prepare test\n"); + char *stmt_sql = calloc(1, 1048576); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + void *taos = NULL; + taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + // below will make client dead lock + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + + // assert(taos_stmt_close(stmt) == 0); + // stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "select from ?"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into ? values (?,?,?,?,?,?,?,?,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into super values (?,?,?,?,?,?,?,?,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into ? values (?,?,?,?,?,?,?,?,1,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_close(stmt) == 0); + + free(stmt_sql); + printf("finish taos_stmt_prepare test\n"); +} + +void taos_stmt_set_tbname_test() { + printf("start taos_stmt_set_tbname test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1, 200); + // ASM ERROR + // assert(taos_stmt_set_tbname(stmt, name) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_set_tbname(stmt, name) != 0); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + sprintf(name, "super"); + assert(stmt != NULL); + assert(taos_stmt_set_tbname(stmt, name) == 0); + free(name); + free(stmt_sql); + taos_stmt_close(stmt); + printf("finish taos_stmt_set_tbname test\n"); +} + +void taos_stmt_set_tbname_tags_test() { + printf("start taos_stmt_set_tbname_tags test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1,20); + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND)); + // ASM ERROR + // assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create stable super(ts timestamp, c1 int) tags (id int)"); + execute_simple_sql(taos, "create table tb using super tags (1)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? using super tags (?) values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + sprintf(name, "tb"); + assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + int t = 1; + tags->buffer_length = TSDB_DATA_TYPE_INT; + tags->buffer_length = sizeof(uint32_t); + tags->buffer = &t; + tags->length = &tags->buffer_length; + tags->is_null = NULL; + assert(taos_stmt_set_tbname_tags(stmt, name, tags) == 0); + free(stmt_sql); + free(name); + free(tags); + taos_stmt_close(stmt); + printf("finish taos_stmt_set_tbname_tags test\n"); +} + +void taos_stmt_set_sub_tbname_test() { + printf("start taos_stmt_set_sub_tbname test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1, 200); + // ASM ERROR + // assert(taos_stmt_set_sub_tbname(stmt, name) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create stable super(ts timestamp, c1 int) tags (id int)"); + execute_simple_sql(taos, "create table tb using super tags (1)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_set_sub_tbname(stmt, name) != 0); + sprintf(name, "tb"); + assert(taos_stmt_set_sub_tbname(stmt, name) == 0); + // assert(taos_load_table_info(taos, "super, tb") == 0); + // assert(taos_stmt_set_sub_tbname(stmt, name) == 0); + free(name); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_set_sub_tbname test\n"); +} + +void taos_stmt_bind_param_test() { + printf("start taos_stmt_bind_param test\n"); + TAOS_STMT *stmt = NULL; + TAOS_BIND *binds = NULL; + assert(taos_stmt_bind_param(stmt, binds) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_bind_param(stmt, binds) != 0); + free(binds); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_bind_param(stmt, params) != 0); + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + free(params); + free(stmt_sql); + taos_stmt_close(stmt); + printf("finish taos_stmt_bind_param test\n"); +} + +void taos_stmt_bind_single_param_batch_test() { + printf("start taos_stmt_bind_single_param_batch test\n"); + TAOS_STMT *stmt = NULL; + TAOS_MULTI_BIND *bind = NULL; + assert(taos_stmt_bind_single_param_batch(stmt, bind, 0) != 0); + printf("finish taos_stmt_bind_single_param_batch test\n"); +} + +void taos_stmt_bind_param_batch_test() { + printf("start taos_stmt_bind_param_batch test\n"); + TAOS_STMT *stmt = NULL; + TAOS_MULTI_BIND *bind = NULL; + assert(taos_stmt_bind_param_batch(stmt, bind) != 0); + printf("finish taos_stmt_bind_param_batch test\n"); +} + +void taos_stmt_add_batch_test() { + printf("start taos_stmt_add_batch test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_add_batch(stmt) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_add_batch(stmt) != 0); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + assert(taos_stmt_add_batch(stmt) == 0); + free(params); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_add_batch test\n"); +} + +void taos_stmt_execute_test() { + printf("start taos_stmt_execute test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_execute(stmt) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_execute(stmt) != 0); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_execute(stmt) != 0); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_execute(stmt) != 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + assert(taos_stmt_execute(stmt) != 0); + assert(taos_stmt_add_batch(stmt) == 0); + assert(taos_stmt_execute(stmt) == 0); + free(params); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_execute test\n"); +} + +void taos_stmt_use_result_test() { + printf("start taos_stmt_use_result test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_use_result(stmt) == NULL); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + printf("finish taos_stmt_use_result test\n"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + TAOS_RES* result = taos_stmt_use_result(stmt); + int rows = 0; + TAOS_ROW row; + while ((row = taos_fetch_row(result))) { + rows++; + } + printf("rows: %d\n", rows); + taos_free_result(result); + // assert(taos_stmt_use_result(stmt) == NULL); + assert(taos_stmt_close(stmt) == 0); +} + +void taos_stmt_close_test() { + printf("start taos_stmt_close test\n"); + // ASM ERROR + // TAOS_STMT *stmt = NULL; + // assert(taos_stmt_close(stmt) != 0); + printf("finish taos_stmt_close test\n"); +} + +void test_api_reliability() { + // ASM catch memory leak + taos_stmt_init_test(); + taos_stmt_preprare_test(); + taos_stmt_set_tbname_test(); + taos_stmt_set_tbname_tags_test(); + taos_stmt_set_sub_tbname_test(); + taos_stmt_bind_param_test(); + taos_stmt_bind_single_param_batch_test(); + taos_stmt_bind_param_batch_test(); + taos_stmt_add_batch_test(); + taos_stmt_execute_test(); + taos_stmt_use_result_test(); + taos_stmt_close_test(); +} + +int main(int argc, char *argv[]) { + test_api_reliability(); + return 0; +} \ No newline at end of file From d453be0785236097dd6a58eae778486d0e00b556 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 Aug 2021 10:26:50 +0800 Subject: [PATCH 07/49] fix: fix altering schema too much corrupting meta file --- src/tsdb/src/tsdbMeta.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 96e86a6d99..547030a431 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1250,8 +1250,14 @@ static int tsdbEncodeTable(void **buf, STable *pTable) { tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); tlen += tdEncodeKVRow(buf, pTable->tagVal); } else { - tlen += taosEncodeFixedU8(buf, (uint8_t)taosArrayGetSize(pTable->schema)); - for (int i = 0; i < taosArrayGetSize(pTable->schema); i++) { + uint32_t arraySize = (uint32_t)taosArrayGetSize(pTable->schema); + if(arraySize > UINT8_MAX) { + tlen += taosEncodeFixedU8(buf, 0); + tlen += taosEncodeFixedU32(buf, arraySize); + } else { + tlen += taosEncodeFixedU8(buf, (uint8_t)arraySize); + } + for (uint32_t i = 0; i < arraySize; i++) { STSchema *pSchema = taosArrayGetP(pTable->schema, i); tlen += tdEncodeSchema(buf, pSchema); } @@ -1284,8 +1290,11 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); buf = tdDecodeKVRow(buf, &(pTable->tagVal)); } else { - uint8_t nSchemas; - buf = taosDecodeFixedU8(buf, &nSchemas); + uint32_t nSchemas; + buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); + if(nSchemas == 0) { + buf = taosDecodeFixedU32(buf, &nSchemas); + } for (int i = 0; i < nSchemas; i++) { STSchema *pSchema; buf = tdDecodeSchema(buf, &pSchema); @@ -1485,4 +1494,4 @@ static void tsdbFreeTableSchema(STable *pTable) { taosArrayDestroy(pTable->schema); } -} \ No newline at end of file +} From 13723b04b88ca012fd02447b42f86d1ac2dda642 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 19 Aug 2021 10:30:02 +0800 Subject: [PATCH 08/49] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 148c33106a..261e3e8162 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.7.1") + SET(TD_VER_NUMBER "2.1.7.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 859d40cf69..ea5ce3bc52 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.1.7.1' +version: '2.1.7.2' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.7.1 + - usr/lib/libtaos.so.2.1.7.2 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 22d3dda4b5171697d2f824c6dc41b6d4fc39e034 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 Aug 2021 10:42:28 +0800 Subject: [PATCH 09/49] fix: init problem --- src/tsdb/src/tsdbMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 547030a431..8407a0519a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1290,7 +1290,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); buf = tdDecodeKVRow(buf, &(pTable->tagVal)); } else { - uint32_t nSchemas; + uint32_t nSchemas = 0; buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); if(nSchemas == 0) { buf = taosDecodeFixedU32(buf, &nSchemas); From 8104f47d7905f88661c17b4b3d9a8b3f8cceee49 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 19 Aug 2021 13:03:11 +0800 Subject: [PATCH 10/49] [TD-6155]:leave free tag expr at the end of sql parsing --- src/client/src/tscSQLParser.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 235ba15c8b..30a46ccc48 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4884,10 +4884,6 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { tSqlExprDestroy(pCondExpr->pTableCond); } - if (pCondExpr->pTagCond) { - tSqlExprDestroy(pCondExpr->pTagCond); - } - if (pCondExpr->pColumnCond) { tSqlExprDestroy(pCondExpr->pColumnCond); } From f56a0d810d47953f601ca4e513534a9798d2264e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 19 Aug 2021 13:03:11 +0800 Subject: [PATCH 11/49] [TD-6155]:leave free tag expr at the end of sql parsing --- src/client/src/tscSQLParser.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 235ba15c8b..30a46ccc48 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4884,10 +4884,6 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { tSqlExprDestroy(pCondExpr->pTableCond); } - if (pCondExpr->pTagCond) { - tSqlExprDestroy(pCondExpr->pTagCond); - } - if (pCondExpr->pColumnCond) { tSqlExprDestroy(pCondExpr->pColumnCond); } From a46727f325899072bb634b286435f4491df3d933 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 19 Aug 2021 14:18:18 +0800 Subject: [PATCH 12/49] [TD-6209]add case for TD-6027 --- tests/pytest/alter/alter_table.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/pytest/alter/alter_table.py b/tests/pytest/alter/alter_table.py index a5acb7a73e..33e0aec727 100644 --- a/tests/pytest/alter/alter_table.py +++ b/tests/pytest/alter/alter_table.py @@ -102,6 +102,20 @@ class TDTestCase: print("check2: i=%d colIdx=%d" % (i, colIdx)) tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3)) + def alter_table_255_times(self): # add case for TD-6207 + for i in range(255): + tdLog.info("alter table st add column cb%d int"%i) + tdSql.execute("alter table st add column cb%d int"%i) + tdSql.execute("insert into t0 (ts,c1) values(now,1)") + tdSql.execute("reset query cache") + tdSql.query("select * from st") + tdSql.execute("create table mt(ts timestamp, i int)") + tdSql.execute("insert into mt values(now,11)") + tdSql.query("select * from mt") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("describe db.st") + def run(self): # Setup params db = "db" @@ -131,12 +145,14 @@ class TDTestCase: tdSql.checkData(0, i, self.rowNum * (size - i)) - tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float)") - tdSql.execute("create table t0 using st tags(null)") + tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float,t2 int,t3 double)") + tdSql.execute("create table t0 using st tags(null,1,2.3)") tdSql.execute("alter table t0 set tag t1=2.1") tdSql.query("show tables") tdSql.checkRows(2) + self.alter_table_255_times() + def stop(self): tdSql.close() From e914a0e83fd27b44ea8fd809aa467756997a1aae Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 19 Aug 2021 14:13:17 +0800 Subject: [PATCH 13/49] [ci skip] update scirpt for csv generator --- .../pytest/insert/insertFromCSVPerformance.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py index f3b9c2734d..487497631a 100644 --- a/tests/pytest/insert/insertFromCSVPerformance.py +++ b/tests/pytest/insert/insertFromCSVPerformance.py @@ -28,7 +28,7 @@ class insertFromCSVPerformace: self.tbName = tbName self.branchName = branchName self.type = buildType - self.ts = 1500074556514 + self.ts = 1500000000000 self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -46,13 +46,20 @@ class insertFromCSVPerformace: config = self.config) def writeCSV(self): - with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + tsset = set() + rows = 0 + with open('test4.csv','w', encoding='utf-8', newline='') as csvFile: writer = csv.writer(csvFile, dialect='excel') - for i in range(1000000): - newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) - d = datetime.datetime.fromtimestamp(newTimestamp / 1000) - dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) - writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + while True: + newTimestamp = self.ts + random.randint(1, 10) * 10000000000 + random.randint(1, 10) * 1000000000 + random.randint(1, 10) * 100000000 + random.randint(1, 10) * 10000000 + random.randint(1, 10) * 1000000 + random.randint(1, 10) * 100000 + random.randint(1, 10) * 10000 + random.randint(1, 10) * 1000 + random.randint(1, 10) * 100 + random.randint(1, 10) * 10 + random.randint(1, 10) + if newTimestamp not in tsset: + tsset.add(newTimestamp) + d = datetime.datetime.fromtimestamp(newTimestamp / 1000) + dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) + writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + rows += 1 + if rows == 2000000: + break def removCSVHeader(self): data = pd.read_csv("ordered.csv") @@ -71,7 +78,9 @@ class insertFromCSVPerformace: cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") startTime = time.time() cursor.execute("insert into t1 file 'outoforder.csv'") - totalTime += time.time() - startTime + totalTime += time.time() - startTime + time.sleep(1) + out_of_order_time = (float) (totalTime / 10) print("Out of Order - Insert time: %f" % out_of_order_time) @@ -81,7 +90,8 @@ class insertFromCSVPerformace: cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") startTime = time.time() cursor.execute("insert into t2 file 'ordered.csv'") - totalTime += time.time() - startTime + totalTime += time.time() - startTime + time.sleep(1) in_order_time = (float) (totalTime / 10) print("In order - Insert time: %f" % in_order_time) From b0b3810ffb248b6abf38d85778e0da4801ed0a68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 19 Aug 2021 18:02:55 +0800 Subject: [PATCH 14/49] [td-6229] Send the query request in parallel if the number of subquery exceeds the threshold. --- src/client/src/tscSubquery.c | 57 +++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 422d07ccde..75307087b1 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -15,8 +15,9 @@ #define _GNU_SOURCE #include "os.h" - #include "texpr.h" + +#include "tsched.h" #include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" @@ -2423,6 +2424,26 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { return terrno; } +typedef struct SPair { + int32_t first; + int32_t second; +} SPair; + +static void doSendQueryReqs(SSchedMsg* pSchedMsg) { + SSqlObj* pSql = pSchedMsg->ahandle; + SPair* p = pSchedMsg->msg; + + for(int32_t i = p->first; i < p->second; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + SRetrieveSupport* pSupport = pSub->param; + + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); + tscProcessSql(pSub); + } + + tfree(p); +} + int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2546,13 +2567,33 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { doCleanupSubqueries(pSql, i); return pRes->code; } - - for(int32_t j = 0; j < pState->numOfSub; ++j) { - SSqlObj* pSub = pSql->pSubs[j]; - SRetrieveSupport* pSupport = pSub->param; - - tscDebug("0x%"PRIx64" sub:%p launch subquery, orderOfSub:%d.", pSql->self, pSub, pSupport->subqueryIndex); - tscProcessSql(pSub); + + // concurrently sent the query requests. + const int32_t MAX_REQUEST_PER_TASK = 8; + + int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; + assert(numOfTasks >= 1); + + int32_t num = (pState->numOfSub/numOfTasks) + 1; + tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); + + for(int32_t j = 0; j < numOfTasks; ++j) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = doSendQueryReqs; + schedMsg.ahandle = (void*)pSql; + + schedMsg.thandle = NULL; + SPair* p = calloc(1, sizeof(SPair)); + p->first = j * num; + + if (j == numOfTasks - 1) { + p->second = pState->numOfSub; + } else { + p->second = (j + 1) * num; + } + + schedMsg.msg = p; + taosScheduleTask(tscQhandle, &schedMsg); } return TSDB_CODE_SUCCESS; From 0afe768d767b8f9e7db929abb73d5fe39b177848 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 18:20:57 +0800 Subject: [PATCH 15/49] [TD-6228] fix tag condtion is too long error --- src/client/src/tscServer.c | 8 ++++---- src/client/src/tscUtil.c | 16 ++++++++++++++++ src/inc/taosmsg.h | 2 +- src/query/src/qExecutor.c | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f8c072a239..953914c504 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -621,7 +621,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - + int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); @@ -643,8 +643,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { tableSerialize = totalTables * sizeof(STableIdInfo); } - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg, int32_t *succeed) { @@ -1099,7 +1099,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); + pQueryMsg->tagCondLen = htonl(pCond->len); memcpy(pMsg, pCond->cond, pCond->len); pMsg += pCond->len; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 773243891f..5e59f54c88 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2989,4 +2989,20 @@ STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { return p; } +int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo) { + int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); + int32_t len = 0; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); + for (int32_t j = 0; j < pCol->numOfFilters; ++j) { + SColumnFilterInfo *pColFilter = &pCol->filterInfo[j]; + len += sizeof(SColumnFilterInfo); + if (pColFilter->filterstr) { + len += (int32_t)pColFilter->len + 1; + } + } + } + return len; +} diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index d7ac7dd277..2d78210844 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -494,7 +494,7 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; SSessionWindow sw; // session window - uint16_t tagCondLen; // tag length in current query + uint32_t tagCondLen; // tag length in current query uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6b32508f07..f96d33f060 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6172,7 +6172,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); From d837e47cb6e9ec24925d9d50789484436306176e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 18:47:37 +0800 Subject: [PATCH 16/49] [TD-6228] fix tag condtion is too long error --- src/client/inc/tscUtil.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 47b2865313..a26f03003e 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -301,6 +301,7 @@ uint32_t tscGetTableMetaMaxSize(); int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); +int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); From 7c7f89a3eb7176313c15450fe4a0b84e85f0e4c3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 21:04:04 +0800 Subject: [PATCH 17/49] [TD-6228] fix tag condtion is too long error --- src/client/src/tscServer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 953914c504..599767803c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -643,6 +643,12 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { tableSerialize = totalTables * sizeof(STableIdInfo); } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } From 411543694be7de2b5f300d789fd4314bcca4b6a7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 19 Aug 2021 21:29:52 +0800 Subject: [PATCH 18/49] Hotfix/sangshuduo/td 5136 taosdemo rework for master (#7473) * cherry pick from develop branch. * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * change malloc to calloc for pid allocation. * fix typo. * fix binary length default value and group id logic. * fix help msg. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9a7d3cd25c..316fed117d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -752,12 +752,13 @@ static void printHelp() { "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC."); printf("%s%s%s%s\n", indent, "-b", indent, "The data_type of columns, default: FLOAT, INT, FLOAT."); - printf("%s%s%s%s\n", indent, "-w", indent, - "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s%s%s%d\n", indent, "-w", indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is ", + g_args.len_of_binary); printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, - "The number of columns per record. Default is ", + "The number of columns per record. Demo mode by default is ", DEFAULT_DATATYPE_NUM, - ". Max values is ", + " (float, int, float). Max values is ", MAX_NUM_COLUMNS); printf("%s%s%s%s\n", indent, indent, indent, "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); From 7019c8f3331eff6e18c228f1461e389bbaaffaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=A1=E6=AF=94=E5=8D=A1=E6=AF=94?= <30525741+jackwener@users.noreply.github.com> Date: Thu, 19 Aug 2021 21:31:12 +0800 Subject: [PATCH 19/49] Update docs.md --- documentation20/cn/01.evaluation/docs.md | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 7f70ccec56..4aba82a54c 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -2,18 +2,18 @@ ## TDengine 简介 -TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。 +TDengine 是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL 数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。 -TDengine的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与Hadoop等典型的大数据平台相比，它具有如下鲜明的特点： +TDengine 的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine 还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与 Hadoop 等典型的大数据平台相比，它具有如下鲜明的特点： -* __10倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少2万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。 -* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。 -* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 -* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, MATLAB随时进行。 -* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 -* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。 +* __10倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少 2 万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。 +* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的 1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的 1/10。 +* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成 Kafka/Redis/HBase/Spark/HDFS 等软件，大幅降低应用开发和维护的复杂度成本。 +* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过 Shell, Python, R, MATLAB 随时进行。 +* __与第三方工具无缝连接__：不用一行代码，即可与 Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R 等集成。后续将支持 OPC, Hadoop, Spark 等，BI 工具也将无缝连接。 +* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类标准 SQL，支持 RESTful，支持 Python/Java/C/C++/C#/Go/Node.js, 与 MySQL 相似，零学习成本。 -采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 +采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。 ![TDengine技术生态图](page://images/eco_system.png)
图 1. TDengine技术生态图
@@ -21,11 +21,12 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 ## TDengine 总体适用场景 -作为一个IOT大数据平台，TDengine的典型适用场景是在IOT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。 +作为一个 IOT 大数据平台，TDengine 的典型适用场景是在 IOT 范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如 CRM，ERP 等，不在本文讨论范围内。 ### 数据源特点和需求 -从数据源角度，设计人员可以从下面几个角度分析TDengine在目标应用系统里面的适用性。 + +从数据源角度，设计人员可以从下面几个角度分析 TDengine 在目标应用系统里面的适用性。 |数据源特点和需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| @@ -34,6 +35,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 |数据源数目巨大| | | √ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。| ### 系统架构要求 + |系统架构要求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求简单可靠的系统架构| | | √ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。| @@ -41,12 +43,14 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 |标准化规范| | | √ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范。| ### 系统功能需求 + |系统功能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求完整的内置数据处理算法| | √ | |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。| |需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能。| ### 系统性能需求 + |系统性能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求较大的总体处理能力| | | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| @@ -54,6 +58,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 |要求快速处理小粒度数据| | | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| ### 系统维护需求 + |系统维护需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求系统可靠运行| | | √ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。| From 97281931da9b60325c1902d997d01ae9a3e4796d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 20 Aug 2021 01:09:32 +0800 Subject: [PATCH 20/49] [TD-6219]: remove few taosdemo macros. (#7469) --- src/kit/taosdemo/taosdemo.c | 62 ------------------------------------- 1 file changed, 62 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 316fed117d..343c3f2e03 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -53,14 +53,6 @@ #include "taoserror.h" #include "tutil.h" -#define STMT_IFACE_ENABLED 1 -#define NANO_SECOND_ENABLED 1 -#define SET_THREADNAME_ENABLED 1 - -#if SET_THREADNAME_ENABLED == 0 -#define setThreadName(name) -#endif - #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -295,9 +287,7 @@ typedef struct SSuperTable_S { uint64_t lenOfTagOfOneRow; char* sampleDataBuf; -#if STMT_IFACE_ENABLED == 1 char* sampleBindArray; -#endif //int sampleRowCount; //int sampleUsePos; @@ -733,11 +723,7 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-P", indent, "The TCP/IP port number to use for the connection. Default is 0."); printf("%s%s%s%s\n", indent, "-I", indent, -#if STMT_IFACE_ENABLED == 1 "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); -#else - "The interface (taosc, rest) taosdemo uses. Default is 'taosc'."); -#endif printf("%s%s%s%s\n", indent, "-d", indent, "Destination database. Default is 'test'."); printf("%s%s%s%s\n", indent, "-a", indent, @@ -850,10 +836,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = TAOSC_IFACE; } else if (0 == strcasecmp(argv[i], "rest")) { arguments->iface = REST_IFACE; -#if STMT_IFACE_ENABLED == 1 } else if (0 == strcasecmp(argv[i], "stmt")) { arguments->iface = STMT_IFACE; -#endif } else { errorPrint("%s", "\n\t-I need a valid string following!\n"); exit(EXIT_FAILURE); @@ -1695,9 +1679,7 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))) { printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); @@ -1888,9 +1870,7 @@ static void printfInsertMetaToFile(FILE* fp) { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); @@ -2093,10 +2073,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); -#if NANO_SECOND_ENABLED == 1 } if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); -#endif } else { tt = (time_t)(val / 1000); } @@ -2118,10 +2096,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(buf + pos, ".%06d", (int)(val % 1000000)); -#if NANO_SECOND_ENABLED == 1 } else if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); -#endif } else { sprintf(buf + pos, ".%03d", (int)(val % 1000)); } @@ -3182,10 +3158,8 @@ int createDatabasesAndStables(char *command) { " fsync %d", g_Dbs.db[i].dbCfg.fsync); } if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, @@ -4264,10 +4238,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { g_Dbs.db[i].superTbls[j].iface= REST_IFACE; -#if STMT_IFACE_ENABLED == 1 } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; -#endif } else { errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", __func__, __LINE__, stbIface->valuestring); @@ -5076,7 +5048,6 @@ static void postFreeResource() { free(g_Dbs.db[i].superTbls[j].sampleDataBuf); g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; } -#if STMT_IFACE_ENABLED == 1 if (g_Dbs.db[i].superTbls[j].sampleBindArray) { for (int k = 0; k < MAX_SAMPLES_ONCE_FROM_FILE; k++) { uintptr_t *tmp = (uintptr_t *)(*(uintptr_t *)( @@ -5091,7 +5062,6 @@ static void postFreeResource() { } } tmfree((char *)g_Dbs.db[i].superTbls[j].sampleBindArray); -#endif if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { free(g_Dbs.db[i].superTbls[j].tagDataBuf); @@ -5384,7 +5354,6 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) } break; -#if STMT_IFACE_ENABLED == 1 case STMT_IFACE: debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); @@ -5397,7 +5366,6 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) } affectedRows = k; break; -#endif default: errorPrint("%s() LN%d: unknown insert mode: %d\n", @@ -5770,7 +5738,6 @@ static int64_t generateInterlaceDataWithoutStb( return k; } -#if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType( TAOS_BIND *bind, char *dataType, int32_t dataLen, @@ -6605,7 +6572,6 @@ static int32_t prepareStbStmtWithSample( return k; } -#endif static int32_t generateStbProgressiveData( SSuperTable *stbInfo, @@ -6806,7 +6772,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 if (sourceRand) { generated = prepareStbStmtRand( pThreadInfo, @@ -6826,9 +6791,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTime, &(pThreadInfo->samplePos)); } -#else - generated = -1; -#endif } else { generated = generateStbInterlaceData( pThreadInfo, @@ -6846,16 +6808,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, tableName, batchPerTbl, startTime); -#if STMT_IFACE_ENABLED == 1 generated = prepareStmtWithoutStb( pThreadInfo, tableName, batchPerTbl, insertRows, i, startTime); -#else - generated = -1; -#endif } else { generated = generateInterlaceDataWithoutStb( tableName, batchPerTbl, @@ -7068,7 +7026,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 if (sourceRand) { generated = prepareStbStmtRand( pThreadInfo, @@ -7087,9 +7044,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { insertRows, i, start_time, &(pThreadInfo->samplePos)); } -#else - generated = -1; -#endif } else { generated = generateStbProgressiveData( stbInfo, @@ -7101,16 +7055,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { } } else { if (g_args.iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 generated = prepareStmtWithoutStb( pThreadInfo, tableName, g_args.num_of_RPR, insertRows, i, start_time); -#else - generated = -1; -#endif } else { generated = generateProgressiveDataWithoutStb( tableName, @@ -7330,7 +7280,6 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * return 0; } -#if STMT_IFACE_ENABLED == 1 static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) { stbInfo->sampleBindArray = calloc(1, sizeof(char *) * MAX_SAMPLES_ONCE_FROM_FILE); @@ -7401,7 +7350,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return 0; } -#endif static void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* stbInfo) { @@ -7412,10 +7360,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, timePrec = TSDB_TIME_PRECISION_MILLI; } else if (0 == strncasecmp(precision, "us", 2)) { timePrec = TSDB_TIME_PRECISION_MICRO; -#if NANO_SECOND_ENABLED == 1 } else if (0 == strncasecmp(precision, "ns", 2)) { timePrec = TSDB_TIME_PRECISION_NANO; -#endif } else { errorPrint("Not support precision: %s\n", precision); exit(EXIT_FAILURE); @@ -7558,7 +7504,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, memset(pids, 0, threads * sizeof(pthread_t)); memset(infos, 0, threads * sizeof(threadInfo)); -#if STMT_IFACE_ENABLED == 1 char *stmtBuffer = calloc(1, BUFFER_SIZE); assert(stmtBuffer); if ((g_args.iface == STMT_IFACE) @@ -7599,7 +7544,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, parseSampleFileToStmt(stbInfo, timePrec); } } -#endif for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; @@ -7627,7 +7571,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(EXIT_FAILURE); } -#if STMT_IFACE_ENABLED == 1 if ((g_args.iface == STMT_IFACE) || ((stbInfo) && (stbInfo->iface == STMT_IFACE))) { @@ -7655,7 +7598,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, } pThreadInfo->bind_ts = malloc(sizeof(int64_t)); } -#endif } else { pThreadInfo->taos = NULL; } @@ -7681,9 +7623,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } -#if STMT_IFACE_ENABLED == 1 free(stmtBuffer); -#endif for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); @@ -7698,12 +7638,10 @@ static void startMultiThreadInsertData(int threads, char* db_name, for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; -#if STMT_IFACE_ENABLED == 1 if (pThreadInfo->stmt) { taos_stmt_close(pThreadInfo->stmt); tmfree((char *)pThreadInfo->bind_ts); } -#endif tsem_destroy(&(pThreadInfo->lock_sem)); taos_close(pThreadInfo->taos); From 9c04da10a0046d2ed72e55467e4a354bd8372878 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 20 Aug 2021 09:43:36 +0800 Subject: [PATCH 21/49] block union for nested query --- src/client/src/tscSQLParser.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 30a46ccc48..f1349d7d41 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -888,6 +888,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SELECT: { + const char * msg1 = "no nested query supported in union clause"; code = loadAllTableMeta(pSql, pInfo); if (code != TSDB_CODE_SUCCESS) { return code; @@ -901,6 +902,10 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); + if (size > 1 && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + // normalizeSqlNode(pSqlNode); // normalize the column name in each function if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { return code; From 1b5798431167177d2196204eec4a1757e7a63439 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 20 Aug 2021 10:15:15 +0800 Subject: [PATCH 22/49] Hotfix/sangshuduo/td 5844 cmdline parameters align for master (#7479) * [TD-5844]: make cmd line parameter similar. * fix test case align with taosdemo change. * fix windows stack overflow issue. * fix mac compile error. * fix taosdemo cmdline parameter in tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py * make taos.exe use mysql style password input. * make taos shell and taosdump use mysql style password input. * determine scanf return value. * cherry pick from develop/feature branch. * cherry pick with ab6cde02626bd8ba496b98a65e37e1f98c9ff596 * cherry pick with d6bfe443fd21ded247bd8509fb77b7b18e112357 Co-authored-by: Shuduo Sang --- src/inc/taosdef.h | 2 ++ src/kit/shell/inc/shell.h | 1 - src/kit/shell/src/shellDarwin.c | 14 +++++++++---- src/kit/shell/src/shellLinux.c | 23 ++++++++++++--------- src/kit/shell/src/shellWindows.c | 14 +++++++++---- src/kit/taosdemo/taosdemo.c | 23 +++++++++++---------- src/kit/taosdump/taosdump.c | 34 +++++++++++++++++++++----------- 7 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 94be247b0d..bc4ddbe067 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -87,6 +87,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_PASS "taosdata" #endif +#define SHELL_MAX_PASSWORD_LEN 20 + #define TSDB_TRUE 1 #define TSDB_FALSE 0 #define TSDB_OK 0 diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 2374150c52..c9d58fd3e1 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -25,7 +25,6 @@ #define MAX_USERNAME_SIZE 64 #define MAX_DBNAME_SIZE 64 #define MAX_IP_SIZE 20 -#define MAX_PASSWORD_SIZE 20 #define MAX_HISTORY_SIZE 1000 #define MAX_COMMAND_SIZE 1048586 #define HISTORY_FILE ".taos_history" diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 5ca4537aeb..9161860f07 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -66,7 +66,7 @@ void printHelp() { char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { wordexp_t full_path; @@ -81,19 +81,25 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strncmp(argv[i], "-p", 2) == 0) { + else if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { strcpy(tsOsName, "Darwin"); printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%s", g_password) > 1) { fprintf(stderr, "password read error\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } arguments->password = g_password; + strcpy(argv[i], ""); + argc -= 1; } // for management port else if (strcmp(argv[i], "-P") == 0) { diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index d051d3535e..b177e22e86 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -47,8 +47,8 @@ static struct argp_option options[] = { {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"check", 'k', "CHECK", 0, "Check tables."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, - {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, + {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; @@ -74,7 +74,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } break; - case 't': + case 'z': arguments->timezone = arg; break; case 'u': @@ -160,22 +160,27 @@ static struct argp argp = {options, parse_opt, args_doc, doc}; char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; -static void parse_password( +static void parse_args( int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { - if (strncmp(argv[i], "-p", 2) == 0) { + if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { strcpy(tsOsName, "Linux"); printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%20s", g_password) > 1) { fprintf(stderr, "password reading error\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); } arguments->password = g_password; arguments->is_use_passwd = true; @@ -190,7 +195,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { argp_program_version = verType; if (argc > 1) { - parse_password(argc, argv, arguments); + parse_args(argc, argv, arguments); } argp_parse(&argp, argc, argv, 0, 0, arguments); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index bf9afe4b80..d7f96631b6 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -64,7 +64,7 @@ void printHelp() { exit(EXIT_SUCCESS); } -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { @@ -78,20 +78,26 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strncmp(argv[i], "-p", 2) == 0) { + else if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { arguments->is_use_passwd = true; strcpy(tsOsName, "Windows"); printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%s", g_password) > 1) { fprintf(stderr, "password read error!\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } arguments->password = g_password; + strcpy(argv[i], ""); + argc -= 1; } // for management port else if (strcmp(argv[i], "-P") == 0) { diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 343c3f2e03..75da9d9f4b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -69,7 +69,6 @@ extern char configDir[]; #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) #define MAX_USERNAME_SIZE 64 -#define MAX_PASSWORD_SIZE 16 #define MAX_HOSTNAME_SIZE 253 // https://man7.org/linux/man-pages/man7/hostname.7.html #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space @@ -208,7 +207,7 @@ typedef struct SArguments_S { uint16_t port; uint16_t iface; char * user; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char * database; int replica; char * tb_prefix; @@ -356,7 +355,7 @@ typedef struct SDbs_S { uint16_t port; char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char resultFile[MAX_FILE_NAME_LEN]; bool use_metric; bool insert_only; @@ -422,7 +421,7 @@ typedef struct SQueryMetaInfo_S { uint16_t port; struct sockaddr_in serv_addr; char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char dbName[TSDB_DB_NAME_LEN]; char queryMode[SMALL_BUFF_LEN]; // taosc, rest @@ -858,7 +857,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } taosSetConsoleEcho(true); } else { - tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(arguments->password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } } else if (strcmp(argv[i], "-o") == 0) { if (argc == i+1) { @@ -3780,9 +3779,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, "taosdata", SHELL_MAX_PASSWORD_LEN); } cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); @@ -4516,9 +4515,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); + tstrncpy(g_queryInfo.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; + tstrncpy(g_queryInfo.password, "taosdata", SHELL_MAX_PASSWORD_LEN);; } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, @@ -8826,7 +8825,7 @@ static void initOfInsertMeta() { tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_Dbs.port = 6030; tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); g_Dbs.threadCount = 2; g_Dbs.use_metric = g_args.use_metric; @@ -8839,7 +8838,7 @@ static void initOfQueryMeta() { tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_queryInfo.port = 6030; tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); } static void setParaFromArg() { @@ -8853,7 +8852,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); } - tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, g_args.password, SHELL_MAX_PASSWORD_LEN); if (g_args.port) { g_Dbs.port = g_args.port; diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index c54b8da1b7..6b553b3824 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -243,19 +243,15 @@ static struct argp_option options[] = { {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, {"debug", 'g', 0, 0, "Print debug info.", 8}, - {"verbose", 'b', 0, 0, "Print verbose debug info.", 9}, - {"performanceprint", 'm', 0, 0, "Print performance debug info.", 10}, {0} }; -#define MAX_PASSWORD_SIZE 20 - /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option char *host; char *user; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; uint16_t port; char cversion[12]; uint16_t mysqlFlag; @@ -432,7 +428,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; // dump unit option case 'A': - g_args.all_databases = true; break; case 'D': g_args.databases = true; @@ -555,11 +550,14 @@ static void parse_precision_first( } } -static void parse_password( +static void parse_args( int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { - if (strncmp(argv[i], "-p", 2) == 0) { - if (strlen(argv[i]) == 2) { + if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); taosSetConsoleEcho(false); if(scanf("%20s", arguments->password) > 1) { @@ -567,10 +565,22 @@ static void parse_password( } taosSetConsoleEcho(true); } else { - tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(arguments->password, (char *)(argv[i] + 2), + SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); } - argv[i] = ""; + } else if (strcmp(argv[i], "-gg") == 0) { + arguments->verbose_print = true; + strcpy(argv[i], ""); + } else if (strcmp(argv[i], "-PP") == 0) { + arguments->performance_print = true; + strcpy(argv[i], ""); + } else if (strcmp(argv[i], "-A") == 0) { + g_args.all_databases = true; + } else { + continue; } + } } @@ -639,7 +649,7 @@ int main(int argc, char *argv[]) { if (argc > 1) { parse_precision_first(argc, argv, &g_args); parse_timestamp(argc, argv, &g_args); - parse_password(argc, argv, &g_args); + parse_args(argc, argv, &g_args); } argp_parse(&argp, argc, argv, 0, 0, &g_args); From 8ae068ff2e2686b179efd5fdd8da2a276fd532fe Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Aug 2021 10:25:03 +0800 Subject: [PATCH 23/49] [TD-6046] fix bottom,top,diff,derivative ts bugs --- src/client/src/tscGlobalmerge.c | 2 +- src/client/src/tscSQLParser.c | 7 ++- src/query/inc/qExecutor.h | 2 + src/query/src/qExecutor.c | 44 ++++++++++++++++--- tests/pytest/functions/function_bottom.py | 15 +++++++ tests/pytest/functions/function_derivative.py | 22 ++++++++++ tests/pytest/functions/function_diff.py | 17 +++++++ tests/pytest/functions/function_top.py | 15 +++++++ 8 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index e696d54abd..2b635e8d83 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -643,7 +643,7 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD for(int32_t j = 0; j < numOfExpr; ++j) { pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows); if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) { - pCtx[j].ptsOutputBuf = pCtx[0].pOutput; + if(j>0) pCtx[j].ptsOutputBuf = pCtx[j-1].pOutput; } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 30a46ccc48..a13ec08408 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2603,13 +2603,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // set the first column ts for diff query if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); SColumnList ids = createColumnList(1, 0, 0); - insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); @@ -2882,7 +2881,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -5693,7 +5692,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2); + return invalidOperationMsg(pMsgBuf, msg2);qExecutor.h } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index e8b7855769..2b9f952d76 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -597,6 +597,8 @@ bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilter void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); + void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 844b8ec683..26ae714c27 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3616,7 +3616,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i // set the timestamp output buffer for top/bottom/diff query int32_t fid = pCtx[i].functionId; if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } } @@ -3651,7 +3651,37 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + if(i>0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + } + } +} + +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + bool needCopyTs = false; + int32_t tsNum = 0; + char *src = NULL; + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { + needCopyTs = true; + if (i > 0 && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){ + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data + src = pColRes->pData; + } + }else if(functionId == TSDB_FUNC_TS_DUMMY) { + tsNum++; + } + } + + if (!needCopyTs) return; + if (tsNum < 2) return; + if (src == NULL) return; + + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if(functionId == TSDB_FUNC_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); } } } @@ -3851,7 +3881,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } if (!pResInfo->initialized) { @@ -3912,7 +3942,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } /* @@ -5698,6 +5728,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return pRes; } @@ -5723,8 +5754,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { if (*newgroup) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return pInfo->pRes; + break; } else { // init output buffer for a new group data for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); @@ -5754,7 +5784,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { break; } } - + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index abb9ac48e7..e9e5003f6f 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -104,6 +104,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 2) + + tdSql.query("select ts,bottom(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") + + + tdSql.query("select ts,bottom(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") #TD-2457 bottom + interval + order by tdSql.error('select top(col2,1) from test interval(1y) order by col2;') diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 9d60129672..d4d8ab80a6 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -54,6 +54,28 @@ class TDTestCase: tdSql.query("select derivative(col, 10s, 0) from stb group by tbname") tdSql.checkRows(10) + tdSql.query("select ts,derivative(col, 10s, 1),ts from stb group by tbname") + tdSql.checkRows(4) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(3, 0, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") + + tdSql.query("select ts,derivative(col, 10s, 1),ts from tb1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 1, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:20.009") + + tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)") + + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname") tdSql.query("select derivative(col, 10s, 1) from tb1") diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py index fba3b4c0d4..cd5894c165 100644 --- a/tests/pytest/functions/function_diff.py +++ b/tests/pytest/functions/function_diff.py @@ -94,6 +94,23 @@ class TDTestCase: tdSql.error("select diff(col13) from test") tdSql.error("select diff(col14) from test") + tdSql.query("select ts,diff(col1),ts from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + + tdSql.query("select ts,diff(col1),ts from test group by tbname") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index f8318402b5..9824afc19f 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -117,6 +117,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + tdSql.query("select ts,top(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") + + + tdSql.query("select ts,top(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") #TD-2563 top + super_table + interval tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") From deb5279ab7be9fd81a6377160d9b38697649edc5 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 20 Aug 2021 10:37:07 +0800 Subject: [PATCH 24/49] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 84ef060196..24d451e8f7 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.13") + SET(TD_VER_NUMBER "2.0.20.15") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b08ae53ecd..2c19e83b47 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.13' +version: '2.0.20.15' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.13 + - usr/lib/libtaos.so.2.0.20.15 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 8d9ed8e7f9ba6437fb2dfae8bd971d04d1481a74 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 20 Aug 2021 10:46:21 +0800 Subject: [PATCH 25/49] [TD-6234]: add unit test case for TD-6011 --- tests/pytest/query/filterNoKeyword.py | 83 +++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/pytest/query/filterNoKeyword.py diff --git a/tests/pytest/query/filterNoKeyword.py b/tests/pytest/query/filterNoKeyword.py new file mode 100644 index 0000000000..34d74efd82 --- /dev/null +++ b/tests/pytest/query/filterNoKeyword.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("======= Verify filter for bool, nchar and binary type =========") + tdLog.debug( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + tdSql.execute( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + + tdSql.execute("create table st1 using st tags(true, 'table1', '水表')") + for i in range(1, 6): + tdSql.execute( + "insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f)" % + (self.ts + i, i % + 2, i, i, + i, i, i, i, 1.0, 1.0)) + + # =============Data type keywords cannot be used in filter==================== + # timestamp + tdSql.error("select * from st where timestamp = 1629417600") + + # bool + tdSql.error("select * from st where bool = false") + + #binary + tdSql.error("select * from st where binary = 'taosdata'") + + # nchar + tdSql.error("select * from st where nchar = '涛思数据'") + + # tinyint + tdSql.error("select * from st where tinyint = 127") + + # smallint + tdSql.error("select * from st where smallint = 32767") + + # int + tdSql.error("select * from st where INTEGER = 2147483647") + tdSql.error("select * from st where int = 2147483647") + + # bigint + tdSql.error("select * from st where bigint = 2147483647") + + # float + tdSql.error("select * from st where float = 3.4E38") + + # double + tdSql.error("select * from st where double = 1.7E308") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From f9f96640731135e4952d4c87410b28b8d77eba32 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 20 Aug 2021 10:46:21 +0800 Subject: [PATCH 26/49] [TD-6234]: add unit test case for TD-6011 --- tests/pytest/query/filterNoKeyword.py | 83 +++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/pytest/query/filterNoKeyword.py diff --git a/tests/pytest/query/filterNoKeyword.py b/tests/pytest/query/filterNoKeyword.py new file mode 100644 index 0000000000..34d74efd82 --- /dev/null +++ b/tests/pytest/query/filterNoKeyword.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("======= Verify filter for bool, nchar and binary type =========") + tdLog.debug( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + tdSql.execute( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + + tdSql.execute("create table st1 using st tags(true, 'table1', '水表')") + for i in range(1, 6): + tdSql.execute( + "insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f)" % + (self.ts + i, i % + 2, i, i, + i, i, i, i, 1.0, 1.0)) + + # =============Data type keywords cannot be used in filter==================== + # timestamp + tdSql.error("select * from st where timestamp = 1629417600") + + # bool + tdSql.error("select * from st where bool = false") + + #binary + tdSql.error("select * from st where binary = 'taosdata'") + + # nchar + tdSql.error("select * from st where nchar = '涛思数据'") + + # tinyint + tdSql.error("select * from st where tinyint = 127") + + # smallint + tdSql.error("select * from st where smallint = 32767") + + # int + tdSql.error("select * from st where INTEGER = 2147483647") + tdSql.error("select * from st where int = 2147483647") + + # bigint + tdSql.error("select * from st where bigint = 2147483647") + + # float + tdSql.error("select * from st where float = 3.4E38") + + # double + tdSql.error("select * from st where double = 1.7E308") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8f8e86bb8aec06a4f4d70ef048d9d0f8a4ea0fd2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Aug 2021 10:52:46 +0800 Subject: [PATCH 27/49] [TD-6046] fix ts top/bottom/diff/derivative ts error --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a13ec08408..1f62dc8afa 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5692,7 +5692,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2);qExecutor.h + return invalidOperationMsg(pMsgBuf, msg2); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); From b77d26a1d1249913a7b3e9ed67a4224510cd8631 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Fri, 20 Aug 2021 12:49:01 +0800 Subject: [PATCH 28/49] [TD-6176] add stmt query test script --- tests/script/api/stmt_function.c | 149 +++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 15 deletions(-) diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 13ae90c211..64573ec994 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -17,6 +17,20 @@ void execute_simple_sql(void *taos, char *sql) { taos_free_result(result); } +void print_result(TAOS_RES* res) { + if (res == NULL) { + exit(EXIT_FAILURE); + } + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res))) { + char temp[256] = {0}; + taos_print_row(temp, row, fields, num_fields); + printf("get result: %s\n", temp); + } +} + void taos_stmt_init_test() { printf("start taos_stmt_init test \n"); void *taos = NULL; @@ -329,28 +343,129 @@ void taos_stmt_execute_test() { printf("finish taos_stmt_execute test\n"); } +void taos_stmt_use_result_query(void *taos, char *col, int type) { + TAOS_STMT *stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char *stmt_sql = calloc(1, 1024); + struct { + int64_t c1; + int32_t c2; + int64_t c3; + float c4; + double c5; + char c6[8]; + int16_t c7; + int8_t c8; + int8_t c9; + char c10[32]; + } v = {0}; + v.c1 = (int64_t)1591060628000; + v.c2 = (int32_t)1; + v.c3 = (int64_t)1; + v.c4 = (float)1; + v.c5 = (double)1; + strcpy(v.c6, "abcdefgh"); + v.c7 = 1; + v.c8 = 1; + v.c9 = 1; + strcpy(v.c10, "一二三四五六七八"); + uintptr_t c10len=strlen(v.c10); + sprintf(stmt_sql, "select * from stmt_test.t1 where %s = ?", col); + printf("stmt_sql: %s\n", stmt_sql); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + TAOS_BIND *params = calloc(1, sizeof(TAOS_BIND)); + params->buffer_type = type; + params->is_null = NULL; + switch(type){ + case TSDB_DATA_TYPE_TIMESTAMP: + params->buffer_length = sizeof(v.c1); + params->buffer = &v.c1; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_INT: + params->buffer_length = sizeof(v.c2); + params->buffer = &v.c2; + params->length = ¶ms->buffer_length; + case TSDB_DATA_TYPE_BIGINT: + params->buffer_length = sizeof(v.c3); + params->buffer = &v.c3; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_FLOAT: + params->buffer_length = sizeof(v.c4); + params->buffer = &v.c4; + params->length = ¶ms->buffer_length; + case TSDB_DATA_TYPE_DOUBLE: + params->buffer_length = sizeof(v.c5); + params->buffer = &v.c5; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_BINARY: + params->buffer_length = sizeof(v.c6); + params->buffer = &v.c6; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_SMALLINT: + params->buffer_length = sizeof(v.c7); + params->buffer = &v.c7; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_TINYINT: + params->buffer_length = sizeof(v.c8); + params->buffer = &v.c8; + params->length = ¶ms->buffer_length; + case TSDB_DATA_TYPE_BOOL: + params->buffer_length = sizeof(v.c9); + params->buffer = &v.c9; + params->length = ¶ms->buffer_length; + break; + case TSDB_DATA_TYPE_NCHAR: + params->buffer_length = sizeof(v.c10); + params->buffer = &v.c10; + params->length = &c10len; + break; + default: + printf("Cannnot find type: %d\n", type); + break; + + } + assert(taos_stmt_bind_param(stmt, params) == 0); + assert(taos_stmt_execute(stmt) == 0); + TAOS_RES* result = taos_stmt_use_result(stmt); + assert(result != NULL); + print_result(result); + assert(taos_stmt_close(stmt) == 0); + free(params); + free(stmt_sql); + taos_free_result(result); +} + void taos_stmt_use_result_test() { printf("start taos_stmt_use_result test\n"); - TAOS_STMT *stmt = NULL; - assert(taos_stmt_use_result(stmt) == NULL); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); if(taos == NULL) { printf("Cannot connect to tdengine server\n"); exit(EXIT_FAILURE); } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); + execute_simple_sql(taos, "create table t1 using super tags (1, 1, 1, 1, 'abcdefgh',1,1,1,'一二三四五六七八')"); + execute_simple_sql(taos, "insert into t1 values (1591060628000, 1, 1, 1, 1, 'abcdefgh',1,1,1,'一二三四五六七八', now)"); + execute_simple_sql(taos, "insert into t1 values (1591060628001, 1, 1, 1, 1, 'abcdefgh',1,1,1,'一二三四五六七八', now)"); + + taos_stmt_use_result_query(taos, "c1", TSDB_DATA_TYPE_INT); + taos_stmt_use_result_query(taos, "c2", TSDB_DATA_TYPE_BIGINT); + taos_stmt_use_result_query(taos, "c3", TSDB_DATA_TYPE_FLOAT); + taos_stmt_use_result_query(taos, "c4", TSDB_DATA_TYPE_DOUBLE); + taos_stmt_use_result_query(taos, "c5", TSDB_DATA_TYPE_BINARY); + taos_stmt_use_result_query(taos, "c6", TSDB_DATA_TYPE_SMALLINT); + taos_stmt_use_result_query(taos, "c7", TSDB_DATA_TYPE_TINYINT); + taos_stmt_use_result_query(taos, "c8", TSDB_DATA_TYPE_BOOL); + taos_stmt_use_result_query(taos, "c9", TSDB_DATA_TYPE_NCHAR); + printf("finish taos_stmt_use_result test\n"); - stmt = taos_stmt_init(taos); - assert(stmt != NULL); - TAOS_RES* result = taos_stmt_use_result(stmt); - int rows = 0; - TAOS_ROW row; - while ((row = taos_fetch_row(result))) { - rows++; - } - printf("rows: %d\n", rows); - taos_free_result(result); - // assert(taos_stmt_use_result(stmt) == NULL); - assert(taos_stmt_close(stmt) == 0); } void taos_stmt_close_test() { @@ -373,11 +488,15 @@ void test_api_reliability() { taos_stmt_bind_param_batch_test(); taos_stmt_add_batch_test(); taos_stmt_execute_test(); - taos_stmt_use_result_test(); taos_stmt_close_test(); } +void test_query() { + taos_stmt_use_result_test(); +} + int main(int argc, char *argv[]) { test_api_reliability(); + test_query(); return 0; } \ No newline at end of file From af7517b35544d6016ad8d36702723f398a0a1ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=A1=E6=AF=94=E5=8D=A1=E6=AF=94?= <30525741+jackwener@users.noreply.github.com> Date: Fri, 20 Aug 2021 15:35:54 +0800 Subject: [PATCH 29/49] Update docs.md --- documentation20/cn/01.evaluation/docs.md | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 4aba82a54c..9a9669645c 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -30,38 +30,38 @@ TDengine 的模块之一是时序数据库。但除此之外，为减少研发 |数据源特点和需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|总体数据量巨大| | | √ |TDengine在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。| -|数据输入速度偶尔或者持续巨大| | | √ | TDengine的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。| -|数据源数目巨大| | | √ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。| +|总体数据量巨大| | | √ | TDengine 在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。| +|数据输入速度偶尔或者持续巨大| | | √ | TDengine 的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。| +|数据源数目巨大| | | √ | TDengine 设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。| ### 系统架构要求 |系统架构要求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求简单可靠的系统架构| | | √ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。| -|要求容错和高可靠| | | √ |TDengine的集群功能，自动提供容错灾备等高可靠功能。| -|标准化规范| | | √ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范。| +|要求简单可靠的系统架构| | | √ | TDengine 的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。| +|要求容错和高可靠| | | √ | TDengine 的集群功能，自动提供容错灾备等高可靠功能。| +|标准化规范| | | √ | TDengine 使用标准的SQL语言提供主要功能，遵守标准化规范。| ### 系统功能需求 |系统功能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求完整的内置数据处理算法| | √ | |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。| -|需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能。| +|要求完整的内置数据处理算法| | √ | | TDengine 的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。| +|需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑 TDengine 和关系型数据系统配合实现系统功能。| ### 系统性能需求 |系统性能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求较大的总体处理能力| | | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| -|要求高速处理数据 | | | √ |TDengine的专门为IOT优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。| -|要求快速处理小粒度数据| | | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| +|要求较大的总体处理能力| | | √ | TDengine 的集群功能可以轻松地让多服务器配合达成处理能力的提升。| +|要求高速处理数据 | | | √ | TDengine 的专门为 IOT 优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。| +|要求快速处理小粒度数据| | | √ |这方面 TDengine 性能可以完全对标关系型和 NoSQL 型数据处理系统。| ### 系统维护需求 |系统维护需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求系统可靠运行| | | √ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。| +|要求系统可靠运行| | | √ | TDengine 的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。| |要求运维学习成本可控| | | √ |同上。| -|要求市场有大量人才储备| √ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。| +|要求市场有大量人才储备| √ | | | TDengine 作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。| From adf1e47032e4ab685087baa16777bd04ed124dae Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 20 Aug 2021 16:08:30 +0800 Subject: [PATCH 30/49] In some emphasized text, spaces are also added between the English and Chinese text. --- documentation20/cn/01.evaluation/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 9a9669645c..4263391dbc 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -6,8 +6,8 @@ TDengine 是涛思数据面对高速增长的物联网大数据市场和技术 TDengine 的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine 还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与 Hadoop 等典型的大数据平台相比，它具有如下鲜明的特点： -* __10倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少 2 万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。 -* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的 1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的 1/10。 +* __10 倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少 2 万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。 +* __硬件或云服务成本降至 1/5__：由于超强性能，计算资源不到通用大数据方案的 1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的 1/10。 * __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成 Kafka/Redis/HBase/Spark/HDFS 等软件，大幅降低应用开发和维护的复杂度成本。 * __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过 Shell, Python, R, MATLAB 随时进行。 * __与第三方工具无缝连接__：不用一行代码，即可与 Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R 等集成。后续将支持 OPC, Hadoop, Spark 等，BI 工具也将无缝连接。 From c26d34a6847c483db6769c1f27ffea1ba09f19f2 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 20 Aug 2021 18:15:59 +0800 Subject: [PATCH 31/49] fix bug --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f1349d7d41..c6614d0054 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -902,7 +902,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); - if (size > 1 && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } From 1733d74619ec55465f8b6402a6917eef78b8486b Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 Aug 2021 18:45:15 +0800 Subject: [PATCH 32/49] [TD-6068] add test case for TD-6068 --- tests/pytest/functions/queryTestCases.py | 123 ++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py index f320db43af..8089bcccdc 100644 --- a/tests/pytest/functions/queryTestCases.py +++ b/tests/pytest/functions/queryTestCases.py @@ -884,6 +884,126 @@ class TDTestCase: pass + def td6068(self): + tdLog.printNoPrefix("==========TD-6068==========") + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("use db") + + tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool) tags(t1 int)") + + for i in range(100): + sql = f"create table db.t{i} using db.stb1 tags({i})" + tdSql.execute(sql) + tdSql.execute(f"insert into db.t{i} values (now-10h, {i}, {i+random.random()}, now-10h, 'a_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-9h, {i+random.randint(1,10)}, {i+random.random()}, now-9h, 'a_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} values (now-8h, {i+random.randint(1,10)}, {i+random.random()}, now-8h, 'b_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-7h, {i+random.randint(1,10)}, {i+random.random()}, now-7h, 'b_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} values (now-6h, {i+random.randint(1,10)}, {i+random.random()}, now-6h, 'c_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-5h, {i+random.randint(1,10)}, {i+random.random()}, now-5h, 'c_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} (ts)values (now-4h)") + tdSql.execute(f"insert into db.t{i} (ts)values (now-11h)") + tdSql.execute(f"insert into db.t{i} (ts)values (now-450m)") + + tdSql.query("select ts as t,derivative(c1, 10m, 0) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, derivative(c1, 1h, 0) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.checkCols(4) + tdSql.query("select ts as t, derivative(c1, 1s, 0) from t1") + tdSql.query("select ts as t, derivative(c1, 1d, 0) from t1") + tdSql.error("select ts as t, derivative(c1, 1h, 0) from stb1") + tdSql.query("select ts as t, derivative(c2, 1h, 0) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, derivative(c3, 1h, 0) from t1") + tdSql.error("select ts as t, derivative(c4, 1h, 0) from t1") + tdSql.query("select ts as t, derivative(c5, 1h, 0) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, derivative(c6, 1h, 0) from t1") + tdSql.error("select ts as t, derivative(t1, 1h, 0) from t1") + + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, diff(c1) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.checkCols(4) + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.error("select ts as t, diff(c1) from stb1") + tdSql.query("select ts as t, diff(c2) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, diff(c3) from t1") + tdSql.error("select ts as t, diff(c4) from t1") + tdSql.query("select ts as t, diff(c5) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, diff(c6) from t1") + tdSql.error("select ts as t, diff(t1) from t1") + tdSql.error("select ts as t, diff(c1, c2) from t1") + + tdSql.error("select ts as t, bottom(c1, 0) from t1") + tdSql.query("select ts as t, bottom(c1, 5) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, bottom(c1, 5) from stb1") + tdSql.checkRows(5) + tdSql.query("select ts as t, bottom(c1, 5) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.query("select ts as t, bottom(c1, 8) from t1") + tdSql.checkRows(6) + tdSql.query("select ts as t, bottom(c2, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, bottom(c3, 5) from t1") + tdSql.error("select ts as t, bottom(c4, 5) from t1") + tdSql.query("select ts as t, bottom(c5, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, bottom(c6, 5) from t1") + tdSql.error("select ts as t, bottom(c5, 8) as b from t1 order by b") + tdSql.error("select ts as t, bottom(t1, 1) from t1") + tdSql.error("select ts as t, bottom(t1, 1) from stb1") + tdSql.error("select ts as t, bottom(t1, 3) from stb1 order by c3") + tdSql.error("select ts as t, bottom(t1, 3) from t1 order by c3") + + + tdSql.error("select ts as t, top(c1, 0) from t1") + tdSql.query("select ts as t, top(c1, 5) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, top(c1, 5) from stb1") + tdSql.checkRows(5) + tdSql.query("select ts as t, top(c1, 5) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.query("select ts as t, top(c1, 8) from t1") + tdSql.checkRows(6) + tdSql.query("select ts as t, top(c2, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, top(c3, 5) from t1") + tdSql.error("select ts as t, top(c4, 5) from t1") + tdSql.query("select ts as t, top(c5, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, top(c6, 5) from t1") + tdSql.error("select ts as t, top(c5, 8) as b from t1 order by b") + tdSql.error("select ts as t, top(t1, 1) from t1") + tdSql.error("select ts as t, top(t1, 1) from stb1") + tdSql.error("select ts as t, top(t1, 3) from stb1 order by c3") + tdSql.error("select ts as t, top(t1, 3) from t1 order by c3") + + pass + + def run(self): # master branch @@ -891,8 +1011,9 @@ class TDTestCase: # self.td4082() # self.td4288() # self.td4724() - self.td5798() + # self.td5798() # self.td5935() + self.td6068() # develop branch # self.td4097() From 8d5c67ee5ded0080b2780d8f958375d160f1a2f5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Aug 2021 19:17:02 +0800 Subject: [PATCH 33/49] [TD-6164]: fix deadlock with fopen's malloc --- src/os/src/linux/linuxEnv.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index e3eadbc94b..a08a5aeed6 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -44,14 +44,20 @@ char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { sprintf(cmdline, "/proc/%d/cmdline", pid); - FILE* f = fopen(cmdline, "r"); - if (f) { - size_t size; - size = fread(cmdline, sizeof(char), 1024, f); - if (size > 0) { - if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; - } - fclose(f); + + int fd = open(cmdline, O_RDONLY); + if (fd >= 0) { + int n = read(fd, cmdline, sizeof(cmdline) - 1); + if (n < 0) n = 0; + + if (n > 0 && cmdline[n - 1] == '\n') --n; + + cmdline[n] = 0; + + close(fd); + } else { + cmdline[0] = 0; } + return cmdline; } From c3d19d52c9944947425f8cde0c9248787550f84c Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Fri, 20 Aug 2021 19:57:15 +0800 Subject: [PATCH 34/49] TD-6162 repaire error in demo.py (#7422) --- src/connector/python/examples/demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/python/examples/demo.py b/src/connector/python/examples/demo.py index 6c7c03f3e2..3bc09046f3 100644 --- a/src/connector/python/examples/demo.py +++ b/src/connector/python/examples/demo.py @@ -2,7 +2,7 @@ import taos conn = taos.connect(host='127.0.0.1', user='root', - passworkd='taodata', + password='taosdata', database='log') cursor = conn.cursor() From a8f1208dbd46eb6262bd19c52dc6fd0a6c5fa35b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 20 Aug 2021 22:48:23 +0800 Subject: [PATCH 35/49] [td-6229] enable the last_row cache while query condition exists. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d6af2e2ef1..3926564b50 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7283,7 +7283,7 @@ int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t i } pQuerySqlNode->pWhere = NULL; - if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI && (!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER))) { pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } From de12a8ef467994a4178434bd0007c2e42118c129 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 21 Aug 2021 16:21:26 +0800 Subject: [PATCH 36/49] [TD-6251]: taosdemo error msg with datetime info. (#7501) --- src/kit/taosdemo/taosdemo.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 75da9d9f4b..6099ac9803 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -659,7 +659,21 @@ static FILE * g_fpOfInsertResult = NULL; fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ - do { fprintf(stderr, " \033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, " \033[0m"); } while(0) + do {\ + struct tm Tm, *ptm;\ + struct timeval timeSecs; \ + time_t curTime;\ + gettimeofday(&timeSecs, NULL); \ + curTime = timeSecs.tv_sec;\ + ptm = localtime_r(&curTime, &Tm);\ + fprintf(stderr, " \033[31m");\ + fprintf(stderr, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ",\ + ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,\ + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec,\ + taosGetSelfPthreadId());\ + fprintf(stderr, "ERROR: "fmt, __VA_ARGS__);\ + fprintf(stderr, " \033[0m");\ + } while(0) // for strncpy buffer overflow #define min(a, b) (((a) < (b)) ? (a) : (b)) From ffcb8f6a4a785bd2b4c36846b51a0a928056be22 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 21 Aug 2021 21:58:17 +0800 Subject: [PATCH 37/49] [TD-6253]: taosdump cmdline param verification. (#7506) --- src/kit/taosdump/taosdump.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 6b553b3824..30b5d91b10 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -62,6 +62,20 @@ typedef struct { #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, "\033[0m"); } while(0) +static bool isStringNumber(char *input) +{ + int len = strlen(input); + if (0 == len) { + return false; + } + + for (int i = 0; i < len; i++) { + if (!isdigit(input[i])) + return false; + } + + return true; +} // -------------------------- SHOW DATABASE INTERFACE----------------------- enum _show_db_index { @@ -472,6 +486,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { g_args.table_batch = atoi(arg); break; case 'T': + if (!isStringNumber(arg)) { + errorPrint("%s", "\n\t-T need a number following!\n"); + exit(EXIT_FAILURE); + } g_args.thread_num = atoi(arg); break; case OPT_ABORT: From 9aa505f7df04ab543bad81748becf5a27be77995 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 22 Aug 2021 21:49:47 +0800 Subject: [PATCH 38/49] Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#7510) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. * fix resource leak reported by coverity scan. * fix taosdemo coverity scan issue. * fix tcsetattr and getchar return value determination bug. Co-authored-by: Shuduo Sang --- src/kit/shell/src/shellLinux.c | 4 +++- src/os/src/linux/osSystem.c | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index b177e22e86..859542cde9 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -177,7 +177,9 @@ static void parse_args( fprintf(stderr, "password reading error\n"); } taosSetConsoleEcho(true); - getchar(); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } } else { tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); strcpy(argv[i], "-p"); diff --git a/src/os/src/linux/osSystem.c b/src/os/src/linux/osSystem.c index 0cdb20dbdb..a82149dccb 100644 --- a/src/os/src/linux/osSystem.c +++ b/src/os/src/linux/osSystem.c @@ -63,12 +63,12 @@ int taosSetConsoleEcho(bool on) } if (on) - term.c_lflag|=ECHOFLAGS; + term.c_lflag |= ECHOFLAGS; else - term.c_lflag &=~ECHOFLAGS; + term.c_lflag &= ~ECHOFLAGS; - err = tcsetattr(STDIN_FILENO,TCSAFLUSH,&term); - if (err == -1 && err == EINTR) { + err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); + if (err == -1 || err == EINTR) { perror("Cannot set the attribution of the terminal"); return -1; } From 91ab5f22d4f77f77a7b74cb2fda328efed2309d4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 23 Aug 2021 15:04:26 +0800 Subject: [PATCH 39/49] [TD-6194]: taosdemo wrong data (#7517) --- src/kit/taosdemo/taosdemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6099ac9803..f222266ee8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5216,7 +5216,8 @@ static int64_t generateStbRowData( tmpLen = strlen(tmp); tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, INT_BUFF_LEN)); } else { - errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); + errorPrint( "Not support data type: %s\n", + stbInfo->columns[i].dataType); return -1; } @@ -5229,8 +5230,7 @@ static int64_t generateStbRowData( return 0; } - dataLen -= 1; - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); + tstrncpy(pstr + dataLen - 1, ")", 2); verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen); verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); From c4dcadc73a560d4382b938686ffa604362dd95e5 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 23 Aug 2021 15:29:18 +0800 Subject: [PATCH 40/49] [TD-6269][ci-skip]: disable queryWithOrderLimit.py before figuring out why it is hanging --- tests/pytest/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index e444a0a318..ccc7953580 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -255,7 +255,7 @@ python3 ./test.py -f query/queryTsisNull.py python3 ./test.py -f query/subqueryFilter.py python3 ./test.py -f query/nestedQuery/queryInterval.py python3 ./test.py -f query/queryStateWindow.py -python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py +# python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/queryCnameDisplay.py python3 ./test.py -f query/operator_cost.py From 0fbfa44005bd1235c4eefbe530a53ea250065d0b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 23 Aug 2021 17:52:44 +0800 Subject: [PATCH 41/49] [TD-6188] add new benchmark tool --- packaging/tools/make_install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 7851587c82..55ca1174c9 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -142,6 +142,7 @@ function install_bin() { if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/perfMonitor || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/set_core || : fi @@ -167,6 +168,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : fi From bcf74ecd4300178e4b5b924d8f2eb15e793dc938 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 23 Aug 2021 20:42:11 +0800 Subject: [PATCH 42/49] [TD-6194]: taosdemo wrong data (#7527) From 1ae4876550fc145d17a33543bbd6c3aa18ecc4d5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 24 Aug 2021 06:47:59 +0800 Subject: [PATCH 43/49] [TD-5852]: taosdemo data generation race. (#7533) --- src/kit/taosdemo/taosdemo.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f222266ee8..4fc378dc2b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1336,7 +1336,7 @@ static char *rand_bool_str(){ static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randbool_buff + (cursor * BOOL_BUFF_LEN); + return g_randbool_buff + ((cursor % MAX_PREPARED_RAND) * BOOL_BUFF_LEN); } static int32_t rand_bool(){ @@ -1351,7 +1351,8 @@ static char *rand_tinyint_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randtinyint_buff + (cursor * TINYINT_BUFF_LEN); + return g_randtinyint_buff + + ((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN); } static int32_t rand_tinyint() @@ -1367,7 +1368,8 @@ static char *rand_smallint_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randsmallint_buff + (cursor * SMALLINT_BUFF_LEN); + return g_randsmallint_buff + + ((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN); } static int32_t rand_smallint() @@ -1383,7 +1385,7 @@ static char *rand_int_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint_buff + (cursor * INT_BUFF_LEN); + return g_randint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); } static int32_t rand_int() @@ -1399,7 +1401,8 @@ static char *rand_bigint_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randbigint_buff + (cursor * BIGINT_BUFF_LEN); + return g_randbigint_buff + + ((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN); } static int64_t rand_bigint() @@ -1415,7 +1418,7 @@ static char *rand_float_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randfloat_buff + (cursor * FLOAT_BUFF_LEN); + return g_randfloat_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); } @@ -1432,7 +1435,8 @@ static char *demo_current_float_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_rand_current_buff + (cursor * FLOAT_BUFF_LEN); + return g_rand_current_buff + + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); } static float UNUSED_FUNC demo_current_float() @@ -1449,7 +1453,8 @@ static char *demo_voltage_int_str() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_rand_voltage_buff + (cursor * INT_BUFF_LEN); + return g_rand_voltage_buff + + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); } static int32_t UNUSED_FUNC demo_voltage_int() @@ -1464,7 +1469,7 @@ static char *demo_phase_float_str() { static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_rand_phase_buff + (cursor * FLOAT_BUFF_LEN); + return g_rand_phase_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); } static float UNUSED_FUNC demo_phase_float(){ @@ -5199,7 +5204,8 @@ static int64_t generateStbRowData( "SMALLINT", 8)) { tmp = rand_smallint_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, SMALLINT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, + min(tmpLen + 1, SMALLINT_BUFF_LEN)); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "TINYINT", 7)) { tmp = rand_tinyint_str(); @@ -5212,9 +5218,9 @@ static int64_t generateStbRowData( tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, BOOL_BUFF_LEN)); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "TIMESTAMP", 9)) { - tmp = rand_int_str(); + tmp = rand_bigint_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, INT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, BIGINT_BUFF_LEN)); } else { errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); From 567d0fe8da077e8653d27c8410673f4a75a1e59c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 08:33:44 +0800 Subject: [PATCH 44/49] [TD-6264]:backport _block_dist() from 2.1 to 2.0 to fix core dump due to tbuf length limitation --- src/inc/tsdb.h | 7 +- src/query/src/qAggMain.c | 148 +++++++++++++------- src/query/src/qExecutor.c | 10 +- src/query/src/qUtil.c | 16 ++- src/tsdb/inc/tsdbCommit.h | 2 + src/tsdb/src/tsdbRead.c | 12 +- src/util/inc/tarray.h | 8 ++ src/util/src/tarray.c | 5 + tests/script/general/compute/block_dist.sim | 6 +- 9 files changed, 158 insertions(+), 56 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85ee9f0443..da33a2ba73 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -211,7 +211,7 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct SFileBlockInfo { - int32_t numOfRows; + int32_t numBlocksOfStep; } SFileBlockInfo; typedef struct { @@ -225,13 +225,18 @@ typedef struct { SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; +#define TSDB_BLOCK_DIST_STEP_ROWS 16 typedef struct { uint16_t rowSize; uint16_t numOfFiles; uint32_t numOfTables; uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; int32_t firstSeekTimeUs; uint32_t numOfRowsInMemTable; + uint32_t numOfSmallBlocks; SArray *dataBlockInfos; } STableBlockDist; diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index b3996fb55a..b0f7decfc6 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4794,7 +4794,7 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (int16_t) pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4802,51 +4802,85 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) { +static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); + pDist->numOfTables += pSrc->numOfTables; pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; + pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; pDist->numOfFiles += pSrc->numOfFiles; pDist->totalSize += pSrc->totalSize; + pDist->totalRows += pSrc->totalRows; - if (pDist->dataBlockInfos == NULL) { - pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo)); + if (pResInfo->hasResult == DATA_SET_FLAG) { + pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + } else { + pDist->maxRows = pSrc->maxRows; + pDist->minRows = pSrc->minRows; + + int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++maxSteps; + } + pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(pDist->dataBlockInfos, maxSteps); } - taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos)); + size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); + for (int32_t i = 0; i < steps; ++i) { + int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; + SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); + blockInfo->numBlocksOfStep += srcNumBlocks; + } } void block_func_merge(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); STableBlockDist info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - mergeTableBlockDist(pDist, &info); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + mergeTableBlockDist(pResInfo, &info); + + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; } -static int32_t doGetPercentile(const SArray* pArray, double rate) { - int32_t len = (int32_t)taosArrayGetSize(pArray); - if (len <= 0) { - return 0; +void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, + double* percents, int32_t* percentiles) { + if (totalBlocks == 0) { + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = 0; + } + return; } - assert(rate >= 0 && rate <= 1.0); - int idx = (int32_t)((len - 1) * rate); + SArray *blocksInfos = pTableBlockDist->dataBlockInfos; + size_t numSteps = taosArrayGetSize(blocksInfos); + size_t cumulativeBlocks = 0; - return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows; -} + int percentIndex = 0; + for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { + int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; + if (numStepBlocks == 0) continue; + cumulativeBlocks += numStepBlocks; -static int compareBlockInfo(const void *pLeft, const void *pRight) { - int32_t left = ((SFileBlockInfo *)pLeft)->numOfRows; - int32_t right = ((SFileBlockInfo *)pRight)->numOfRows; + while (percentIndex < numOfPercents) { + double blockRank = totalBlocks * percents[percentIndex]; + if (blockRank <= cumulativeBlocks) { + percentiles[percentIndex] = indexStep; + ++percentIndex; + } else { + break; + } + } + } - if (left > right) return 1; - if (left < right) return -1; - return 0; + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; + } } void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { @@ -4854,41 +4888,56 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { return; } - int64_t min = INT64_MAX, max = INT64_MIN, avg = 0; - SArray* blockInfos= pTableBlockDist->dataBlockInfos; - int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + SArray* blockInfos = pTableBlockDist->dataBlockInfos; + uint64_t totalRows = pTableBlockDist->totalRows; + size_t numSteps = taosArrayGetSize(blockInfos); + int64_t totalBlocks = 0; + int64_t min = -1, max = -1, avg = 0; - for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + for (int32_t i = 0; i < numSteps; i++) { SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t rows = blockInfo->numOfRows; - - min = MIN(min, rows); - max = MAX(max, rows); - totalRows += rows; + int64_t blocks = blockInfo->numBlocksOfStep; + totalBlocks += blocks; } avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - taosArraySort(blockInfos, compareBlockInfo); + min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; + max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; + + double stdDev = 0; + if (totalBlocks > 0) { + double variance = 0; + for (int32_t i = 0; i < numSteps; i++) { + SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); + int64_t blocks = blockInfo->numBlocksOfStep; + int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); + variance += blocks * (rows - avg) * (rows - avg); + } + variance = variance / totalBlocks; + stdDev = sqrt(variance); + } + + double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; + int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); + getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); uint64_t totalLen = pTableBlockDist->totalSize; int32_t rowSize = pTableBlockDist->rowSize; - + int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; + double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; int sz = sprintf(result + VARSTR_HEADER_SIZE, "summary: \n\t " "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t " - "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]", - doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10), - doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30), - doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50), - doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70), - doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90), - doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99), - min, max, avg, 0.0, - totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows), - pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs); + "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " + "RowsInMem=[%d] \n\t", + percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], + percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], + min, max, avg, stdDev, + totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, + pTableBlockDist->numOfRowsInMemTable); varDataSetLen(result, sz); UNUSED(sz); } @@ -4897,9 +4946,14 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (int16_t)pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; generateBlockDistResult(pDist, pCtx->pOutput); + if (pDist->dataBlockInfos != NULL) { + taosArrayDestroy(pDist->dataBlockInfos); + pDist->dataBlockInfos = NULL; + } + // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f96d33f060..b2e63707cb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4546,7 +4546,15 @@ static SSDataBlock* doBlockInfoScan(void* param) { STableBlockDist tableBlockDist = {0}; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - tableBlockDist.dataBlockInfos = taosArrayInit(512, sizeof(SFileBlockInfo)); + + int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; + if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++numRowSteps; + } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index aa793add84..ed069c6a91 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -581,7 +581,11 @@ void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { tbufWriteUint32(bw, pDist->numOfTables); tbufWriteUint16(bw, pDist->numOfFiles); tbufWriteUint64(bw, pDist->totalSize); + tbufWriteUint64(bw, pDist->totalRows); + tbufWriteInt32(bw, pDist->maxRows); + tbufWriteInt32(bw, pDist->minRows); tbufWriteUint32(bw, pDist->numOfRowsInMemTable); + tbufWriteUint32(bw, pDist->numOfSmallBlocks); tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); // compress the binary string @@ -616,13 +620,17 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi pDist->numOfTables = tbufReadUint32(&br); pDist->numOfFiles = tbufReadUint16(&br); pDist->totalSize = tbufReadUint64(&br); + pDist->totalRows = tbufReadUint64(&br); + pDist->maxRows = tbufReadInt32(&br); + pDist->minRows = tbufReadInt32(&br); pDist->numOfRowsInMemTable = tbufReadUint32(&br); - int64_t numOfBlocks = tbufReadUint64(&br); + pDist->numOfSmallBlocks = tbufReadUint32(&br); + int64_t numSteps = tbufReadUint64(&br); bool comp = tbufReadUint8(&br); uint32_t compLen = tbufReadUint32(&br); - size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo)); + size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); char* outputBuf = NULL; if (comp) { @@ -633,12 +641,12 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo)); + assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); } else { outputBuf = (char*) tbufReadBinary(&br, &originalLen); } - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo)); + pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { tfree(outputBuf); } diff --git a/src/tsdb/inc/tsdbCommit.h b/src/tsdb/inc/tsdbCommit.h index 5e740081d1..f1c3a91746 100644 --- a/src/tsdb/inc/tsdbCommit.h +++ b/src/tsdb/inc/tsdbCommit.h @@ -29,6 +29,8 @@ typedef struct { int64_t size; } SKVRecord; +#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) + void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn); int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index be686fcffd..1babb01530 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2104,6 +2104,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); // find the start data block in file @@ -2121,6 +2122,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { @@ -2136,7 +2138,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); @@ -2177,7 +2179,13 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist pTableBlockInfo->totalSize += pBlock[j].len; int32_t numOfRows = pBlock[j].numOfRows; - taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows); + pTableBlockInfo->totalRows += numOfRows; + if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; + if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; + if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1; + int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; + SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); + blockInfo->numBlocksOfStep++; } } } diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index f2e268c2d4..677be0051e 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -98,6 +98,14 @@ void* taosArrayGetLast(const SArray* pArray); */ size_t taosArrayGetSize(const SArray* pArray); +/** + * set the size of array + * @param pArray + * @param size size of the array + * @return + */ +void taosArraySetSize(SArray* pArray, size_t size); + /** * insert data into array * @param pArray diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 4dde5dbba2..6fb4029a84 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -110,6 +110,11 @@ void* taosArrayGetLast(const SArray* pArray) { size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; } +void taosArraySetSize(SArray* pArray, size_t size) { + assert(size <= pArray->capacity); + pArray->size = size; +} + void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { if (pArray == NULL || pData == NULL) { return NULL; diff --git a/tests/script/general/compute/block_dist.sim b/tests/script/general/compute/block_dist.sim index 51cf903654..5343c1db28 100644 --- a/tests/script/general/compute/block_dist.sim +++ b/tests/script/general/compute/block_dist.sim @@ -84,6 +84,10 @@ if $rows != 1 then return -1 endi +print ============== TD-5998 +sql_error select _block_dist() from (select * from $nt) +sql_error select _block_dist() from (select * from $mt) + print =============== clear sql drop database $db sql show databases @@ -91,4 +95,4 @@ if $rows != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 7d0ae7942ca84b48887b9f4b9404de0837b35682 Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 24 Aug 2021 11:48:55 +0800 Subject: [PATCH 45/49] fix bug --- src/client/src/tscUtil.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1af8eaac2e..1a953cfd3d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3778,6 +3778,8 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self); if (code && !((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry)) { + pParentSql->res.code = code; + tscAsyncResultOnError(pParentSql); return; } @@ -3858,6 +3860,7 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pNew->signature = pNew; pNew->sqlstr = strdup(pSql->sqlstr); pNew->fp = tscSubqueryCompleteCallback; + pNew->fetchFp = tscSubqueryCompleteCallback; pNew->maxRetry = pSql->maxRetry; pNew->cmd.resColumnId = TSDB_RES_COL_ID; From 3a5e4b04aa1496a5c6090c3459be9eaba9158027 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 24 Aug 2021 17:57:05 +0800 Subject: [PATCH 46/49] [TD-6313]: improve error handling if loading taos failed in python (#7542) --- src/connector/python/taos/cinterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 660707bfcd..d9db1272ef 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -268,7 +268,7 @@ def _load_taos(): try: return load_func[platform.system()]() except: - sys.exit('unsupported platform to TDengine connector') + raise InterfaceError('unsupported platform or failed to load taos client library') class CTaosInterface(object): From 6f8a5c4d1ec3e63ac4f66cadf8ddd0aaab309b2d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 24 Aug 2021 18:00:31 +0800 Subject: [PATCH 47/49] [TD-6303]: taosdemo -f miss filename. (#7543) --- src/kit/taosdemo/taosdemo.c | 499 +++++++++++++++++------------------- 1 file changed, 239 insertions(+), 260 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4fc378dc2b..38dc47ad01 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -659,6 +659,13 @@ static FILE * g_fpOfInsertResult = NULL; fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ + do {\ + fprintf(stderr, " \033[31m");\ + fprintf(stderr, "ERROR: "fmt, __VA_ARGS__);\ + fprintf(stderr, " \033[0m");\ + } while(0) + +#define errorPrint2(fmt, ...) \ do {\ struct tm Tm, *ptm;\ struct timeval timeSecs; \ @@ -671,8 +678,8 @@ static FILE * g_fpOfInsertResult = NULL; ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,\ ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec,\ taosGetSelfPthreadId());\ - fprintf(stderr, "ERROR: "fmt, __VA_ARGS__);\ fprintf(stderr, " \033[0m");\ + errorPrint(fmt, __VA_ARGS__);\ } while(0) // for strncpy buffer overflow @@ -815,6 +822,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { arguments->demo_mode = false; + + if (NULL == argv[i+1]) { + printHelp(); + errorPrint("%s", "\n\t-f need a valid json file following!\n"); + exit(EXIT_FAILURE); + } arguments->metaFile = argv[++i]; } else if (strcmp(argv[i], "-c") == 0) { if (argc == i+1) { @@ -1227,7 +1240,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); if (code != 0) { if (!quiet) { - errorPrint("Failed to execute %s, reason: %s\n", + errorPrint2("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); } taos_free_result(res); @@ -1249,7 +1262,7 @@ static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo) { pThreadInfo->fp = fopen(pThreadInfo->filePath, "at"); if (pThreadInfo->fp == NULL) { - errorPrint( + errorPrint2( "%s() LN%d, failed to open result file: %s, result will not save to file\n", __func__, __LINE__, pThreadInfo->filePath); return; @@ -1268,7 +1281,7 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { char* databuf = (char*) calloc(1, 100*1024*1024); if (databuf == NULL) { - errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", + errorPrint2("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__); return ; } @@ -1308,7 +1321,7 @@ static void selectAndGetResult( if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", + errorPrint2("%s() LN%d, failed to execute sql:%s, reason:%s\n", __func__, __LINE__, command, taos_errstr(res)); taos_free_result(res); return; @@ -1327,7 +1340,7 @@ static void selectAndGetResult( } } else { - errorPrint("%s() LN%d, unknown query mode: %s\n", + errorPrint2("%s() LN%d, unknown query mode: %s\n", __func__, __LINE__, g_queryInfo.queryMode); } } @@ -2177,7 +2190,7 @@ static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { FILE* fp = fopen(fname, "at"); if (fp == NULL) { - errorPrint("%s() LN%d, failed to open file: %s\n", + errorPrint2("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname); return -1; } @@ -2224,7 +2237,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { int32_t code = taos_errno(res); if (code != 0) { - errorPrint( "failed to run , reason: %s\n", + errorPrint2("failed to run , reason: %s\n", taos_errstr(res)); return -1; } @@ -2240,7 +2253,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); if (dbInfos[count] == NULL) { - errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count); + errorPrint2("failed to allocate memory for some dbInfo[%d]\n", count); return -1; } @@ -2393,7 +2406,7 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port request_buf = malloc(req_buf_len); if (NULL == request_buf) { - errorPrint("%s", "ERROR, cannot allocate memory.\n"); + errorPrint("%s", "cannot allocate memory.\n"); exit(EXIT_FAILURE); } @@ -2532,7 +2545,7 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { - errorPrint("%s() LN%d, calloc failed! size:%d\n", + errorPrint2("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1); return NULL; } @@ -2632,7 +2645,7 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int64_t tableSeq) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64",", rand_bigint()); } else { - errorPrint("No support data type: %s\n", stbInfo->tags[i].dataType); + errorPrint2("No support data type: %s\n", stbInfo->tags[i].dataType); tmfree(dataBuf); return NULL; } @@ -2671,7 +2684,7 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { lenOfOneRow += TIMESTAMP_BUFF_LEN; } else { - errorPrint("get error data type : %s\n", dataType); + errorPrint2("get error data type : %s\n", dataType); exit(EXIT_FAILURE); } } @@ -2702,7 +2715,7 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "DOUBLE") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; } else { - errorPrint("get error tag type : %s\n", dataType); + errorPrint2("get error tag type : %s\n", dataType); exit(EXIT_FAILURE); } } @@ -2739,7 +2752,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, if (code != 0) { taos_free_result(res); taos_close(taos); - errorPrint("%s() LN%d, failed to run command %s\n", + errorPrint2("%s() LN%d, failed to run command %s\n", __func__, __LINE__, command); exit(EXIT_FAILURE); } @@ -2751,7 +2764,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, if (NULL == childTblName) { taos_free_result(res); taos_close(taos); - errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__); + errorPrint2("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__); exit(EXIT_FAILURE); } } @@ -2761,7 +2774,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, int32_t* len = taos_fetch_lengths(res); if (0 == strlen((char *)row[0])) { - errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", + errorPrint2("%s() LN%d, No.%"PRId64" table return empty name\n", __func__, __LINE__, count); exit(EXIT_FAILURE); } @@ -2782,7 +2795,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, tmfree(childTblName); taos_free_result(res); taos_close(taos); - errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n", + errorPrint2("%s() LN%d, realloc fail for save child table name of %s.%s\n", __func__, __LINE__, dbName, sTblName); exit(EXIT_FAILURE); } @@ -2879,7 +2892,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, int childTblCount = 10000; superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (superTbls->childTblName == NULL) { - errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + errorPrint2("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); return -1; } getAllChildNameOfSuperTable(taos, dbName, @@ -2905,7 +2918,7 @@ static int createSuperTable( int lenOfOneRow = 0; if (superTbl->columnCount == 0) { - errorPrint("%s() LN%d, super table column count is %d\n", + errorPrint2("%s() LN%d, super table column count is %d\n", __func__, __LINE__, superTbl->columnCount); free(command); return -1; @@ -2969,7 +2982,7 @@ static int createSuperTable( } else { taos_close(taos); free(command); - errorPrint("%s() LN%d, config error data type : %s\n", + errorPrint2("%s() LN%d, config error data type : %s\n", __func__, __LINE__, dataType); exit(EXIT_FAILURE); } @@ -2982,7 +2995,7 @@ static int createSuperTable( if (NULL == superTbl->colsOfCreateChildTable) { taos_close(taos); free(command); - errorPrint("%s() LN%d, Failed when calloc, size:%d", + errorPrint2("%s() LN%d, Failed when calloc, size:%d", __func__, __LINE__, len+1); exit(EXIT_FAILURE); } @@ -2992,7 +3005,7 @@ static int createSuperTable( __func__, __LINE__, superTbl->colsOfCreateChildTable); if (superTbl->tagCount == 0) { - errorPrint("%s() LN%d, super table tag count is %d\n", + errorPrint2("%s() LN%d, super table tag count is %d\n", __func__, __LINE__, superTbl->tagCount); free(command); return -1; @@ -3059,7 +3072,7 @@ static int createSuperTable( } else { taos_close(taos); free(command); - errorPrint("%s() LN%d, config error tag type : %s\n", + errorPrint2("%s() LN%d, config error tag type : %s\n", __func__, __LINE__, dataType); exit(EXIT_FAILURE); } @@ -3074,7 +3087,7 @@ static int createSuperTable( "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbl->sTblName, cols, tags); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - errorPrint( "create supertable %s failed!\n\n", + errorPrint2("create supertable %s failed!\n\n", superTbl->sTblName); free(command); return -1; @@ -3090,7 +3103,7 @@ int createDatabasesAndStables(char *command) { int ret = 0; taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + errorPrint2("Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); return -1; } @@ -3186,7 +3199,7 @@ int createDatabasesAndStables(char *command) { if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { taos_close(taos); - errorPrint( "\ncreate database %s failed!\n\n", + errorPrint("\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); return -1; } @@ -3216,7 +3229,7 @@ int createDatabasesAndStables(char *command) { ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); if (0 != ret) { - errorPrint("\nget super table %s.%s info failed!\n\n", + errorPrint2("\nget super table %s.%s info failed!\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); continue; } @@ -3244,7 +3257,7 @@ static void* createTable(void *sarg) pThreadInfo->buffer = calloc(buff_len, 1); if (pThreadInfo->buffer == NULL) { - errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); + errorPrint2("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); exit(EXIT_FAILURE); } @@ -3266,7 +3279,7 @@ static void* createTable(void *sarg) } else { if (stbInfo == NULL) { free(pThreadInfo->buffer); - errorPrint("%s() LN%d, use metric, but super table info is NULL\n", + errorPrint2("%s() LN%d, use metric, but super table info is NULL\n", __func__, __LINE__); exit(EXIT_FAILURE); } else { @@ -3313,7 +3326,7 @@ static void* createTable(void *sarg) len = 0; if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, NO_INSERT_TYPE, false)){ - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + errorPrint2("queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); free(pThreadInfo->buffer); return NULL; } @@ -3329,7 +3342,7 @@ static void* createTable(void *sarg) if (0 != len) { if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, NO_INSERT_TYPE, false)) { - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + errorPrint2("queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); } } @@ -3374,7 +3387,7 @@ static int startMultiThreadCreateChildTable( db_name, g_Dbs.port); if (pThreadInfo->taos == NULL) { - errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", + errorPrint2("%s() LN%d, Failed to connect to TDengine, reason:%s\n", __func__, __LINE__, taos_errstr(NULL)); free(pids); free(infos); @@ -3549,7 +3562,7 @@ static int readSampleFromCsvFileToMem( FILE* fp = fopen(stbInfo->sampleFile, "r"); if (fp == NULL) { - errorPrint( "Failed to open sample file: %s, reason:%s\n", + errorPrint("Failed to open sample file: %s, reason:%s\n", stbInfo->sampleFile, strerror(errno)); return -1; } @@ -3561,7 +3574,7 @@ static int readSampleFromCsvFileToMem( readLen = tgetline(&line, &n, fp); if (-1 == readLen) { if(0 != fseek(fp, 0, SEEK_SET)) { - errorPrint( "Failed to fseek file: %s, reason:%s\n", + errorPrint("Failed to fseek file: %s, reason:%s\n", stbInfo->sampleFile, strerror(errno)); fclose(fp); return -1; @@ -3604,7 +3617,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( // columns cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); if (columns && columns->type != cJSON_Array) { - printf("ERROR: failed to read json, columns not found\n"); + errorPrint("%s", "failed to read json, columns not found\n"); goto PARSE_OVER; } else if (NULL == columns) { superTbls->columnCount = 0; @@ -3614,8 +3627,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( int columnSize = cJSON_GetArraySize(columns); if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) { - errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", - __func__, __LINE__, TSDB_MAX_COLUMNS); + errorPrint("failed to read json, column size overflow, max column size is %d\n", + TSDB_MAX_COLUMNS); goto PARSE_OVER; } @@ -3633,8 +3646,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (countObj && countObj->type == cJSON_Number) { count = countObj->valueint; } else if (countObj && countObj->type != cJSON_Number) { - errorPrint("%s() LN%d, failed to read json, column count not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, column count not found\n"); goto PARSE_OVER; } else { count = 1; @@ -3645,8 +3657,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( cJSON *dataType = cJSON_GetObjectItem(column, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - errorPrint("%s() LN%d: failed to read json, column type not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, column type not found\n"); goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, DATATYPE_BUFF_LEN); @@ -3674,8 +3685,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { - errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", - __func__, __LINE__, MAX_NUM_COLUMNS); + errorPrint("failed to read json, column size overflow, allowed max column size is %d\n", + MAX_NUM_COLUMNS); goto PARSE_OVER; } @@ -3686,15 +3697,14 @@ static bool getColumnAndTagTypeFromInsertJsonFile( // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, tags not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, tags not found\n"); goto PARSE_OVER; } int tagSize = cJSON_GetArraySize(tags); if (tagSize > TSDB_MAX_TAGS) { - errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", - __func__, __LINE__, TSDB_MAX_TAGS); + errorPrint("failed to read json, tags size overflow, max tag size is %d\n", + TSDB_MAX_TAGS); goto PARSE_OVER; } @@ -3708,7 +3718,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (countObj && countObj->type == cJSON_Number) { count = countObj->valueint; } else if (countObj && countObj->type != cJSON_Number) { - printf("ERROR: failed to read json, column count not found\n"); + errorPrint("%s", "failed to read json, column count not found\n"); goto PARSE_OVER; } else { count = 1; @@ -3719,8 +3729,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( cJSON *dataType = cJSON_GetObjectItem(tag, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, tag type not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, tag type not found\n"); goto PARSE_OVER; } tstrncpy(columnCase.dataType, dataType->valuestring, @@ -3730,8 +3739,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (dataLen && dataLen->type == cJSON_Number) { columnCase.dataLen = dataLen->valueint; } else if (dataLen && dataLen->type != cJSON_Number) { - errorPrint("%s() LN%d, failed to read json, column len not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, column len not found\n"); goto PARSE_OVER; } else { columnCase.dataLen = 0; @@ -3746,16 +3754,16 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } if (index > TSDB_MAX_TAGS) { - errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", - __func__, __LINE__, TSDB_MAX_TAGS); + errorPrint("failed to read json, tags size overflow, allowed max tag count is %d\n", + TSDB_MAX_TAGS); goto PARSE_OVER; } superTbls->tagCount = index; if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) { - errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", - __func__, __LINE__, TSDB_MAX_COLUMNS); + errorPrint("columns + tags is more than allowed max columns count: %d\n", + TSDB_MAX_COLUMNS); goto PARSE_OVER; } ret = true; @@ -3778,7 +3786,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!host) { tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { - printf("ERROR: failed to read json, host not found\n"); + errorPrint("%s", "failed to read json, host not found\n"); goto PARSE_OVER; } @@ -3816,7 +3824,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!threads) { g_Dbs.threadCount = 1; } else { - printf("ERROR: failed to read json, threads not found\n"); + errorPrint("%s", "failed to read json, threads not found\n"); goto PARSE_OVER; } @@ -3826,32 +3834,28 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!threads2) { g_Dbs.threadCountByCreateTbl = 1; } else { - errorPrint("%s() LN%d, failed to read json, threads2 not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, threads2 not found\n"); goto PARSE_OVER; } cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); if (gInsertInterval && gInsertInterval->type == cJSON_Number) { if (gInsertInterval->valueint <0) { - errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert interval input mistake\n"); goto PARSE_OVER; } g_args.insert_interval = gInsertInterval->valueint; } else if (!gInsertInterval) { g_args.insert_interval = 0; } else { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert_interval input mistake\n"); goto PARSE_OVER; } cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { if (interlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, interlace_rows input mistake\n"); goto PARSE_OVER; } @@ -3859,8 +3863,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, interlace_rows input mistake\n"); goto PARSE_OVER; } @@ -3933,14 +3936,14 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { - printf("ERROR: failed to read json, databases not found\n"); + errorPrint("%s", "failed to read json, databases not found\n"); goto PARSE_OVER; } int dbSize = cJSON_GetArraySize(dbs); if (dbSize > MAX_DB_COUNT) { errorPrint( - "ERROR: failed to read json, databases size overflow, max database is %d\n", + "failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); goto PARSE_OVER; } @@ -3953,13 +3956,13 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); if (!dbinfo || dbinfo->type != cJSON_Object) { - printf("ERROR: failed to read json, dbinfo not found\n"); + errorPrint("%s", "failed to read json, dbinfo not found\n"); goto PARSE_OVER; } cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { - printf("ERROR: failed to read json, db name not found\n"); + errorPrint("%s", "failed to read json, db name not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); @@ -3974,8 +3977,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!drop) { g_Dbs.db[i].drop = g_args.drop_database; } else { - errorPrint("%s() LN%d, failed to read json, drop input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, drop input mistake\n"); goto PARSE_OVER; } @@ -3987,7 +3989,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!precision) { memset(g_Dbs.db[i].dbCfg.precision, 0, SMALL_BUFF_LEN); } else { - printf("ERROR: failed to read json, precision not found\n"); + errorPrint("%s", "failed to read json, precision not found\n"); goto PARSE_OVER; } @@ -3997,7 +3999,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!update) { g_Dbs.db[i].dbCfg.update = -1; } else { - printf("ERROR: failed to read json, update not found\n"); + errorPrint("%s", "failed to read json, update not found\n"); goto PARSE_OVER; } @@ -4007,7 +4009,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!replica) { g_Dbs.db[i].dbCfg.replica = -1; } else { - printf("ERROR: failed to read json, replica not found\n"); + errorPrint("%s", "failed to read json, replica not found\n"); goto PARSE_OVER; } @@ -4017,7 +4019,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!keep) { g_Dbs.db[i].dbCfg.keep = -1; } else { - printf("ERROR: failed to read json, keep not found\n"); + errorPrint("%s", "failed to read json, keep not found\n"); goto PARSE_OVER; } @@ -4027,7 +4029,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!days) { g_Dbs.db[i].dbCfg.days = -1; } else { - printf("ERROR: failed to read json, days not found\n"); + errorPrint("%s", "failed to read json, days not found\n"); goto PARSE_OVER; } @@ -4037,7 +4039,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!cache) { g_Dbs.db[i].dbCfg.cache = -1; } else { - printf("ERROR: failed to read json, cache not found\n"); + errorPrint("%s", "failed to read json, cache not found\n"); goto PARSE_OVER; } @@ -4047,7 +4049,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!blocks) { g_Dbs.db[i].dbCfg.blocks = -1; } else { - printf("ERROR: failed to read json, block not found\n"); + errorPrint("%s", "failed to read json, block not found\n"); goto PARSE_OVER; } @@ -4067,7 +4069,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!minRows) { g_Dbs.db[i].dbCfg.minRows = 0; // 0 means default } else { - printf("ERROR: failed to read json, minRows not found\n"); + errorPrint("%s", "failed to read json, minRows not found\n"); goto PARSE_OVER; } @@ -4077,7 +4079,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxRows) { g_Dbs.db[i].dbCfg.maxRows = 0; // 0 means default } else { - printf("ERROR: failed to read json, maxRows not found\n"); + errorPrint("%s", "failed to read json, maxRows not found\n"); goto PARSE_OVER; } @@ -4087,7 +4089,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!comp) { g_Dbs.db[i].dbCfg.comp = -1; } else { - printf("ERROR: failed to read json, comp not found\n"); + errorPrint("%s", "failed to read json, comp not found\n"); goto PARSE_OVER; } @@ -4097,7 +4099,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!walLevel) { g_Dbs.db[i].dbCfg.walLevel = -1; } else { - printf("ERROR: failed to read json, walLevel not found\n"); + errorPrint("%s", "failed to read json, walLevel not found\n"); goto PARSE_OVER; } @@ -4107,7 +4109,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!cacheLast) { g_Dbs.db[i].dbCfg.cacheLast = -1; } else { - printf("ERROR: failed to read json, cacheLast not found\n"); + errorPrint("%s", "failed to read json, cacheLast not found\n"); goto PARSE_OVER; } @@ -4127,24 +4129,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!fsync) { g_Dbs.db[i].dbCfg.fsync = -1; } else { - errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, fsync input mistake\n"); goto PARSE_OVER; } // super_talbes cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); if (!stables || stables->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, super_tables not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, super_tables not found\n"); goto PARSE_OVER; } int stbSize = cJSON_GetArraySize(stables); if (stbSize > MAX_SUPER_TABLE_COUNT) { errorPrint( - "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", - __func__, __LINE__, MAX_SUPER_TABLE_COUNT); + "failed to read json, supertable size overflow, max supertable is %d\n", + MAX_SUPER_TABLE_COUNT); goto PARSE_OVER; } @@ -4157,8 +4157,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, stb name not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, stb name not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, @@ -4166,7 +4165,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { - printf("ERROR: failed to read json, childtable_prefix not found\n"); + errorPrint("%s", "failed to read json, childtable_prefix not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, @@ -4187,7 +4186,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!autoCreateTbl) { g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; } else { - printf("ERROR: failed to read json, auto_create_table not found\n"); + errorPrint("%s", "failed to read json, auto_create_table not found\n"); goto PARSE_OVER; } @@ -4197,7 +4196,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!batchCreateTbl) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; } else { - printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); + errorPrint("%s", "failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } @@ -4217,8 +4216,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!childTblExists) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; } else { - errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", - __func__, __LINE__); + errorPrint("%s", + "failed to read json, child_table_exists not found\n"); goto PARSE_OVER; } @@ -4228,8 +4227,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", - __func__, __LINE__); + errorPrint("%s", + "failed to read json, childtable_count input mistake\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; @@ -4244,8 +4243,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", min(SMALL_BUFF_LEN, strlen("rand") + 1)); } else { - errorPrint("%s() LN%d, failed to read json, data_source not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, data_source not found\n"); goto PARSE_OVER; } @@ -4259,8 +4257,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; } else { - errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", - __func__, __LINE__, stbIface->valuestring); + errorPrint("failed to read json, insert_mode %s not recognized\n", + stbIface->valuestring); goto PARSE_OVER; } } else if (!stbIface) { @@ -4274,7 +4272,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if ((childTbl_limit) && (g_Dbs.db[i].drop != true) && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if (childTbl_limit->type != cJSON_Number) { - printf("ERROR: failed to read json, childtable_limit\n"); + errorPrint("%s", "failed to read json, childtable_limit\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; @@ -4287,7 +4285,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if ((childTbl_offset->type != cJSON_Number) || (0 > childTbl_offset->valueint)) { - printf("ERROR: failed to read json, childtable_offset\n"); + errorPrint("%s", "failed to read json, childtable_offset\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; @@ -4303,7 +4301,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", TSDB_DB_NAME_LEN); } else { - printf("ERROR: failed to read json, start_timestamp not found\n"); + errorPrint("%s", "failed to read json, start_timestamp not found\n"); goto PARSE_OVER; } @@ -4313,7 +4311,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!timestampStep) { g_Dbs.db[i].superTbls[j].timeStampStep = g_args.timestamp_step; } else { - printf("ERROR: failed to read json, timestamp_step not found\n"); + errorPrint("%s", "failed to read json, timestamp_step not found\n"); goto PARSE_OVER; } @@ -4328,7 +4326,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", SMALL_BUFF_LEN); } else { - printf("ERROR: failed to read json, sample_format not found\n"); + errorPrint("%s", "failed to read json, sample_format not found\n"); goto PARSE_OVER; } @@ -4343,7 +4341,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); } else { - printf("ERROR: failed to read json, sample_file not found\n"); + errorPrint("%s", "failed to read json, sample_file not found\n"); goto PARSE_OVER; } @@ -4361,7 +4359,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); g_Dbs.db[i].superTbls[j].tagSource = 0; } else { - printf("ERROR: failed to read json, tags_file not found\n"); + errorPrint("%s", "failed to read json, tags_file not found\n"); goto PARSE_OVER; } @@ -4377,8 +4375,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxSqlLen) { g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; } else { - errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, stbMaxSqlLen input mistake\n"); goto PARSE_OVER; } /* @@ -4395,31 +4392,28 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!multiThreadWriteOneTbl) { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; } else { - printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); + errorPrint("%s", "failed to read json, multiThreadWriteOneTbl not found\n"); goto PARSE_OVER; } */ cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); if (insertRows && insertRows->type == cJSON_Number) { if (insertRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert_rows input mistake\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; } else if (!insertRows) { g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; } else { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert_rows input mistake\n"); goto PARSE_OVER; } cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { if (stbInterlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, interlace rows input mistake\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; @@ -4437,8 +4431,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { errorPrint( - "%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); + "%s", "failed to read json, interlace rows input mistake\n"); goto PARSE_OVER; } @@ -4454,7 +4447,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!disorderRatio) { g_Dbs.db[i].superTbls[j].disorderRatio = 0; } else { - printf("ERROR: failed to read json, disorderRatio not found\n"); + errorPrint("%s", "failed to read json, disorderRatio not found\n"); goto PARSE_OVER; } @@ -4464,7 +4457,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!disorderRange) { g_Dbs.db[i].superTbls[j].disorderRange = 1000; } else { - printf("ERROR: failed to read json, disorderRange not found\n"); + errorPrint("%s", "failed to read json, disorderRange not found\n"); goto PARSE_OVER; } @@ -4472,8 +4465,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (insertInterval && insertInterval->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; if (insertInterval->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert_interval input mistake\n"); goto PARSE_OVER; } } else if (!insertInterval) { @@ -4481,8 +4473,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__, g_args.insert_interval); g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; } else { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, insert_interval input mistake\n"); goto PARSE_OVER; } @@ -4514,7 +4505,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!host) { tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { - printf("ERROR: failed to read json, host not found\n"); + errorPrint("%s", "failed to read json, host not found\n"); goto PARSE_OVER; } @@ -4552,23 +4543,21 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!answerPrompt) { g_args.answer_yes = false; } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; } cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); if (gQueryTimes && gQueryTimes->type == cJSON_Number) { if (gQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + errorPrint("%s()", "failed to read json, query_times input mistake\n"); goto PARSE_OVER; } g_args.query_times = gQueryTimes->valueint; } else if (!gQueryTimes) { g_args.query_times = 1; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, query_times input mistake\n"); goto PARSE_OVER; } @@ -4576,7 +4565,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); } else if (!dbs) { - printf("ERROR: failed to read json, databases not found\n"); + errorPrint("%s", "failed to read json, databases not found\n"); goto PARSE_OVER; } @@ -4590,7 +4579,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.queryMode, "taosc", min(SMALL_BUFF_LEN, strlen("taosc") + 1)); } else { - printf("ERROR: failed to read json, query_mode not found\n"); + errorPrint("%s", "failed to read json, query_mode not found\n"); goto PARSE_OVER; } @@ -4600,7 +4589,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.specifiedQueryInfo.concurrent = 1; g_queryInfo.specifiedQueryInfo.sqlCount = 0; } else if (specifiedQuery->type != cJSON_Object) { - printf("ERROR: failed to read json, super_table_query not found\n"); + errorPrint("%s", "failed to read json, super_table_query not found\n"); goto PARSE_OVER; } else { cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval"); @@ -4615,8 +4604,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes->valueint <= 0) { errorPrint( - "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, specifiedQueryTimes->valueint); + "failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + specifiedQueryTimes->valueint); goto PARSE_OVER; } @@ -4633,8 +4622,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { errorPrint( - "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", - __func__, __LINE__, + "query sqlCount %d or concurrent %d is not correct.\n", g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); goto PARSE_OVER; @@ -4652,8 +4640,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) { g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, async mode input error\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, async mode input error\n"); goto PARSE_OVER; } } else { @@ -4676,7 +4663,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", restart->valuestring)) { g_queryInfo.specifiedQueryInfo.subscribeRestart = false; } else { - printf("ERROR: failed to read json, subscribe restart error\n"); + errorPrint("%s", "failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { @@ -4692,7 +4679,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", keepProgress->valuestring)) { g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); + errorPrint("%s", "failed to read json, subscribe keepProgress error\n"); goto PARSE_OVER; } } else { @@ -4704,15 +4691,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!specifiedSqls) { g_queryInfo.specifiedQueryInfo.sqlCount = 0; } else if (specifiedSqls->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, super sqls not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, super sqls not found\n"); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(specifiedSqls); if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", - __func__, __LINE__, + errorPrint("failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", superSqlSize, g_queryInfo.specifiedQueryInfo.concurrent, MAX_QUERY_SQL_COUNT); @@ -4726,7 +4711,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("ERROR: failed to read json, sql not found\n"); + errorPrint("%s", "failed to read json, sql not found\n"); goto PARSE_OVER; } tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], @@ -4766,7 +4751,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { - printf("ERROR: failed to read json, super query result file not found\n"); + errorPrint("%s", + "failed to read json, super query result file not found\n"); goto PARSE_OVER; } } @@ -4779,7 +4765,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.threadCnt = 1; g_queryInfo.superQueryInfo.sqlCount = 0; } else if (superQuery->type != cJSON_Object) { - printf("ERROR: failed to read json, sub_table_query not found\n"); + errorPrint("%s", "failed to read json, sub_table_query not found\n"); ret = true; goto PARSE_OVER; } else { @@ -4793,24 +4779,22 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); if (superQueryTimes && superQueryTimes->type == cJSON_Number) { if (superQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, superQueryTimes->valueint); + errorPrint("failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + superQueryTimes->valueint); goto PARSE_OVER; } g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; } else if (!superQueryTimes) { g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, query_times input mistake\n"); goto PARSE_OVER; } cJSON* threads = cJSON_GetObjectItem(superQuery, "threads"); if (threads && threads->type == cJSON_Number) { if (threads->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, threads input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, threads input mistake\n"); goto PARSE_OVER; } @@ -4832,8 +4816,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, TSDB_TABLE_NAME_LEN); } else { - errorPrint("%s() LN%d, failed to read json, super table name input error\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, super table name input error\n"); goto PARSE_OVER; } @@ -4845,8 +4828,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("async", superAsyncMode->valuestring)) { g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, async mode input error\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, async mode input error\n"); goto PARSE_OVER; } } else { @@ -4856,8 +4838,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval"); if (superInterval && superInterval->type == cJSON_Number) { if (superInterval->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interval input mistake\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, interval input mistake\n"); goto PARSE_OVER; } g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint; @@ -4875,7 +4856,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", subrestart->valuestring)) { g_queryInfo.superQueryInfo.subscribeRestart = false; } else { - printf("ERROR: failed to read json, subscribe restart error\n"); + errorPrint("%s", "failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { @@ -4891,7 +4872,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); + errorPrint("%s", + "failed to read json, subscribe super table keepProgress error\n"); goto PARSE_OVER; } } else { @@ -4928,14 +4910,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; } else if (superSqls->type != cJSON_Array) { - errorPrint("%s() LN%d: failed to read json, super sqls not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, super sqls not found\n"); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", - __func__, __LINE__, MAX_QUERY_SQL_COUNT); + errorPrint("failed to read json, query sql size overflow, max is %d\n", + MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } @@ -4947,8 +4928,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, sql not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, sql not found\n"); goto PARSE_OVER; } tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, @@ -4962,8 +4942,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (NULL == result) { memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { - errorPrint("%s() LN%d, failed to read json, sub query result file not found\n", - __func__, __LINE__); + errorPrint("%s", "failed to read json, sub query result file not found\n"); goto PARSE_OVER; } } @@ -4981,7 +4960,7 @@ static bool getInfoFromJsonFile(char* file) { FILE *fp = fopen(file, "r"); if (!fp) { - printf("failed to read %s, reason:%s\n", file, strerror(errno)); + errorPrint("failed to read %s, reason:%s\n", file, strerror(errno)); return false; } @@ -4992,14 +4971,14 @@ static bool getInfoFromJsonFile(char* file) { if (len <= 0) { free(content); fclose(fp); - printf("failed to read %s, content is null", file); + errorPrint("failed to read %s, content is null", file); return false; } content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - printf("ERROR: failed to cjson parse %s, invalid json format\n", file); + errorPrint("failed to cjson parse %s, invalid json format\n", file); goto PARSE_OVER; } @@ -5012,13 +4991,13 @@ static bool getInfoFromJsonFile(char* file) { } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { g_args.test_mode = SUBSCRIBE_TEST; } else { - printf("ERROR: failed to read json, filetype not support\n"); + errorPrint("%s", "failed to read json, filetype not support\n"); goto PARSE_OVER; } } else if (!filetype) { g_args.test_mode = INSERT_TEST; } else { - printf("ERROR: failed to read json, filetype not found\n"); + errorPrint("%s", "failed to read json, filetype not found\n"); goto PARSE_OVER; } @@ -5028,8 +5007,8 @@ static bool getInfoFromJsonFile(char* file) { || (SUBSCRIBE_TEST == g_args.test_mode)) { ret = getMetaFromQueryJsonFile(root); } else { - errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", - __func__, __LINE__); + errorPrint("%s", + "input json file type error! please input correct file type: insert or query or subscribe\n"); goto PARSE_OVER; } @@ -5147,7 +5126,7 @@ static int64_t generateStbRowData( || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", 5))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { - errorPrint( "binary or nchar length overflow, max size:%u\n", + errorPrint2("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); return -1; } @@ -5159,7 +5138,7 @@ static int64_t generateStbRowData( } char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); if (NULL == buf) { - errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen); + errorPrint2("calloc failed! size:%d\n", stbInfo->columns[i].dataLen); return -1; } rand_string(buf, stbInfo->columns[i].dataLen); @@ -5222,7 +5201,7 @@ static int64_t generateStbRowData( tmpLen = strlen(tmp); tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, BIGINT_BUFF_LEN)); } else { - errorPrint( "Not support data type: %s\n", + errorPrint2("Not support data type: %s\n", stbInfo->columns[i].dataType); return -1; } @@ -5274,7 +5253,7 @@ static int64_t generateData(char *recBuf, char **data_type, } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { char *s = malloc(lenOfBinary + 1); if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + errorPrint2("%s() LN%d, memory allocation %d bytes failed\n", __func__, __LINE__, lenOfBinary + 1); exit(EXIT_FAILURE); } @@ -5284,7 +5263,7 @@ static int64_t generateData(char *recBuf, char **data_type, } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { char *s = malloc(lenOfBinary + 1); if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + errorPrint2("%s() LN%d, memory allocation %d bytes failed\n", __func__, __LINE__, lenOfBinary + 1); exit(EXIT_FAILURE); } @@ -5311,7 +5290,7 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) { sampleDataBuf = calloc( stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); if (sampleDataBuf == NULL) { - errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", + errorPrint2("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", __func__, __LINE__, stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); @@ -5322,7 +5301,7 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) { int ret = readSampleFromCsvFileToMem(stbInfo); if (0 != ret) { - errorPrint("%s() LN%d, read sample from csv file failed.\n", + errorPrint2("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__); tmfree(sampleDataBuf); stbInfo->sampleDataBuf = NULL; @@ -5377,7 +5356,7 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); if (0 != taos_stmt_execute(pThreadInfo->stmt)) { - errorPrint("%s() LN%d, failied to execute insert statement. reason: %s\n", + errorPrint2("%s() LN%d, failied to execute insert statement. reason: %s\n", __func__, __LINE__, taos_stmt_errstr(pThreadInfo->stmt)); fprintf(stderr, "\n\033[31m === Please reduce batch number if WAL size exceeds limit. ===\033[0m\n\n"); @@ -5387,7 +5366,7 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) break; default: - errorPrint("%s() LN%d: unknown insert mode: %d\n", + errorPrint2("%s() LN%d: unknown insert mode: %d\n", __func__, __LINE__, stbInfo->iface); affectedRows = 0; } @@ -5615,7 +5594,7 @@ static int generateStbSQLHead( tableSeq % stbInfo->tagSampleCount); } if (NULL == tagsValBuf) { - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + errorPrint2("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); return -1; } @@ -5766,7 +5745,7 @@ static int32_t prepareStmtBindArrayByType( if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { if (dataLen > TSDB_MAX_BINARY_LEN) { - errorPrint( "binary length overflow, max size:%u\n", + errorPrint2("binary length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); return -1; } @@ -5789,7 +5768,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "NCHAR", strlen("NCHAR"))) { if (dataLen > TSDB_MAX_BINARY_LEN) { - errorPrint( "nchar length overflow, max size:%u\n", + errorPrint2("nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); return -1; } @@ -5937,7 +5916,7 @@ static int32_t prepareStmtBindArrayByType( value, &tmpEpoch, strlen(value), timePrec, 0)) { free(bind_ts2); - errorPrint("Input %s, time format error!\n", value); + errorPrint2("Input %s, time format error!\n", value); return -1; } *bind_ts2 = tmpEpoch; @@ -5953,7 +5932,7 @@ static int32_t prepareStmtBindArrayByType( bind->length = &bind->buffer_length; bind->is_null = NULL; } else { - errorPrint( "No support data type: %s\n", dataType); + errorPrint2("Not support data type: %s\n", dataType); return -1; } @@ -5970,7 +5949,7 @@ static int32_t prepareStmtBindArrayByTypeForRand( if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { if (dataLen > TSDB_MAX_BINARY_LEN) { - errorPrint( "binary length overflow, max size:%u\n", + errorPrint2("binary length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); return -1; } @@ -5993,7 +5972,7 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else if (0 == strncasecmp(dataType, "NCHAR", strlen("NCHAR"))) { if (dataLen > TSDB_MAX_BINARY_LEN) { - errorPrint( "nchar length overflow, max size:%u\n", + errorPrint2("nchar length overflow, max size: %u\n", (uint32_t)TSDB_MAX_BINARY_LEN); return -1; } @@ -6145,7 +6124,7 @@ static int32_t prepareStmtBindArrayByTypeForRand( if (TSDB_CODE_SUCCESS != taosParseTime( value, &tmpEpoch, strlen(value), timePrec, 0)) { - errorPrint("Input %s, time format error!\n", value); + errorPrint2("Input %s, time format error!\n", value); return -1; } *bind_ts2 = tmpEpoch; @@ -6163,7 +6142,7 @@ static int32_t prepareStmtBindArrayByTypeForRand( *ptr += bind->buffer_length; } else { - errorPrint( "No support data type: %s\n", dataType); + errorPrint2("No support data type: %s\n", dataType); return -1; } @@ -6181,7 +6160,7 @@ static int32_t prepareStmtWithoutStb( TAOS_STMT *stmt = pThreadInfo->stmt; int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { - errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", + errorPrint2("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", tableName, ret, taos_stmt_errstr(stmt)); return ret; } @@ -6190,7 +6169,7 @@ static int32_t prepareStmtWithoutStb( char *bindArray = malloc(sizeof(TAOS_BIND) * (g_args.num_of_CPR + 1)); if (bindArray == NULL) { - errorPrint("Failed to allocate %d bind params\n", + errorPrint2("Failed to allocate %d bind params\n", (g_args.num_of_CPR + 1)); return -1; } @@ -6231,13 +6210,13 @@ static int32_t prepareStmtWithoutStb( } } if (0 != taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray)) { - errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_bind_param() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); break; } // if msg > 3MB, break if (0 != taos_stmt_add_batch(stmt)) { - errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_add_batch() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); break; } @@ -6260,7 +6239,7 @@ static int32_t prepareStbStmtBindTag( { char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); return -1; } @@ -6292,7 +6271,7 @@ static int32_t prepareStbStmtBindRand( { char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); return -1; } @@ -6395,7 +6374,7 @@ static int32_t prepareStbStmtRand( } if (NULL == tagsValBuf) { - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + errorPrint2("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); return -1; } @@ -6403,7 +6382,7 @@ static int32_t prepareStbStmtRand( char *tagsArray = calloc(1, sizeof(TAOS_BIND) * stbInfo->tagCount); if (NULL == tagsArray) { tmfree(tagsValBuf); - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + errorPrint2("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); return -1; } @@ -6422,14 +6401,14 @@ static int32_t prepareStbStmtRand( tmfree(tagsArray); if (0 != ret) { - errorPrint("%s() LN%d, stmt_set_tbname_tags() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_set_tbname_tags() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } } else { ret = taos_stmt_set_tbname(stmt, tableName); if (0 != ret) { - errorPrint("%s() LN%d, stmt_set_tbname() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_set_tbname() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } @@ -6437,7 +6416,7 @@ static int32_t prepareStbStmtRand( char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + errorPrint2("%s() LN%d, Failed to allocate %d bind params\n", __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } @@ -6456,7 +6435,7 @@ static int32_t prepareStbStmtRand( } ret = taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); if (0 != ret) { - errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_bind_param() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); free(bindArray); return -1; @@ -6464,7 +6443,7 @@ static int32_t prepareStbStmtRand( // if msg > 3MB, break ret = taos_stmt_add_batch(stmt); if (0 != ret) { - errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_add_batch() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); free(bindArray); return -1; @@ -6508,7 +6487,7 @@ static int32_t prepareStbStmtWithSample( } if (NULL == tagsValBuf) { - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + errorPrint2("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); return -1; } @@ -6516,7 +6495,7 @@ static int32_t prepareStbStmtWithSample( char *tagsArray = calloc(1, sizeof(TAOS_BIND) * stbInfo->tagCount); if (NULL == tagsArray) { tmfree(tagsValBuf); - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + errorPrint2("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); return -1; } @@ -6535,14 +6514,14 @@ static int32_t prepareStbStmtWithSample( tmfree(tagsArray); if (0 != ret) { - errorPrint("%s() LN%d, stmt_set_tbname_tags() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_set_tbname_tags() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } } else { ret = taos_stmt_set_tbname(stmt, tableName); if (0 != ret) { - errorPrint("%s() LN%d, stmt_set_tbname() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_set_tbname() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } @@ -6564,14 +6543,14 @@ static int32_t prepareStbStmtWithSample( } ret = taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); if (0 != ret) { - errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_bind_param() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } // if msg > 3MB, break ret = taos_stmt_add_batch(stmt); if (0 != ret) { - errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", + errorPrint2("%s() LN%d, stmt_add_batch() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); return -1; } @@ -6732,7 +6711,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->buffer = calloc(maxSqlLen, 1); if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + errorPrint2( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", __func__, __LINE__, maxSqlLen, strerror(errno)); return NULL; } @@ -6780,7 +6759,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { getTableName(tableName, pThreadInfo, tableSeq); if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", + errorPrint2("[%d] %s() LN%d, getTableName return null\n", pThreadInfo->threadID, __func__, __LINE__); free(pThreadInfo->buffer); return NULL; @@ -6847,7 +6826,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %d\n", + errorPrint2("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); goto free_of_interlace; } else if (generated == 0) { @@ -6901,7 +6880,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampUs(); if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", + errorPrint2("[%d] %s() LN%d Failed to insert records of batch %d\n", pThreadInfo->threadID, __func__, __LINE__, batchPerTbl); if (batchPerTbl > 0) { @@ -6928,7 +6907,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalDelay += delay; if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", + errorPrint2("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch, affectedRows, pThreadInfo->buffer); goto free_of_interlace; @@ -6986,7 +6965,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->buffer = calloc(maxSqlLen, 1); if (NULL == pThreadInfo->buffer) { - errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n", + errorPrint2("Failed to alloc %"PRIu64" bytes, reason:%s\n", maxSqlLen, strerror(errno)); return NULL; @@ -7027,7 +7006,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", + errorPrint2("[%d] %s() LN%d, getTableName return null\n", pThreadInfo->threadID, __func__, __LINE__); free(pThreadInfo->buffer); return NULL; @@ -7116,7 +7095,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->totalDelay += delay; if (affectedRows < 0) { - errorPrint("%s() LN%d, affected rows: %d\n", + errorPrint2("%s() LN%d, affected rows: %d\n", __func__, __LINE__, affectedRows); goto free_of_progressive; } @@ -7278,7 +7257,7 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * uint16_t rest_port = port + TSDB_PORT_HTTP; struct hostent *server = gethostbyname(host); if ((server == NULL) || (server->h_addr == NULL)) { - errorPrint("%s", "ERROR, no such host"); + errorPrint2("%s", "no such host"); return -1; } @@ -7303,7 +7282,7 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) { stbInfo->sampleBindArray = calloc(1, sizeof(char *) * MAX_SAMPLES_ONCE_FROM_FILE); if (stbInfo->sampleBindArray == NULL) { - errorPrint("%s() LN%d, Failed to allocate %"PRIu64" bind array buffer\n", + errorPrint2("%s() LN%d, Failed to allocate %"PRIu64" bind array buffer\n", __func__, __LINE__, (uint64_t)sizeof(char *) * MAX_SAMPLES_ONCE_FROM_FILE); return -1; } @@ -7312,7 +7291,7 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) { char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + errorPrint2("%s() LN%d, Failed to allocate %d bind params\n", __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } @@ -7344,7 +7323,7 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) char *bindBuffer = calloc(1, index + 1); if (bindBuffer == NULL) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); return -1; } @@ -7382,7 +7361,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } else if (0 == strncasecmp(precision, "ns", 2)) { timePrec = TSDB_TIME_PRECISION_NANO; } else { - errorPrint("Not support precision: %s\n", precision); + errorPrint2("Not support precision: %s\n", precision); exit(EXIT_FAILURE); } } @@ -7412,7 +7391,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample")))) { if (0 != prepareSampleDataForSTable(stbInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", + errorPrint2("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__); exit(EXIT_FAILURE); } @@ -7422,7 +7401,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); if (NULL == taos0) { - errorPrint("%s() LN%d, connect to server fail , reason: %s\n", + errorPrint2("%s() LN%d, connect to server fail , reason: %s\n", __func__, __LINE__, taos_errstr(NULL)); exit(EXIT_FAILURE); } @@ -7477,7 +7456,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, limit * TSDB_TABLE_NAME_LEN); if (stbInfo->childTblName == NULL) { taos_close(taos0); - errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + errorPrint2("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); exit(EXIT_FAILURE); } @@ -7583,7 +7562,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, g_Dbs.password, db_name, g_Dbs.port); if (NULL == pThreadInfo->taos) { free(infos); - errorPrint( + errorPrint2( "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", __func__, __LINE__, taos_errstr(NULL)); @@ -7599,7 +7578,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (NULL == pThreadInfo->stmt) { free(pids); free(infos); - errorPrint( + errorPrint2( "%s() LN%d, failed init stmt, reason: %s\n", __func__, __LINE__, taos_errstr(NULL)); @@ -7611,7 +7590,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, free(pids); free(infos); free(stmtBuffer); - errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", + errorPrint2("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", ret, taos_stmt_errstr(pThreadInfo->stmt)); exit(EXIT_FAILURE); } @@ -7755,7 +7734,7 @@ static void *readTable(void *sarg) { char *tb_prefix = pThreadInfo->tb_prefix; FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); + errorPrint2("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); free(command); return NULL; } @@ -7791,7 +7770,7 @@ static void *readTable(void *sarg) { int32_t code = taos_errno(pSql); if (code != 0) { - errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); + errorPrint2("Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); fclose(fp); @@ -7873,7 +7852,7 @@ static void *readMetric(void *sarg) { int32_t code = taos_errno(pSql); if (code != 0) { - errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); + errorPrint2("Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); fclose(fp); @@ -7920,7 +7899,7 @@ static int insertTestProcess() { debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); if (NULL == g_fpOfInsertResult) { - errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); + errorPrint("Failed to open %s for save result\n", g_Dbs.resultFile); return -1; } @@ -8022,7 +8001,7 @@ static void *specifiedTableQuery(void *sarg) { NULL, g_queryInfo.port); if (taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + errorPrint2("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } else { @@ -8034,7 +8013,7 @@ static void *specifiedTableQuery(void *sarg) { sprintf(sqlStr, "use %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); - errorPrint( "use database %s failed!\n\n", + errorPrint("use database %s failed!\n\n", g_queryInfo.dbName); return NULL; } @@ -8200,7 +8179,7 @@ static int queryTestProcess() { NULL, g_queryInfo.port); if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", + errorPrint("Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(EXIT_FAILURE); } @@ -8258,7 +8237,7 @@ static int queryTestProcess() { taos_close(taos); free(infos); free(pids); - errorPrint( "use database %s failed!\n\n", + errorPrint2("use database %s failed!\n\n", g_queryInfo.dbName); return -1; } @@ -8356,7 +8335,7 @@ static int queryTestProcess() { static void stable_sub_callback( TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + errorPrint2("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); return; } @@ -8369,7 +8348,7 @@ static void stable_sub_callback( static void specified_sub_callback( TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + errorPrint2("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); return; } @@ -8408,7 +8387,7 @@ static TAOS_SUB* subscribeImpl( } if (tsub == NULL) { - errorPrint("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + errorPrint2("failed to create subscription. topic:%s, sql:%s\n", topic, sql); return NULL; } @@ -8439,7 +8418,7 @@ static void *superSubscribe(void *sarg) { g_queryInfo.dbName, g_queryInfo.port); if (pThreadInfo->taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + errorPrint2("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); free(subSqlStr); return NULL; @@ -8450,7 +8429,7 @@ static void *superSubscribe(void *sarg) { sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); - errorPrint( "use database %s failed!\n\n", + errorPrint2("use database %s failed!\n\n", g_queryInfo.dbName); free(subSqlStr); return NULL; @@ -8586,7 +8565,7 @@ static void *specifiedSubscribe(void *sarg) { g_queryInfo.dbName, g_queryInfo.port); if (pThreadInfo->taos == NULL) { - errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + errorPrint2("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } @@ -8693,7 +8672,7 @@ static int subscribeTestProcess() { g_queryInfo.dbName, g_queryInfo.port); if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", + errorPrint2("Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(EXIT_FAILURE); } @@ -8721,7 +8700,7 @@ static int subscribeTestProcess() { g_queryInfo.specifiedQueryInfo.sqlCount); } else { if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { - errorPrint("%s() LN%d, sepcified query sqlCount %d.\n", + errorPrint2("%s() LN%d, sepcified query sqlCount %d.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); exit(EXIT_FAILURE); @@ -8738,7 +8717,7 @@ static int subscribeTestProcess() { g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + errorPrint2("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(EXIT_FAILURE); } @@ -8773,7 +8752,7 @@ static int subscribeTestProcess() { g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", + errorPrint2("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); // taos_close(taos); exit(EXIT_FAILURE); @@ -9039,7 +9018,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) memcpy(cmd + cmd_len, line, read_len); if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) { - errorPrint("%s() LN%d, queryDbExec %s failed!\n", + errorPrint2("%s() LN%d, queryDbExec %s failed!\n", __func__, __LINE__, cmd); tmfree(cmd); tmfree(line); @@ -9113,7 +9092,7 @@ static void queryResult() { g_Dbs.port); if (pThreadInfo->taos == NULL) { free(pThreadInfo); - errorPrint( "Failed to connect to TDengine, reason:%s\n", + errorPrint2("Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(EXIT_FAILURE); } @@ -9135,7 +9114,7 @@ static void testCmdLine() { if (strlen(configDir)) { wordexp_t full_path; if (wordexp(configDir, &full_path, 0) != 0) { - errorPrint( "Invalid path %s\n", configDir); + errorPrint("Invalid path %s\n", configDir); return; } taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); From 78da4fedceb3b965145e03e9f9c2c60b8ea4c888 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Aug 2021 18:40:26 +0800 Subject: [PATCH 48/49] minor changes --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3314798d6d..efdc644f2c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2440,7 +2440,7 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SRetrieveSupport* pSupport = pSub->param; tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); - tscProcessSql(pSub); + tscBuildAndSendRequest(pSub, NULL); } tfree(p); From aad82de6fcd940d0409853fb8696a2b2004115df Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Aug 2021 19:32:20 +0800 Subject: [PATCH 49/49] TS-177 change default log len from 100 to 512 --- src/plugins/monitor/src/monMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 6e583fe0df..fea793fa86 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -34,7 +34,7 @@ #define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} #define SQL_LENGTH 1030 -#define LOG_LEN_STR 100 +#define LOG_LEN_STR 512 #define IP_LEN_STR TSDB_EP_LEN #define CHECK_INTERVAL 1000