From b4eee354813f7626fb3ce8ec1fb474d03868eced Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 Aug 2021 17:31:20 +0800 Subject: [PATCH 01/23] [TD-6011]: where clause including 'bool' Keyword causes core dump --- src/common/src/tvariant.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 9e0f7ffc74..42ece19588 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -31,12 +31,12 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { switch (token->type) { case TSDB_DATA_TYPE_BOOL: { - int32_t k = strncasecmp(token->z, "true", 4); - if (k == 0) { + if (strncasecmp(token->z, "true", 4) == 0) { pVar->i64 = TSDB_TRUE; - } else { - assert(strncasecmp(token->z, "false", 5) == 0); + } else if (strncasecmp(token->z, "false", 5) == 0) { pVar->i64 = TSDB_FALSE; + } else { + return; } break; From 27dab670f4790ef84e7ae79aa6ab63550efa8634 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 19 Aug 2021 09:36:09 +0800 Subject: [PATCH 02/23] fix illegal sql issue --- src/client/src/tscPrepare.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 06a9505086..2418e7dbdb 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -206,6 +206,8 @@ static int normalStmtPrepare(STscStmt* stmt) { return code; } start = i + token.n; + } else if (token.type == TK_ILLEGAL) { + return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid sql"); } i += token.n; From 9449c97ff5c0e145ae899c8237edde143ea60262 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Thu, 19 Aug 2021 09:52:53 +0800 Subject: [PATCH 03/23] [TD-6167] stmt api test --- tests/script/api/makefile | 5 +- tests/script/api/stmt_function.c | 383 +++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 tests/script/api/stmt_function.c diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 7595594cbf..92d0a89b0f 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -6,7 +6,8 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ + -fsanitize=address all: $(TARGET) @@ -14,8 +15,10 @@ exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) gcc $(CFLAGS) ./stmtTest.c -o $(ROOT)stmtTest $(LFLAGS) + gcc $(CFLAGS) ./stmt_function.c -o $(ROOT)stmt_function $(LFLAGS) clean: rm $(ROOT)batchprepare rm $(ROOT)stmtBatchTest rm $(ROOT)stmtTest + rm $(ROOT)stmt_function diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c new file mode 100644 index 0000000000..13ae90c211 --- /dev/null +++ b/tests/script/api/stmt_function.c @@ -0,0 +1,383 @@ +#include +#include +#include +#include "taos.h" +#include +#include +#include +#include + +void execute_simple_sql(void *taos, char *sql) { + TAOS_RES *result = taos_query(taos, sql); + if ( result == NULL || taos_errno(result) != 0) { + printf( "failed to %s, Reason: %s\n" , sql, taos_errstr(result)); + taos_free_result(result); + exit(EXIT_FAILURE); + } + taos_free_result(result); +} + +void taos_stmt_init_test() { + printf("start taos_stmt_init test \n"); + void *taos = NULL; + TAOS_STMT *stmt = NULL; + stmt = taos_stmt_init(taos); + assert(stmt == NULL); + // ASM ERROR + // assert(taos_stmt_close(stmt) != 0); + taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_init test\n"); +} +void taos_stmt_preprare_test() { + printf("start taos_stmt_prepare test\n"); + char *stmt_sql = calloc(1, 1048576); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + void *taos = NULL; + taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + // below will make client dead lock + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + + // assert(taos_stmt_close(stmt) == 0); + // stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "select from ?"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into ? values (?,?,?,?,?,?,?,?,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into super values (?,?,?,?,?,?,?,?,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); + assert(taos_stmt_close(stmt) == 0); + + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + sprintf(stmt_sql, "insert into ? values (?,?,?,?,?,?,?,?,1,?,?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_close(stmt) == 0); + + free(stmt_sql); + printf("finish taos_stmt_prepare test\n"); +} + +void taos_stmt_set_tbname_test() { + printf("start taos_stmt_set_tbname test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1, 200); + // ASM ERROR + // assert(taos_stmt_set_tbname(stmt, name) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_set_tbname(stmt, name) != 0); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + sprintf(name, "super"); + assert(stmt != NULL); + assert(taos_stmt_set_tbname(stmt, name) == 0); + free(name); + free(stmt_sql); + taos_stmt_close(stmt); + printf("finish taos_stmt_set_tbname test\n"); +} + +void taos_stmt_set_tbname_tags_test() { + printf("start taos_stmt_set_tbname_tags test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1,20); + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND)); + // ASM ERROR + // assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create stable super(ts timestamp, c1 int) tags (id int)"); + execute_simple_sql(taos, "create table tb using super tags (1)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? using super tags (?) values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + sprintf(name, "tb"); + assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); + int t = 1; + tags->buffer_length = TSDB_DATA_TYPE_INT; + tags->buffer_length = sizeof(uint32_t); + tags->buffer = &t; + tags->length = &tags->buffer_length; + tags->is_null = NULL; + assert(taos_stmt_set_tbname_tags(stmt, name, tags) == 0); + free(stmt_sql); + free(name); + free(tags); + taos_stmt_close(stmt); + printf("finish taos_stmt_set_tbname_tags test\n"); +} + +void taos_stmt_set_sub_tbname_test() { + printf("start taos_stmt_set_sub_tbname test\n"); + TAOS_STMT *stmt = NULL; + char *name = calloc(1, 200); + // ASM ERROR + // assert(taos_stmt_set_sub_tbname(stmt, name) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create stable super(ts timestamp, c1 int) tags (id int)"); + execute_simple_sql(taos, "create table tb using super tags (1)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_set_sub_tbname(stmt, name) != 0); + sprintf(name, "tb"); + assert(taos_stmt_set_sub_tbname(stmt, name) == 0); + // assert(taos_load_table_info(taos, "super, tb") == 0); + // assert(taos_stmt_set_sub_tbname(stmt, name) == 0); + free(name); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_set_sub_tbname test\n"); +} + +void taos_stmt_bind_param_test() { + printf("start taos_stmt_bind_param test\n"); + TAOS_STMT *stmt = NULL; + TAOS_BIND *binds = NULL; + assert(taos_stmt_bind_param(stmt, binds) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_bind_param(stmt, binds) != 0); + free(binds); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_bind_param(stmt, params) != 0); + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + free(params); + free(stmt_sql); + taos_stmt_close(stmt); + printf("finish taos_stmt_bind_param test\n"); +} + +void taos_stmt_bind_single_param_batch_test() { + printf("start taos_stmt_bind_single_param_batch test\n"); + TAOS_STMT *stmt = NULL; + TAOS_MULTI_BIND *bind = NULL; + assert(taos_stmt_bind_single_param_batch(stmt, bind, 0) != 0); + printf("finish taos_stmt_bind_single_param_batch test\n"); +} + +void taos_stmt_bind_param_batch_test() { + printf("start taos_stmt_bind_param_batch test\n"); + TAOS_STMT *stmt = NULL; + TAOS_MULTI_BIND *bind = NULL; + assert(taos_stmt_bind_param_batch(stmt, bind) != 0); + printf("finish taos_stmt_bind_param_batch test\n"); +} + +void taos_stmt_add_batch_test() { + printf("start taos_stmt_add_batch test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_add_batch(stmt) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_add_batch(stmt) != 0); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + assert(taos_stmt_add_batch(stmt) == 0); + free(params); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_add_batch test\n"); +} + +void taos_stmt_execute_test() { + printf("start taos_stmt_execute test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_execute(stmt) != 0); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + execute_simple_sql(taos, "drop database if exists stmt_test"); + execute_simple_sql(taos, "create database stmt_test"); + execute_simple_sql(taos, "use stmt_test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + assert(taos_stmt_execute(stmt) != 0); + char* stmt_sql = calloc(1, 1000); + sprintf(stmt_sql, "insert into ? values (?,?)"); + assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); + assert(taos_stmt_execute(stmt) != 0); + TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + int64_t ts = (int64_t)1591060628000; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(uint64_t); + params[0].buffer = &ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + int32_t i = (int32_t)21474; + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(int32_t); + params[1].buffer = &i; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + assert(taos_stmt_set_tbname(stmt, "super") == 0); + assert(taos_stmt_execute(stmt) != 0); + assert(taos_stmt_bind_param(stmt, params) == 0); + assert(taos_stmt_execute(stmt) != 0); + assert(taos_stmt_add_batch(stmt) == 0); + assert(taos_stmt_execute(stmt) == 0); + free(params); + free(stmt_sql); + assert(taos_stmt_close(stmt) == 0); + printf("finish taos_stmt_execute test\n"); +} + +void taos_stmt_use_result_test() { + printf("start taos_stmt_use_result test\n"); + TAOS_STMT *stmt = NULL; + assert(taos_stmt_use_result(stmt) == NULL); + void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); + if(taos == NULL) { + printf("Cannot connect to tdengine server\n"); + exit(EXIT_FAILURE); + } + printf("finish taos_stmt_use_result test\n"); + stmt = taos_stmt_init(taos); + assert(stmt != NULL); + TAOS_RES* result = taos_stmt_use_result(stmt); + int rows = 0; + TAOS_ROW row; + while ((row = taos_fetch_row(result))) { + rows++; + } + printf("rows: %d\n", rows); + taos_free_result(result); + // assert(taos_stmt_use_result(stmt) == NULL); + assert(taos_stmt_close(stmt) == 0); +} + +void taos_stmt_close_test() { + printf("start taos_stmt_close test\n"); + // ASM ERROR + // TAOS_STMT *stmt = NULL; + // assert(taos_stmt_close(stmt) != 0); + printf("finish taos_stmt_close test\n"); +} + +void test_api_reliability() { + // ASM catch memory leak + taos_stmt_init_test(); + taos_stmt_preprare_test(); + taos_stmt_set_tbname_test(); + taos_stmt_set_tbname_tags_test(); + taos_stmt_set_sub_tbname_test(); + taos_stmt_bind_param_test(); + taos_stmt_bind_single_param_batch_test(); + taos_stmt_bind_param_batch_test(); + taos_stmt_add_batch_test(); + taos_stmt_execute_test(); + taos_stmt_use_result_test(); + taos_stmt_close_test(); +} + +int main(int argc, char *argv[]) { + test_api_reliability(); + return 0; +} \ No newline at end of file From d453be0785236097dd6a58eae778486d0e00b556 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 Aug 2021 10:26:50 +0800 Subject: [PATCH 04/23] fix: fix altering schema too much corrupting meta file --- src/tsdb/src/tsdbMeta.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 96e86a6d99..547030a431 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1250,8 +1250,14 @@ static int tsdbEncodeTable(void **buf, STable *pTable) { tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); tlen += tdEncodeKVRow(buf, pTable->tagVal); } else { - tlen += taosEncodeFixedU8(buf, (uint8_t)taosArrayGetSize(pTable->schema)); - for (int i = 0; i < taosArrayGetSize(pTable->schema); i++) { + uint32_t arraySize = (uint32_t)taosArrayGetSize(pTable->schema); + if(arraySize > UINT8_MAX) { + tlen += taosEncodeFixedU8(buf, 0); + tlen += taosEncodeFixedU32(buf, arraySize); + } else { + tlen += taosEncodeFixedU8(buf, (uint8_t)arraySize); + } + for (uint32_t i = 0; i < arraySize; i++) { STSchema *pSchema = taosArrayGetP(pTable->schema, i); tlen += tdEncodeSchema(buf, pSchema); } @@ -1284,8 +1290,11 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); buf = tdDecodeKVRow(buf, &(pTable->tagVal)); } else { - uint8_t nSchemas; - buf = taosDecodeFixedU8(buf, &nSchemas); + uint32_t nSchemas; + buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); + if(nSchemas == 0) { + buf = taosDecodeFixedU32(buf, &nSchemas); + } for (int i = 0; i < nSchemas; i++) { STSchema *pSchema; buf = tdDecodeSchema(buf, &pSchema); @@ -1485,4 +1494,4 @@ static void tsdbFreeTableSchema(STable *pTable) { taosArrayDestroy(pTable->schema); } -} \ No newline at end of file +} From 22d3dda4b5171697d2f824c6dc41b6d4fc39e034 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 Aug 2021 10:42:28 +0800 Subject: [PATCH 05/23] fix: init problem --- src/tsdb/src/tsdbMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 547030a431..8407a0519a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1290,7 +1290,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); buf = tdDecodeKVRow(buf, &(pTable->tagVal)); } else { - uint32_t nSchemas; + uint32_t nSchemas = 0; buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); if(nSchemas == 0) { buf = taosDecodeFixedU32(buf, &nSchemas); From 8104f47d7905f88661c17b4b3d9a8b3f8cceee49 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 19 Aug 2021 13:03:11 +0800 Subject: [PATCH 06/23] [TD-6155]:leave free tag expr at the end of sql parsing --- src/client/src/tscSQLParser.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 235ba15c8b..30a46ccc48 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4884,10 +4884,6 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { tSqlExprDestroy(pCondExpr->pTableCond); } - if (pCondExpr->pTagCond) { - tSqlExprDestroy(pCondExpr->pTagCond); - } - if (pCondExpr->pColumnCond) { tSqlExprDestroy(pCondExpr->pColumnCond); } From f56a0d810d47953f601ca4e513534a9798d2264e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 19 Aug 2021 13:03:11 +0800 Subject: [PATCH 07/23] [TD-6155]:leave free tag expr at the end of sql parsing --- src/client/src/tscSQLParser.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 235ba15c8b..30a46ccc48 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4884,10 +4884,6 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { tSqlExprDestroy(pCondExpr->pTableCond); } - if (pCondExpr->pTagCond) { - tSqlExprDestroy(pCondExpr->pTagCond); - } - if (pCondExpr->pColumnCond) { tSqlExprDestroy(pCondExpr->pColumnCond); } From 411543694be7de2b5f300d789fd4314bcca4b6a7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 19 Aug 2021 21:29:52 +0800 Subject: [PATCH 08/23] Hotfix/sangshuduo/td 5136 taosdemo rework for master (#7473) * cherry pick from develop branch. * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * change malloc to calloc for pid allocation. * fix typo. * fix binary length default value and group id logic. * fix help msg. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9a7d3cd25c..316fed117d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -752,12 +752,13 @@ static void printHelp() { "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC."); printf("%s%s%s%s\n", indent, "-b", indent, "The data_type of columns, default: FLOAT, INT, FLOAT."); - printf("%s%s%s%s\n", indent, "-w", indent, - "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s%s%s%d\n", indent, "-w", indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is ", + g_args.len_of_binary); printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, - "The number of columns per record. Default is ", + "The number of columns per record. Demo mode by default is ", DEFAULT_DATATYPE_NUM, - ". Max values is ", + " (float, int, float). Max values is ", MAX_NUM_COLUMNS); printf("%s%s%s%s\n", indent, indent, indent, "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); From 7019c8f3331eff6e18c228f1461e389bbaaffaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=A1=E6=AF=94=E5=8D=A1=E6=AF=94?= <30525741+jackwener@users.noreply.github.com> Date: Thu, 19 Aug 2021 21:31:12 +0800 Subject: [PATCH 09/23] Update docs.md --- documentation20/cn/01.evaluation/docs.md | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 7f70ccec56..4aba82a54c 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -2,18 +2,18 @@ ## TDengine 简介 -TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品,它不依赖任何第三方软件,也不是优化或包装了一个开源的数据库或流式计算产品,而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品,在时序空间大数据处理上,有着自己独到的优势。 +TDengine 是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品,它不依赖任何第三方软件,也不是优化或包装了一个开源的数据库或流式计算产品,而是在吸取众多传统关系型数据库、NoSQL 数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品,在时序空间大数据处理上,有着自己独到的优势。 -TDengine的模块之一是时序数据库。但除此之外,为减少研发的复杂度、系统维护的难度,TDengine还提供缓存、消息队列、订阅、流式计算等功能,为物联网、工业互联网大数据的处理提供全栈的技术方案,是一个高效易用的物联网大数据平台。与Hadoop等典型的大数据平台相比,它具有如下鲜明的特点: +TDengine 的模块之一是时序数据库。但除此之外,为减少研发的复杂度、系统维护的难度,TDengine 还提供缓存、消息队列、订阅、流式计算等功能,为物联网、工业互联网大数据的处理提供全栈的技术方案,是一个高效易用的物联网大数据平台。与 Hadoop 等典型的大数据平台相比,它具有如下鲜明的特点: -* __10倍以上的性能提升__:定义了创新的数据存储结构,单核每秒能处理至少2万次请求,插入数百万个数据点,读出一千万以上数据点,比现有通用数据库快十倍以上。 -* __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的1/10。 -* __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。 -* __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, MATLAB随时进行。 -* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 -* __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类似标准SQL,支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似,零学习成本。 +* __10倍以上的性能提升__:定义了创新的数据存储结构,单核每秒能处理至少 2 万次请求,插入数百万个数据点,读出一千万以上数据点,比现有通用数据库快十倍以上。 +* __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的 1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的 1/10。 +* __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成 Kafka/Redis/HBase/Spark/HDFS 等软件,大幅降低应用开发和维护的复杂度成本。 +* __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过 Shell, Python, R, MATLAB 随时进行。 +* __与第三方工具无缝连接__:不用一行代码,即可与 Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R 等集成。后续将支持 OPC, Hadoop, Spark 等,BI 工具也将无缝连接。 +* __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类标准 SQL,支持 RESTful,支持 Python/Java/C/C++/C#/Go/Node.js, 与 MySQL 相似,零学习成本。 -采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 +采用 TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。 ![TDengine技术生态图](page://images/eco_system.png)
图 1. TDengine技术生态图
@@ -21,11 +21,12 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 ## TDengine 总体适用场景 -作为一个IOT大数据平台,TDengine的典型适用场景是在IOT范畴,而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统,比如CRM,ERP等,不在本文讨论范围内。 +作为一个 IOT 大数据平台,TDengine 的典型适用场景是在 IOT 范畴,而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统,比如 CRM,ERP 等,不在本文讨论范围内。 ### 数据源特点和需求 -从数据源角度,设计人员可以从下面几个角度分析TDengine在目标应用系统里面的适用性。 + +从数据源角度,设计人员可以从下面几个角度分析 TDengine 在目标应用系统里面的适用性。 |数据源特点和需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| @@ -34,6 +35,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 |数据源数目巨大| | | √ |TDengine设计中包含专门针对大量数据源的优化,包括数据的写入和查询,尤其适合高效处理海量(千万或者更多量级)的数据源。| ### 系统架构要求 + |系统架构要求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求简单可靠的系统架构| | | √ |TDengine的系统架构非常简单可靠,自带消息队列,缓存,流式计算,监控等功能,无需集成额外的第三方产品。| @@ -41,12 +43,14 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 |标准化规范| | | √ |TDengine使用标准的SQL语言提供主要功能,遵守标准化规范。| ### 系统功能需求 + |系统功能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求完整的内置数据处理算法| | √ | |TDengine的实现了通用的数据处理算法,但是还没有做到妥善处理各行各业的所有要求,因此特殊类型的处理还需要应用层面处理。| |需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理,或者应该考虑TDengine和关系型数据系统配合实现系统功能。| ### 系统性能需求 + |系统性能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求较大的总体处理能力| | | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| @@ -54,6 +58,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 |要求快速处理小粒度数据| | | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| ### 系统维护需求 + |系统维护需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| |要求系统可靠运行| | | √ |TDengine的系统架构非常稳定可靠,日常维护也简单便捷,对维护人员的要求简洁明了,最大程度上杜绝人为错误和事故。| From 97281931da9b60325c1902d997d01ae9a3e4796d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 20 Aug 2021 01:09:32 +0800 Subject: [PATCH 10/23] [TD-6219]: remove few taosdemo macros. (#7469) --- src/kit/taosdemo/taosdemo.c | 62 ------------------------------------- 1 file changed, 62 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 316fed117d..343c3f2e03 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -53,14 +53,6 @@ #include "taoserror.h" #include "tutil.h" -#define STMT_IFACE_ENABLED 1 -#define NANO_SECOND_ENABLED 1 -#define SET_THREADNAME_ENABLED 1 - -#if SET_THREADNAME_ENABLED == 0 -#define setThreadName(name) -#endif - #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -295,9 +287,7 @@ typedef struct SSuperTable_S { uint64_t lenOfTagOfOneRow; char* sampleDataBuf; -#if STMT_IFACE_ENABLED == 1 char* sampleBindArray; -#endif //int sampleRowCount; //int sampleUsePos; @@ -733,11 +723,7 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-P", indent, "The TCP/IP port number to use for the connection. Default is 0."); printf("%s%s%s%s\n", indent, "-I", indent, -#if STMT_IFACE_ENABLED == 1 "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); -#else - "The interface (taosc, rest) taosdemo uses. Default is 'taosc'."); -#endif printf("%s%s%s%s\n", indent, "-d", indent, "Destination database. Default is 'test'."); printf("%s%s%s%s\n", indent, "-a", indent, @@ -850,10 +836,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = TAOSC_IFACE; } else if (0 == strcasecmp(argv[i], "rest")) { arguments->iface = REST_IFACE; -#if STMT_IFACE_ENABLED == 1 } else if (0 == strcasecmp(argv[i], "stmt")) { arguments->iface = STMT_IFACE; -#endif } else { errorPrint("%s", "\n\t-I need a valid string following!\n"); exit(EXIT_FAILURE); @@ -1695,9 +1679,7 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))) { printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); @@ -1888,9 +1870,7 @@ static void printfInsertMetaToFile(FILE* fp) { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); @@ -2093,10 +2073,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); -#if NANO_SECOND_ENABLED == 1 } if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); -#endif } else { tt = (time_t)(val / 1000); } @@ -2118,10 +2096,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(buf + pos, ".%06d", (int)(val % 1000000)); -#if NANO_SECOND_ENABLED == 1 } else if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); -#endif } else { sprintf(buf + pos, ".%03d", (int)(val % 1000)); } @@ -3182,10 +3158,8 @@ int createDatabasesAndStables(char *command) { " fsync %d", g_Dbs.db[i].dbCfg.fsync); } if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) -#if NANO_SECOND_ENABLED == 1 || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) -#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, @@ -4264,10 +4238,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { g_Dbs.db[i].superTbls[j].iface= REST_IFACE; -#if STMT_IFACE_ENABLED == 1 } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; -#endif } else { errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", __func__, __LINE__, stbIface->valuestring); @@ -5076,7 +5048,6 @@ static void postFreeResource() { free(g_Dbs.db[i].superTbls[j].sampleDataBuf); g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; } -#if STMT_IFACE_ENABLED == 1 if (g_Dbs.db[i].superTbls[j].sampleBindArray) { for (int k = 0; k < MAX_SAMPLES_ONCE_FROM_FILE; k++) { uintptr_t *tmp = (uintptr_t *)(*(uintptr_t *)( @@ -5091,7 +5062,6 @@ static void postFreeResource() { } } tmfree((char *)g_Dbs.db[i].superTbls[j].sampleBindArray); -#endif if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { free(g_Dbs.db[i].superTbls[j].tagDataBuf); @@ -5384,7 +5354,6 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) } break; -#if STMT_IFACE_ENABLED == 1 case STMT_IFACE: debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); @@ -5397,7 +5366,6 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) } affectedRows = k; break; -#endif default: errorPrint("%s() LN%d: unknown insert mode: %d\n", @@ -5770,7 +5738,6 @@ static int64_t generateInterlaceDataWithoutStb( return k; } -#if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType( TAOS_BIND *bind, char *dataType, int32_t dataLen, @@ -6605,7 +6572,6 @@ static int32_t prepareStbStmtWithSample( return k; } -#endif static int32_t generateStbProgressiveData( SSuperTable *stbInfo, @@ -6806,7 +6772,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 if (sourceRand) { generated = prepareStbStmtRand( pThreadInfo, @@ -6826,9 +6791,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTime, &(pThreadInfo->samplePos)); } -#else - generated = -1; -#endif } else { generated = generateStbInterlaceData( pThreadInfo, @@ -6846,16 +6808,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, tableName, batchPerTbl, startTime); -#if STMT_IFACE_ENABLED == 1 generated = prepareStmtWithoutStb( pThreadInfo, tableName, batchPerTbl, insertRows, i, startTime); -#else - generated = -1; -#endif } else { generated = generateInterlaceDataWithoutStb( tableName, batchPerTbl, @@ -7068,7 +7026,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 if (sourceRand) { generated = prepareStbStmtRand( pThreadInfo, @@ -7087,9 +7044,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { insertRows, i, start_time, &(pThreadInfo->samplePos)); } -#else - generated = -1; -#endif } else { generated = generateStbProgressiveData( stbInfo, @@ -7101,16 +7055,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { } } else { if (g_args.iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 generated = prepareStmtWithoutStb( pThreadInfo, tableName, g_args.num_of_RPR, insertRows, i, start_time); -#else - generated = -1; -#endif } else { generated = generateProgressiveDataWithoutStb( tableName, @@ -7330,7 +7280,6 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * return 0; } -#if STMT_IFACE_ENABLED == 1 static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) { stbInfo->sampleBindArray = calloc(1, sizeof(char *) * MAX_SAMPLES_ONCE_FROM_FILE); @@ -7401,7 +7350,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return 0; } -#endif static void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* stbInfo) { @@ -7412,10 +7360,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, timePrec = TSDB_TIME_PRECISION_MILLI; } else if (0 == strncasecmp(precision, "us", 2)) { timePrec = TSDB_TIME_PRECISION_MICRO; -#if NANO_SECOND_ENABLED == 1 } else if (0 == strncasecmp(precision, "ns", 2)) { timePrec = TSDB_TIME_PRECISION_NANO; -#endif } else { errorPrint("Not support precision: %s\n", precision); exit(EXIT_FAILURE); @@ -7558,7 +7504,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, memset(pids, 0, threads * sizeof(pthread_t)); memset(infos, 0, threads * sizeof(threadInfo)); -#if STMT_IFACE_ENABLED == 1 char *stmtBuffer = calloc(1, BUFFER_SIZE); assert(stmtBuffer); if ((g_args.iface == STMT_IFACE) @@ -7599,7 +7544,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, parseSampleFileToStmt(stbInfo, timePrec); } } -#endif for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; @@ -7627,7 +7571,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(EXIT_FAILURE); } -#if STMT_IFACE_ENABLED == 1 if ((g_args.iface == STMT_IFACE) || ((stbInfo) && (stbInfo->iface == STMT_IFACE))) { @@ -7655,7 +7598,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, } pThreadInfo->bind_ts = malloc(sizeof(int64_t)); } -#endif } else { pThreadInfo->taos = NULL; } @@ -7681,9 +7623,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } -#if STMT_IFACE_ENABLED == 1 free(stmtBuffer); -#endif for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); @@ -7698,12 +7638,10 @@ static void startMultiThreadInsertData(int threads, char* db_name, for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; -#if STMT_IFACE_ENABLED == 1 if (pThreadInfo->stmt) { taos_stmt_close(pThreadInfo->stmt); tmfree((char *)pThreadInfo->bind_ts); } -#endif tsem_destroy(&(pThreadInfo->lock_sem)); taos_close(pThreadInfo->taos); From 9c04da10a0046d2ed72e55467e4a354bd8372878 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 20 Aug 2021 09:43:36 +0800 Subject: [PATCH 11/23] block union for nested query --- src/client/src/tscSQLParser.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 30a46ccc48..f1349d7d41 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -888,6 +888,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SELECT: { + const char * msg1 = "no nested query supported in union clause"; code = loadAllTableMeta(pSql, pInfo); if (code != TSDB_CODE_SUCCESS) { return code; @@ -901,6 +902,10 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); + if (size > 1 && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + // normalizeSqlNode(pSqlNode); // normalize the column name in each function if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { return code; From 1b5798431167177d2196204eec4a1757e7a63439 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 20 Aug 2021 10:15:15 +0800 Subject: [PATCH 12/23] Hotfix/sangshuduo/td 5844 cmdline parameters align for master (#7479) * [TD-5844]: make cmd line parameter similar. * fix test case align with taosdemo change. * fix windows stack overflow issue. * fix mac compile error. * fix taosdemo cmdline parameter in tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py * make taos.exe use mysql style password input. * make taos shell and taosdump use mysql style password input. * determine scanf return value. * cherry pick from develop/feature branch. * cherry pick with ab6cde02626bd8ba496b98a65e37e1f98c9ff596 * cherry pick with d6bfe443fd21ded247bd8509fb77b7b18e112357 Co-authored-by: Shuduo Sang --- src/inc/taosdef.h | 2 ++ src/kit/shell/inc/shell.h | 1 - src/kit/shell/src/shellDarwin.c | 14 +++++++++---- src/kit/shell/src/shellLinux.c | 23 ++++++++++++--------- src/kit/shell/src/shellWindows.c | 14 +++++++++---- src/kit/taosdemo/taosdemo.c | 23 +++++++++++---------- src/kit/taosdump/taosdump.c | 34 +++++++++++++++++++++----------- 7 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 94be247b0d..bc4ddbe067 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -87,6 +87,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_PASS "taosdata" #endif +#define SHELL_MAX_PASSWORD_LEN 20 + #define TSDB_TRUE 1 #define TSDB_FALSE 0 #define TSDB_OK 0 diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 2374150c52..c9d58fd3e1 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -25,7 +25,6 @@ #define MAX_USERNAME_SIZE 64 #define MAX_DBNAME_SIZE 64 #define MAX_IP_SIZE 20 -#define MAX_PASSWORD_SIZE 20 #define MAX_HISTORY_SIZE 1000 #define MAX_COMMAND_SIZE 1048586 #define HISTORY_FILE ".taos_history" diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 5ca4537aeb..9161860f07 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -66,7 +66,7 @@ void printHelp() { char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { wordexp_t full_path; @@ -81,19 +81,25 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strncmp(argv[i], "-p", 2) == 0) { + else if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { strcpy(tsOsName, "Darwin"); printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%s", g_password) > 1) { fprintf(stderr, "password read error\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } arguments->password = g_password; + strcpy(argv[i], ""); + argc -= 1; } // for management port else if (strcmp(argv[i], "-P") == 0) { diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index d051d3535e..b177e22e86 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -47,8 +47,8 @@ static struct argp_option options[] = { {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"check", 'k', "CHECK", 0, "Check tables."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, - {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, + {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; @@ -74,7 +74,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } break; - case 't': + case 'z': arguments->timezone = arg; break; case 'u': @@ -160,22 +160,27 @@ static struct argp argp = {options, parse_opt, args_doc, doc}; char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; -static void parse_password( +static void parse_args( int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { - if (strncmp(argv[i], "-p", 2) == 0) { + if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { strcpy(tsOsName, "Linux"); printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%20s", g_password) > 1) { fprintf(stderr, "password reading error\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); } arguments->password = g_password; arguments->is_use_passwd = true; @@ -190,7 +195,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { argp_program_version = verType; if (argc > 1) { - parse_password(argc, argv, arguments); + parse_args(argc, argv, arguments); } argp_parse(&argp, argc, argv, 0, 0, arguments); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index bf9afe4b80..d7f96631b6 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -64,7 +64,7 @@ void printHelp() { exit(EXIT_SUCCESS); } -char g_password[MAX_PASSWORD_SIZE]; +char g_password[SHELL_MAX_PASSWORD_LEN]; void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { @@ -78,20 +78,26 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strncmp(argv[i], "-p", 2) == 0) { + else if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { arguments->is_use_passwd = true; strcpy(tsOsName, "Windows"); printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if (strlen(argv[i]) == 2) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); + taosSetConsoleEcho(false); if (scanf("%s", g_password) > 1) { fprintf(stderr, "password read error!\n"); } + taosSetConsoleEcho(true); getchar(); } else { - tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } arguments->password = g_password; + strcpy(argv[i], ""); + argc -= 1; } // for management port else if (strcmp(argv[i], "-P") == 0) { diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 343c3f2e03..75da9d9f4b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -69,7 +69,6 @@ extern char configDir[]; #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) #define MAX_USERNAME_SIZE 64 -#define MAX_PASSWORD_SIZE 16 #define MAX_HOSTNAME_SIZE 253 // https://man7.org/linux/man-pages/man7/hostname.7.html #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space @@ -208,7 +207,7 @@ typedef struct SArguments_S { uint16_t port; uint16_t iface; char * user; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char * database; int replica; char * tb_prefix; @@ -356,7 +355,7 @@ typedef struct SDbs_S { uint16_t port; char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char resultFile[MAX_FILE_NAME_LEN]; bool use_metric; bool insert_only; @@ -422,7 +421,7 @@ typedef struct SQueryMetaInfo_S { uint16_t port; struct sockaddr_in serv_addr; char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; char dbName[TSDB_DB_NAME_LEN]; char queryMode[SMALL_BUFF_LEN]; // taosc, rest @@ -858,7 +857,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } taosSetConsoleEcho(true); } else { - tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(arguments->password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } } else if (strcmp(argv[i], "-o") == 0) { if (argc == i+1) { @@ -3780,9 +3779,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, "taosdata", SHELL_MAX_PASSWORD_LEN); } cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); @@ -4516,9 +4515,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); + tstrncpy(g_queryInfo.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; + tstrncpy(g_queryInfo.password, "taosdata", SHELL_MAX_PASSWORD_LEN);; } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, @@ -8826,7 +8825,7 @@ static void initOfInsertMeta() { tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_Dbs.port = 6030; tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); g_Dbs.threadCount = 2; g_Dbs.use_metric = g_args.use_metric; @@ -8839,7 +8838,7 @@ static void initOfQueryMeta() { tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); g_queryInfo.port = 6030; tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); } static void setParaFromArg() { @@ -8853,7 +8852,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); } - tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); + tstrncpy(g_Dbs.password, g_args.password, SHELL_MAX_PASSWORD_LEN); if (g_args.port) { g_Dbs.port = g_args.port; diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index c54b8da1b7..6b553b3824 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -243,19 +243,15 @@ static struct argp_option options[] = { {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, {"debug", 'g', 0, 0, "Print debug info.", 8}, - {"verbose", 'b', 0, 0, "Print verbose debug info.", 9}, - {"performanceprint", 'm', 0, 0, "Print performance debug info.", 10}, {0} }; -#define MAX_PASSWORD_SIZE 20 - /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option char *host; char *user; - char password[MAX_PASSWORD_SIZE]; + char password[SHELL_MAX_PASSWORD_LEN]; uint16_t port; char cversion[12]; uint16_t mysqlFlag; @@ -432,7 +428,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; // dump unit option case 'A': - g_args.all_databases = true; break; case 'D': g_args.databases = true; @@ -555,11 +550,14 @@ static void parse_precision_first( } } -static void parse_password( +static void parse_args( int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { - if (strncmp(argv[i], "-p", 2) == 0) { - if (strlen(argv[i]) == 2) { + if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); taosSetConsoleEcho(false); if(scanf("%20s", arguments->password) > 1) { @@ -567,10 +565,22 @@ static void parse_password( } taosSetConsoleEcho(true); } else { - tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + tstrncpy(arguments->password, (char *)(argv[i] + 2), + SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); } - argv[i] = ""; + } else if (strcmp(argv[i], "-gg") == 0) { + arguments->verbose_print = true; + strcpy(argv[i], ""); + } else if (strcmp(argv[i], "-PP") == 0) { + arguments->performance_print = true; + strcpy(argv[i], ""); + } else if (strcmp(argv[i], "-A") == 0) { + g_args.all_databases = true; + } else { + continue; } + } } @@ -639,7 +649,7 @@ int main(int argc, char *argv[]) { if (argc > 1) { parse_precision_first(argc, argv, &g_args); parse_timestamp(argc, argv, &g_args); - parse_password(argc, argv, &g_args); + parse_args(argc, argv, &g_args); } argp_parse(&argp, argc, argv, 0, 0, &g_args); From 8ae068ff2e2686b179efd5fdd8da2a276fd532fe Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Aug 2021 10:25:03 +0800 Subject: [PATCH 13/23] [TD-6046] fix bottom,top,diff,derivative ts bugs --- src/client/src/tscGlobalmerge.c | 2 +- src/client/src/tscSQLParser.c | 7 ++- src/query/inc/qExecutor.h | 2 + src/query/src/qExecutor.c | 44 ++++++++++++++++--- tests/pytest/functions/function_bottom.py | 15 +++++++ tests/pytest/functions/function_derivative.py | 22 ++++++++++ tests/pytest/functions/function_diff.py | 17 +++++++ tests/pytest/functions/function_top.py | 15 +++++++ 8 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index e696d54abd..2b635e8d83 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -643,7 +643,7 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD for(int32_t j = 0; j < numOfExpr; ++j) { pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows); if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) { - pCtx[j].ptsOutputBuf = pCtx[0].pOutput; + if(j>0) pCtx[j].ptsOutputBuf = pCtx[j-1].pOutput; } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 30a46ccc48..a13ec08408 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2603,13 +2603,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // set the first column ts for diff query if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); SColumnList ids = createColumnList(1, 0, 0); - insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); @@ -2882,7 +2881,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -5693,7 +5692,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2); + return invalidOperationMsg(pMsgBuf, msg2);qExecutor.h } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index e8b7855769..2b9f952d76 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -597,6 +597,8 @@ bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilter void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); + void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 844b8ec683..26ae714c27 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3616,7 +3616,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i // set the timestamp output buffer for top/bottom/diff query int32_t fid = pCtx[i].functionId; if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } } @@ -3651,7 +3651,37 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + if(i>0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + } + } +} + +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + bool needCopyTs = false; + int32_t tsNum = 0; + char *src = NULL; + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { + needCopyTs = true; + if (i > 0 && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){ + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data + src = pColRes->pData; + } + }else if(functionId == TSDB_FUNC_TS_DUMMY) { + tsNum++; + } + } + + if (!needCopyTs) return; + if (tsNum < 2) return; + if (src == NULL) return; + + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if(functionId == TSDB_FUNC_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); } } } @@ -3851,7 +3881,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } if (!pResInfo->initialized) { @@ -3912,7 +3942,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } /* @@ -5698,6 +5728,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return pRes; } @@ -5723,8 +5754,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { if (*newgroup) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return pInfo->pRes; + break; } else { // init output buffer for a new group data for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); @@ -5754,7 +5784,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { break; } } - + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index abb9ac48e7..e9e5003f6f 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -104,6 +104,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 2) + + tdSql.query("select ts,bottom(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") + + + tdSql.query("select ts,bottom(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") #TD-2457 bottom + interval + order by tdSql.error('select top(col2,1) from test interval(1y) order by col2;') diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 9d60129672..d4d8ab80a6 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -54,6 +54,28 @@ class TDTestCase: tdSql.query("select derivative(col, 10s, 0) from stb group by tbname") tdSql.checkRows(10) + tdSql.query("select ts,derivative(col, 10s, 1),ts from stb group by tbname") + tdSql.checkRows(4) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(3, 0, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") + + tdSql.query("select ts,derivative(col, 10s, 1),ts from tb1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 1, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:20.009") + + tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)") + + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname") tdSql.query("select derivative(col, 10s, 1) from tb1") diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py index fba3b4c0d4..cd5894c165 100644 --- a/tests/pytest/functions/function_diff.py +++ b/tests/pytest/functions/function_diff.py @@ -94,6 +94,23 @@ class TDTestCase: tdSql.error("select diff(col13) from test") tdSql.error("select diff(col14) from test") + tdSql.query("select ts,diff(col1),ts from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + + tdSql.query("select ts,diff(col1),ts from test group by tbname") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index f8318402b5..9824afc19f 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -117,6 +117,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + tdSql.query("select ts,top(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") + + + tdSql.query("select ts,top(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") #TD-2563 top + super_table + interval tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") From 8d9ed8e7f9ba6437fb2dfae8bd971d04d1481a74 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 20 Aug 2021 10:46:21 +0800 Subject: [PATCH 14/23] [TD-6234]: add unit test case for TD-6011 --- tests/pytest/query/filterNoKeyword.py | 83 +++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/pytest/query/filterNoKeyword.py diff --git a/tests/pytest/query/filterNoKeyword.py b/tests/pytest/query/filterNoKeyword.py new file mode 100644 index 0000000000..34d74efd82 --- /dev/null +++ b/tests/pytest/query/filterNoKeyword.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("======= Verify filter for bool, nchar and binary type =========") + tdLog.debug( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + tdSql.execute( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + + tdSql.execute("create table st1 using st tags(true, 'table1', '水表')") + for i in range(1, 6): + tdSql.execute( + "insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f)" % + (self.ts + i, i % + 2, i, i, + i, i, i, i, 1.0, 1.0)) + + # =============Data type keywords cannot be used in filter==================== + # timestamp + tdSql.error("select * from st where timestamp = 1629417600") + + # bool + tdSql.error("select * from st where bool = false") + + #binary + tdSql.error("select * from st where binary = 'taosdata'") + + # nchar + tdSql.error("select * from st where nchar = '涛思数据'") + + # tinyint + tdSql.error("select * from st where tinyint = 127") + + # smallint + tdSql.error("select * from st where smallint = 32767") + + # int + tdSql.error("select * from st where INTEGER = 2147483647") + tdSql.error("select * from st where int = 2147483647") + + # bigint + tdSql.error("select * from st where bigint = 2147483647") + + # float + tdSql.error("select * from st where float = 3.4E38") + + # double + tdSql.error("select * from st where double = 1.7E308") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8f8e86bb8aec06a4f4d70ef048d9d0f8a4ea0fd2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Aug 2021 10:52:46 +0800 Subject: [PATCH 15/23] [TD-6046] fix ts top/bottom/diff/derivative ts error --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a13ec08408..1f62dc8afa 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5692,7 +5692,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2);qExecutor.h + return invalidOperationMsg(pMsgBuf, msg2); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); From af7517b35544d6016ad8d36702723f398a0a1ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=A1=E6=AF=94=E5=8D=A1=E6=AF=94?= <30525741+jackwener@users.noreply.github.com> Date: Fri, 20 Aug 2021 15:35:54 +0800 Subject: [PATCH 16/23] Update docs.md --- documentation20/cn/01.evaluation/docs.md | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 4aba82a54c..9a9669645c 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -30,38 +30,38 @@ TDengine 的模块之一是时序数据库。但除此之外,为减少研发 |数据源特点和需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|总体数据量巨大| | | √ |TDengine在容量方面提供出色的水平扩展功能,并且具备匹配高压缩的存储结构,达到业界最优的存储效率。| -|数据输入速度偶尔或者持续巨大| | | √ | TDengine的性能大大超过同类产品,可以在同样的硬件环境下持续处理大量的输入数据,并且提供很容易在用户环境里面运行的性能评估工具。| -|数据源数目巨大| | | √ |TDengine设计中包含专门针对大量数据源的优化,包括数据的写入和查询,尤其适合高效处理海量(千万或者更多量级)的数据源。| +|总体数据量巨大| | | √ | TDengine 在容量方面提供出色的水平扩展功能,并且具备匹配高压缩的存储结构,达到业界最优的存储效率。| +|数据输入速度偶尔或者持续巨大| | | √ | TDengine 的性能大大超过同类产品,可以在同样的硬件环境下持续处理大量的输入数据,并且提供很容易在用户环境里面运行的性能评估工具。| +|数据源数目巨大| | | √ | TDengine 设计中包含专门针对大量数据源的优化,包括数据的写入和查询,尤其适合高效处理海量(千万或者更多量级)的数据源。| ### 系统架构要求 |系统架构要求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求简单可靠的系统架构| | | √ |TDengine的系统架构非常简单可靠,自带消息队列,缓存,流式计算,监控等功能,无需集成额外的第三方产品。| -|要求容错和高可靠| | | √ |TDengine的集群功能,自动提供容错灾备等高可靠功能。| -|标准化规范| | | √ |TDengine使用标准的SQL语言提供主要功能,遵守标准化规范。| +|要求简单可靠的系统架构| | | √ | TDengine 的系统架构非常简单可靠,自带消息队列,缓存,流式计算,监控等功能,无需集成额外的第三方产品。| +|要求容错和高可靠| | | √ | TDengine 的集群功能,自动提供容错灾备等高可靠功能。| +|标准化规范| | | √ | TDengine 使用标准的SQL语言提供主要功能,遵守标准化规范。| ### 系统功能需求 |系统功能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求完整的内置数据处理算法| | √ | |TDengine的实现了通用的数据处理算法,但是还没有做到妥善处理各行各业的所有要求,因此特殊类型的处理还需要应用层面处理。| -|需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理,或者应该考虑TDengine和关系型数据系统配合实现系统功能。| +|要求完整的内置数据处理算法| | √ | | TDengine 的实现了通用的数据处理算法,但是还没有做到妥善处理各行各业的所有要求,因此特殊类型的处理还需要应用层面处理。| +|需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理,或者应该考虑 TDengine 和关系型数据系统配合实现系统功能。| ### 系统性能需求 |系统性能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求较大的总体处理能力| | | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| -|要求高速处理数据 | | | √ |TDengine的专门为IOT优化的存储和数据处理的设计,一般可以让系统得到超出同类产品多倍数的处理速度提升。| -|要求快速处理小粒度数据| | | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| +|要求较大的总体处理能力| | | √ | TDengine 的集群功能可以轻松地让多服务器配合达成处理能力的提升。| +|要求高速处理数据 | | | √ | TDengine 的专门为 IOT 优化的存储和数据处理的设计,一般可以让系统得到超出同类产品多倍数的处理速度提升。| +|要求快速处理小粒度数据| | | √ |这方面 TDengine 性能可以完全对标关系型和 NoSQL 型数据处理系统。| ### 系统维护需求 |系统维护需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求系统可靠运行| | | √ |TDengine的系统架构非常稳定可靠,日常维护也简单便捷,对维护人员的要求简洁明了,最大程度上杜绝人为错误和事故。| +|要求系统可靠运行| | | √ | TDengine 的系统架构非常稳定可靠,日常维护也简单便捷,对维护人员的要求简洁明了,最大程度上杜绝人为错误和事故。| |要求运维学习成本可控| | | √ |同上。| -|要求市场有大量人才储备| √ | | |TDengine作为新一代产品,目前人才市场里面有经验的人员还有限。但是学习成本低,我们作为厂家也提供运维的培训和辅助服务。| +|要求市场有大量人才储备| √ | | | TDengine 作为新一代产品,目前人才市场里面有经验的人员还有限。但是学习成本低,我们作为厂家也提供运维的培训和辅助服务。| From adf1e47032e4ab685087baa16777bd04ed124dae Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 20 Aug 2021 16:08:30 +0800 Subject: [PATCH 17/23] In some emphasized text, spaces are also added between the English and Chinese text. --- documentation20/cn/01.evaluation/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/01.evaluation/docs.md b/documentation20/cn/01.evaluation/docs.md index 9a9669645c..4263391dbc 100644 --- a/documentation20/cn/01.evaluation/docs.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -6,8 +6,8 @@ TDengine 是涛思数据面对高速增长的物联网大数据市场和技术 TDengine 的模块之一是时序数据库。但除此之外,为减少研发的复杂度、系统维护的难度,TDengine 还提供缓存、消息队列、订阅、流式计算等功能,为物联网、工业互联网大数据的处理提供全栈的技术方案,是一个高效易用的物联网大数据平台。与 Hadoop 等典型的大数据平台相比,它具有如下鲜明的特点: -* __10倍以上的性能提升__:定义了创新的数据存储结构,单核每秒能处理至少 2 万次请求,插入数百万个数据点,读出一千万以上数据点,比现有通用数据库快十倍以上。 -* __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的 1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的 1/10。 +* __10 倍以上的性能提升__:定义了创新的数据存储结构,单核每秒能处理至少 2 万次请求,插入数百万个数据点,读出一千万以上数据点,比现有通用数据库快十倍以上。 +* __硬件或云服务成本降至 1/5__:由于超强性能,计算资源不到通用大数据方案的 1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的 1/10。 * __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成 Kafka/Redis/HBase/Spark/HDFS 等软件,大幅降低应用开发和维护的复杂度成本。 * __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过 Shell, Python, R, MATLAB 随时进行。 * __与第三方工具无缝连接__:不用一行代码,即可与 Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R 等集成。后续将支持 OPC, Hadoop, Spark 等,BI 工具也将无缝连接。 From c26d34a6847c483db6769c1f27ffea1ba09f19f2 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 20 Aug 2021 18:15:59 +0800 Subject: [PATCH 18/23] fix bug --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f1349d7d41..c6614d0054 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -902,7 +902,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); - if (size > 1 && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } From 1733d74619ec55465f8b6402a6917eef78b8486b Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 Aug 2021 18:45:15 +0800 Subject: [PATCH 19/23] [TD-6068] add test case for TD-6068 --- tests/pytest/functions/queryTestCases.py | 123 ++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py index f320db43af..8089bcccdc 100644 --- a/tests/pytest/functions/queryTestCases.py +++ b/tests/pytest/functions/queryTestCases.py @@ -884,6 +884,126 @@ class TDTestCase: pass + def td6068(self): + tdLog.printNoPrefix("==========TD-6068==========") + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("use db") + + tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool) tags(t1 int)") + + for i in range(100): + sql = f"create table db.t{i} using db.stb1 tags({i})" + tdSql.execute(sql) + tdSql.execute(f"insert into db.t{i} values (now-10h, {i}, {i+random.random()}, now-10h, 'a_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-9h, {i+random.randint(1,10)}, {i+random.random()}, now-9h, 'a_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} values (now-8h, {i+random.randint(1,10)}, {i+random.random()}, now-8h, 'b_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-7h, {i+random.randint(1,10)}, {i+random.random()}, now-7h, 'b_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} values (now-6h, {i+random.randint(1,10)}, {i+random.random()}, now-6h, 'c_{i}', '{i-random.random()}', True)") + tdSql.execute(f"insert into db.t{i} values (now-5h, {i+random.randint(1,10)}, {i+random.random()}, now-5h, 'c_{i}', '{i-random.random()}', FALSE )") + tdSql.execute(f"insert into db.t{i} (ts)values (now-4h)") + tdSql.execute(f"insert into db.t{i} (ts)values (now-11h)") + tdSql.execute(f"insert into db.t{i} (ts)values (now-450m)") + + tdSql.query("select ts as t,derivative(c1, 10m, 0) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, derivative(c1, 1h, 0) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.checkCols(4) + tdSql.query("select ts as t, derivative(c1, 1s, 0) from t1") + tdSql.query("select ts as t, derivative(c1, 1d, 0) from t1") + tdSql.error("select ts as t, derivative(c1, 1h, 0) from stb1") + tdSql.query("select ts as t, derivative(c2, 1h, 0) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, derivative(c3, 1h, 0) from t1") + tdSql.error("select ts as t, derivative(c4, 1h, 0) from t1") + tdSql.query("select ts as t, derivative(c5, 1h, 0) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, derivative(c6, 1h, 0) from t1") + tdSql.error("select ts as t, derivative(t1, 1h, 0) from t1") + + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, diff(c1) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.checkCols(4) + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.query("select ts as t, diff(c1) from t1") + tdSql.error("select ts as t, diff(c1) from stb1") + tdSql.query("select ts as t, diff(c2) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, diff(c3) from t1") + tdSql.error("select ts as t, diff(c4) from t1") + tdSql.query("select ts as t, diff(c5) from t1") + tdSql.checkRows(5) + tdSql.error("select ts as t, diff(c6) from t1") + tdSql.error("select ts as t, diff(t1) from t1") + tdSql.error("select ts as t, diff(c1, c2) from t1") + + tdSql.error("select ts as t, bottom(c1, 0) from t1") + tdSql.query("select ts as t, bottom(c1, 5) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, bottom(c1, 5) from stb1") + tdSql.checkRows(5) + tdSql.query("select ts as t, bottom(c1, 5) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.query("select ts as t, bottom(c1, 8) from t1") + tdSql.checkRows(6) + tdSql.query("select ts as t, bottom(c2, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, bottom(c3, 5) from t1") + tdSql.error("select ts as t, bottom(c4, 5) from t1") + tdSql.query("select ts as t, bottom(c5, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, bottom(c6, 5) from t1") + tdSql.error("select ts as t, bottom(c5, 8) as b from t1 order by b") + tdSql.error("select ts as t, bottom(t1, 1) from t1") + tdSql.error("select ts as t, bottom(t1, 1) from stb1") + tdSql.error("select ts as t, bottom(t1, 3) from stb1 order by c3") + tdSql.error("select ts as t, bottom(t1, 3) from t1 order by c3") + + + tdSql.error("select ts as t, top(c1, 0) from t1") + tdSql.query("select ts as t, top(c1, 5) from t1") + tdSql.checkRows(5) + tdSql.checkCols(3) + for i in range(5): + data=tdSql.getData(i, 0) + tdSql.checkData(i, 1, data) + tdSql.query("select ts as t, top(c1, 5) from stb1") + tdSql.checkRows(5) + tdSql.query("select ts as t, top(c1, 5) from stb1 group by tbname") + tdSql.checkRows(500) + tdSql.query("select ts as t, top(c1, 8) from t1") + tdSql.checkRows(6) + tdSql.query("select ts as t, top(c2, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, top(c3, 5) from t1") + tdSql.error("select ts as t, top(c4, 5) from t1") + tdSql.query("select ts as t, top(c5, 8) from t1") + tdSql.checkRows(6) + tdSql.error("select ts as t, top(c6, 5) from t1") + tdSql.error("select ts as t, top(c5, 8) as b from t1 order by b") + tdSql.error("select ts as t, top(t1, 1) from t1") + tdSql.error("select ts as t, top(t1, 1) from stb1") + tdSql.error("select ts as t, top(t1, 3) from stb1 order by c3") + tdSql.error("select ts as t, top(t1, 3) from t1 order by c3") + + pass + + def run(self): # master branch @@ -891,8 +1011,9 @@ class TDTestCase: # self.td4082() # self.td4288() # self.td4724() - self.td5798() + # self.td5798() # self.td5935() + self.td6068() # develop branch # self.td4097() From de12a8ef467994a4178434bd0007c2e42118c129 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 21 Aug 2021 16:21:26 +0800 Subject: [PATCH 20/23] [TD-6251]: taosdemo error msg with datetime info. (#7501) --- src/kit/taosdemo/taosdemo.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 75da9d9f4b..6099ac9803 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -659,7 +659,21 @@ static FILE * g_fpOfInsertResult = NULL; fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0) #define errorPrint(fmt, ...) \ - do { fprintf(stderr, " \033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, " \033[0m"); } while(0) + do {\ + struct tm Tm, *ptm;\ + struct timeval timeSecs; \ + time_t curTime;\ + gettimeofday(&timeSecs, NULL); \ + curTime = timeSecs.tv_sec;\ + ptm = localtime_r(&curTime, &Tm);\ + fprintf(stderr, " \033[31m");\ + fprintf(stderr, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ",\ + ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,\ + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec,\ + taosGetSelfPthreadId());\ + fprintf(stderr, "ERROR: "fmt, __VA_ARGS__);\ + fprintf(stderr, " \033[0m");\ + } while(0) // for strncpy buffer overflow #define min(a, b) (((a) < (b)) ? (a) : (b)) From ffcb8f6a4a785bd2b4c36846b51a0a928056be22 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 21 Aug 2021 21:58:17 +0800 Subject: [PATCH 21/23] [TD-6253]: taosdump cmdline param verification. (#7506) --- src/kit/taosdump/taosdump.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 6b553b3824..30b5d91b10 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -62,6 +62,20 @@ typedef struct { #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, "\033[0m"); } while(0) +static bool isStringNumber(char *input) +{ + int len = strlen(input); + if (0 == len) { + return false; + } + + for (int i = 0; i < len; i++) { + if (!isdigit(input[i])) + return false; + } + + return true; +} // -------------------------- SHOW DATABASE INTERFACE----------------------- enum _show_db_index { @@ -472,6 +486,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { g_args.table_batch = atoi(arg); break; case 'T': + if (!isStringNumber(arg)) { + errorPrint("%s", "\n\t-T need a number following!\n"); + exit(EXIT_FAILURE); + } g_args.thread_num = atoi(arg); break; case OPT_ABORT: From 9aa505f7df04ab543bad81748becf5a27be77995 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 22 Aug 2021 21:49:47 +0800 Subject: [PATCH 22/23] Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#7510) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. * fix resource leak reported by coverity scan. * fix taosdemo coverity scan issue. * fix tcsetattr and getchar return value determination bug. Co-authored-by: Shuduo Sang --- src/kit/shell/src/shellLinux.c | 4 +++- src/os/src/linux/osSystem.c | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index b177e22e86..859542cde9 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -177,7 +177,9 @@ static void parse_args( fprintf(stderr, "password reading error\n"); } taosSetConsoleEcho(true); - getchar(); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } } else { tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); strcpy(argv[i], "-p"); diff --git a/src/os/src/linux/osSystem.c b/src/os/src/linux/osSystem.c index 0cdb20dbdb..a82149dccb 100644 --- a/src/os/src/linux/osSystem.c +++ b/src/os/src/linux/osSystem.c @@ -63,12 +63,12 @@ int taosSetConsoleEcho(bool on) } if (on) - term.c_lflag|=ECHOFLAGS; + term.c_lflag |= ECHOFLAGS; else - term.c_lflag &=~ECHOFLAGS; + term.c_lflag &= ~ECHOFLAGS; - err = tcsetattr(STDIN_FILENO,TCSAFLUSH,&term); - if (err == -1 && err == EINTR) { + err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); + if (err == -1 || err == EINTR) { perror("Cannot set the attribution of the terminal"); return -1; } From 91ab5f22d4f77f77a7b74cb2fda328efed2309d4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 23 Aug 2021 15:04:26 +0800 Subject: [PATCH 23/23] [TD-6194]: taosdemo wrong data (#7517) --- src/kit/taosdemo/taosdemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6099ac9803..f222266ee8 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5216,7 +5216,8 @@ static int64_t generateStbRowData( tmpLen = strlen(tmp); tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, INT_BUFF_LEN)); } else { - errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); + errorPrint( "Not support data type: %s\n", + stbInfo->columns[i].dataType); return -1; } @@ -5229,8 +5230,7 @@ static int64_t generateStbRowData( return 0; } - dataLen -= 1; - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); + tstrncpy(pstr + dataLen - 1, ")", 2); verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen); verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);