From 3626a2de181af79b9bf13adce6102b0ad7941e8f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 23 Jun 2021 10:12:45 +0800 Subject: [PATCH 01/50] update docker cluster script --- .../OneMnodeMultipleVnodesTest.py | 5 ++-- tests/pytest/dockerCluster/basic.py | 27 ++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py b/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py index ee663f89b0..43e281f437 100644 --- a/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py +++ b/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py @@ -12,9 +12,6 @@ # -*- coding: utf-8 -*- from basic import * -from util.sql import tdSql - - class TDTestCase: @@ -36,4 +33,6 @@ td = TDTestCase() td.init() +## usage: python3 OneMnodeMultipleVnodesTest.py + diff --git a/tests/pytest/dockerCluster/basic.py b/tests/pytest/dockerCluster/basic.py index 50914b0be9..871d69790d 100644 --- a/tests/pytest/dockerCluster/basic.py +++ b/tests/pytest/dockerCluster/basic.py @@ -44,7 +44,16 @@ class BuildDockerCluser: "jnidebugFlag":"135", "qdebugFlag":"135", "maxSQLLength":"1048576" - } + } + cmd = "mkdir -p %s" % self.dockerDir + self.execCmd(cmd) + + cmd = "cp *.yml %s" % self.dockerDir + self.execCmd(cmd) + + cmd = "cp Dockerfile %s" % self.dockerDir + self.execCmd(cmd) + # execute command, and return the output # ref: https://blog.csdn.net/wowocpp/article/details/80775650 @@ -81,7 +90,7 @@ class BuildDockerCluser: def removeFile(self, rootDir, index, dir): cmd = "rm -rf %s/node%d/%s/*" % (rootDir, index, dir) self.execCmd(cmd) - + def clearEnv(self): cmd = "cd %s && docker-compose down --remove-orphans" % self.dockerDir self.execCmd(cmd) @@ -108,10 +117,14 @@ class BuildDockerCluser: self.execCmd(cmd) def updateLocalhosts(self): - cmd = "grep '172.27.0.7 *tdnode1' /etc/hosts" + cmd = "grep '172.27.0.7 *tdnode1' /etc/hosts | sed 's: ::g'" result = self.execCmdAndGetOutput(cmd) - if result and not result.isspace(): + print(result) + if result is None or result.isspace(): + print("==========") cmd = "echo '172.27.0.7 tdnode1' >> /etc/hosts" + display = "echo %s" % cmd + self.execCmd(display) self.execCmd(cmd) def deploy(self): @@ -138,13 +151,13 @@ class BuildDockerCluser: if self.numOfNodes < 2 or self.numOfNodes > 10: print("the number of nodes must be between 2 and 10") exit(0) - self.clearEnv() - self.createDirs() self.updateLocalhosts() self.deploy() def run(self): - cmd = "./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.getTaosdVersion(), self.dockerDir) + cmd = "./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.getTaosdVersion(), self.dockerDir) + display = "echo %s" % cmd + self.execCmd(display) self.execCmd(cmd) self.getConnection() self.createDondes() From d02ce1e2dfb6a885afef0b13b0b601bc75cb7e75 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 15:23:33 +0800 Subject: [PATCH 02/50] Hotfix/sangshuduo/td 4823 taosdemo gettablename (#6606) * [TD-4823]: taosdemo getTableName return empty. * fix typo. * check table name is empty in early stage. --- src/kit/taosdemo/taosdemo.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6b12e66cb9..6314aa6a00 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2497,6 +2497,13 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* pTblName = childTblName; while((row = taos_fetch_row(res)) != NULL) { int32_t* len = taos_fetch_lengths(res); + + if (0 == strlen((char *)row[0])) { + errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", + __func__, __LINE__, count); + exit(-1); + } + tstrncpy(pTblName, (char *)row[0], len[0]+1); //printf("==== sub table name: %s\n", pTblName); count++; @@ -6296,16 +6303,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - // read sample data from file first - if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample")))) { - if (0 != prepareSampleDataForSTable(superTblInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", - __func__, __LINE__); - exit(-1); - } - } - TAOS* taos0 = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); From 44621d8a0f8b1da94bae86b9e3b145ab513baed9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 22:44:46 +0800 Subject: [PATCH 03/50] Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6614) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. --- src/kit/taosdemo/taosdemo.c | 127 ++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6314aa6a00..abf3c74436 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -79,10 +79,9 @@ enum TEST_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) -#define COND_BUF_LEN BUFFER_SIZE - 30 +#define COND_BUF_LEN (BUFFER_SIZE - 30) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 -#define MAX_DB_NAME_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*1024)+20 // max record len: 16*1024, timestamp string and ,('') need extra space @@ -90,7 +89,7 @@ enum TEST_MODE { #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -#define MAX_FILE_NAME_LEN 256 +#define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_DATATYPE 10 @@ -195,13 +194,6 @@ enum _describe_table_index { TSDB_MAX_DESCRIBE_METRIC }; -typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[128]; -} SColDes; - /* Used by main to communicate with parse_opt. */ static char *g_dupstr = NULL; @@ -247,16 +239,16 @@ typedef struct SArguments_S { } SArguments; typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN + 1]; - char dataType[MAX_TB_NAME_SIZE]; + char field[TSDB_COL_NAME_LEN]; + char dataType[16]; uint32_t dataLen; char note[128]; } StrColumn; typedef struct SSuperTable_S { - char sTblName[MAX_TB_NAME_SIZE+1]; - char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - char childTblPrefix[MAX_TB_NAME_SIZE]; + char sTblName[TSDB_TABLE_NAME_LEN]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest uint16_t childTblExists; int64_t childTblCount; @@ -277,8 +269,8 @@ typedef struct SSuperTable_S { int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN+1]; - char tagsFile[MAX_FILE_NAME_LEN+1]; + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; @@ -305,7 +297,7 @@ typedef struct SSuperTable_S { } SSuperTable; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; + char name[TSDB_DB_NAME_LEN]; char create_time[32]; int64_t ntables; int32_t vgroups; @@ -341,11 +333,11 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[MAX_TB_NAME_SIZE]; + char precision[8]; } SDbCfg; typedef struct SDataBase_S { - char dbName[MAX_DB_NAME_SIZE]; + char dbName[TSDB_DB_NAME_LEN]; bool drop; // 0: use exists, 1: if exists, drop then new create SDbCfg dbCfg; uint64_t superTblCount; @@ -353,14 +345,14 @@ typedef struct SDataBase_S { } SDataBase; typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; struct sockaddr_in serv_addr; uint16_t port; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char resultFile[MAX_FILE_NAME_LEN+1]; + char resultFile[MAX_FILE_NAME_LEN]; bool use_metric; bool insert_only; bool do_aggreFunc; @@ -387,7 +379,7 @@ typedef struct SpecifiedQueryInfo_S { bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; int endAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -398,7 +390,7 @@ typedef struct SpecifiedQueryInfo_S { } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { - char sTblName[MAX_TB_NAME_SIZE+1]; + char sTblName[TSDB_TABLE_NAME_LEN]; uint64_t queryInterval; // 0: unlimit > 0 loop/s uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async @@ -407,10 +399,10 @@ typedef struct SuperQueryInfo_S { int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; - char childTblPrefix[MAX_TB_NAME_SIZE]; + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq int sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume; int endAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -420,13 +412,13 @@ typedef struct SuperQueryInfo_S { } SuperQueryInfo; typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; uint16_t port; struct sockaddr_in serv_addr; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char dbName[MAX_DB_NAME_SIZE+1]; + char dbName[TSDB_DB_NAME_LEN]; char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; @@ -438,11 +430,11 @@ typedef struct SThreadInfo_S { TAOS * taos; TAOS_STMT *stmt; int threadID; - char db_name[MAX_DB_NAME_SIZE+1]; + char db_name[TSDB_DB_NAME_LEN]; uint32_t time_precision; char filePath[4096]; FILE *fp; - char tb_prefix[MAX_TB_NAME_SIZE]; + char tb_prefix[TSDB_TABLE_NAME_LEN]; uint64_t start_table_from; uint64_t end_table_to; int64_t ntables; @@ -608,7 +600,7 @@ SArguments g_args = { 1, // query_times 0, // interlace_rows; 30000, // num_of_RPR - (1024*1024), // max_sql_len + (1024*1024), // max_sql_len 10000, // num_of_tables 10000, // num_of_DPT 0, // abort @@ -3038,7 +3030,7 @@ static int startMultiThreadCreateChildTable( for (int64_t i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); pThreadInfo->taos = taos_connect( @@ -3329,7 +3321,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(column, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3344,7 +3336,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, MAX_TB_NAME_SIZE); + columnCase.dataType, strlen(columnCase.dataType) + 1); superTbls->columns[index].dataLen = columnCase.dataLen; index++; } @@ -3400,7 +3392,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3415,7 +3407,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - MAX_TB_NAME_SIZE); + strlen(columnCase.dataType) + 1); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3638,7 +3630,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, db name not found\n"); goto PARSE_OVER; } - tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { @@ -3659,10 +3651,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - MAX_DB_NAME_SIZE); + 8); } else if (!precision) { - //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, 8); } else { printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; @@ -3839,7 +3830,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { @@ -3847,7 +3838,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - MAX_DB_NAME_SIZE); + TSDB_TABLE_NAME_LEN - 20); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl @@ -3915,9 +3906,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, MAX_DB_NAME_SIZE); + dataSource->valuestring, TSDB_DB_NAME_LEN); } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); @@ -3975,10 +3966,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - ts->valuestring, MAX_DB_NAME_SIZE); + ts->valuestring, TSDB_DB_NAME_LEN); } else if (!ts) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - "now", MAX_DB_NAME_SIZE); + "now", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, start_timestamp not found\n"); goto PARSE_OVER; @@ -3998,9 +3989,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, MAX_DB_NAME_SIZE); + sampleFormat->valuestring, TSDB_DB_NAME_LEN); } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; @@ -4245,7 +4236,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { - tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); } else if (!dbs) { printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; @@ -4495,7 +4486,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, super table name input error\n", __func__, __LINE__); @@ -6408,7 +6399,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->time_precision = timePrec; pThreadInfo->superTblInfo = superTblInfo; @@ -6852,7 +6843,7 @@ static void *specifiedTableQuery(void *sarg) { } } - char sqlStr[MAX_DB_NAME_SIZE + 5]; + char sqlStr[TSDB_DB_NAME_LEN + 5]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); @@ -7328,12 +7319,6 @@ static void *superSubscribe(void *sarg) { performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); if (res) { - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->filePath, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - fetchResult(res, pThreadInfo); - } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], @@ -7440,10 +7425,10 @@ static void *specifiedSubscribe(void *sarg) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo); } + fetchResult( + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], + pThreadInfo); g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) @@ -7680,9 +7665,9 @@ static void setParaFromArg(){ g_Dbs.dbCount = 1; g_Dbs.db[0].drop = true; - tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); @@ -7704,7 +7689,7 @@ static void setParaFromArg(){ if (g_args.use_metric) { g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; @@ -7715,7 +7700,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, MAX_TB_NAME_SIZE); + g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].iface = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, @@ -7732,7 +7717,7 @@ static void setParaFromArg(){ } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], MAX_TB_NAME_SIZE); + data_type[i], strlen(data_type[i]) + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -7743,18 +7728,18 @@ static void setParaFromArg(){ for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", MAX_TB_NAME_SIZE); + "BINARY", strlen("BINARY") + 1); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -7890,11 +7875,11 @@ static void queryResult() { pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); } else { pThreadInfo->ntables = g_args.num_of_tables; pThreadInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN); } pThreadInfo->taos = taos_connect( From 5628cb20913c50e2607325742fb6f5c764391e36 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 25 Jun 2021 10:16:18 +0800 Subject: [PATCH 04/50] [TD-4895]: more checking for creating mnode --- src/balance/src/bnMain.c | 21 ++++++++++++++------- src/inc/tbn.h | 1 + src/mnode/src/mnodeDnode.c | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index f022fff6d8..9997d44ca5 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -637,6 +637,19 @@ int32_t bnDropDnode(SDnodeObj *pDnode) { return TSDB_CODE_SUCCESS; } +int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode) { + if (pDnode == NULL) + return 0; + + if (pDnode->isMgmt || pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE + || pDnode->status == TAOS_DN_STATUS_DROPPING + || pDnode->status == TAOS_DN_STATUS_OFFLINE) { + return 0; + } else { + return 1; + } +} + static void bnMonitorDnodeModule() { int32_t numOfMnodes = mnodeGetMnodesNum(); if (numOfMnodes >= tsNumOfMnodes) return; @@ -645,13 +658,7 @@ static void bnMonitorDnodeModule() { SDnodeObj *pDnode = tsBnDnodes.list[i]; if (pDnode == NULL) break; - if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - continue; - } - - if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE) { - continue; - } + if (!bnDnodeCanCreateMnode(pDnode)) continue; mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes); mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true); diff --git a/src/inc/tbn.h b/src/inc/tbn.h index b9f4e3c608..b35f90eb15 100644 --- a/src/inc/tbn.h +++ b/src/inc/tbn.h @@ -31,6 +31,7 @@ void bnReset(); int32_t bnAllocVnodes(struct SVgObj *pVgroup); int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); int32_t bnDropDnode(struct SDnodeObj *pDnode); +int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index ce21af49c2..2325122830 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tgrant.h" -#include "tbn.h" #include "tglobal.h" #include "tconfig.h" #include "tutil.h" @@ -632,7 +631,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum() && !pDnode->isMgmt) { + if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum() + && bnDnodeCanCreateMnode(pDnode)) { bnNotify(); } From 163c58ec1496795c1f7002b9a650e030276300e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:04:17 +0800 Subject: [PATCH 05/50] [td-4894]:fix bug in group by normal columns. --- src/query/src/qExecutor.c | 18 ++++------- tests/script/general/parser/groupby.sim | 40 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index a130a6e7ca..814908a9b3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1282,11 +1282,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn return; } - int64_t* tsList = NULL; SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - if (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - tsList = (int64_t*) pFirstColData->pData; - } + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; STimeWindow w = TSWINDOW_INITIALIZER; @@ -1319,12 +1316,10 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1340,17 +1335,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn memcpy(pInfo->prevData, val, bytes); if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + tfree(pInfo->prevData); } } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 507431f536..6ae5d420d8 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -741,4 +741,44 @@ if $data14 != 2 then return -1 endi +sql create table m1 (ts timestamp, k int, f1 int) tags(a int); +sql create table tm0 using m1 tags(0); +sql create table tm1 using m1 tags(1); + +sql insert into tm0 values('2020-1-1 1:1:1', 1, 10); +sql insert into tm0 values('2020-1-1 1:1:2', 1, 20); +sql insert into tm1 values('2020-2-1 1:1:1', 2, 10); +sql insert into tm1 values('2020-2-1 1:1:2', 2, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 100 +system sh/exec.sh -n dnode1 -s start +sleep 100 + +sql connect +sleep 100 +sql use group_db0; + +print =========================>TD-4894 +sql select count(*),k from m1 group by k; +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 2 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From a2c1754168c35840a2c31c765c674d5465f1a9e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Jun 2021 22:34:30 +0800 Subject: [PATCH 06/50] Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6634) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. * fix crash if no result file. --- src/kit/taosdemo/taosdemo.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index abf3c74436..b5781c6070 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1204,23 +1204,24 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { return ; } - int totalLen = 0; - char temp[16000]; + int64_t totalLen = 0; // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((strlen(pThreadInfo->filePath) > 0) - && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, pThreadInfo); + if (totalLen >= 100*1024*1024 - 32000) { + if (strlen(pThreadInfo->filePath) > 0) + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; + char temp[16000] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; + debugPrint("totalLen: %"PRId64"\n", totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", From 55d3e7a2ea682a682b5d81ac55f9bb8dafd58ee7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Jun 2021 21:45:18 +0800 Subject: [PATCH 07/50] Hotfix/sangshuduo/td 4902 for mastere (#6623) * [TD-4902] FIX with 1 char * Fix prompt of -m @ taosdemo Co-authored-by: SunShine Chan Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b5781c6070..743c4ac668 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -982,9 +982,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->len_of_binary = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + (isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-m need a number following!\n"); + errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); exit(EXIT_FAILURE); } arguments->tb_prefix = argv[++i]; From 1faccd4208d9b87f70b5957403df64696fb1af14 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Sun, 27 Jun 2021 22:41:08 -0700 Subject: [PATCH 08/50] Add init to waltest and make it work --- src/wal/test/waltest.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 9a52a2ca83..0f54a8c53a 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -19,6 +19,7 @@ #include "tglobal.h" #include "tlog.h" #include "twal.h" +#include "tfile.h" int64_t ver = 0; void *pWal = NULL; @@ -36,7 +37,7 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { } int main(int argc, char *argv[]) { - char path[128] = "/home/jhtao/test/wal"; + char path[128] = {0}; int level = 2; int total = 5; int rows = 10000; @@ -72,9 +73,11 @@ int main(int argc, char *argv[]) { printf(" [-h help]: print out this help\n\n"); exit(0); } - } + } taosInitLog("wal.log", 100000, 10); + tfInit(); + walInit(); SWalCfg walCfg = {0}; walCfg.walLevel = level; @@ -122,13 +125,13 @@ int main(int argc, char *argv[]) { printf("index:%" PRId64 " wal:%s\n", index, name); if (code == 0) break; - - index++; } getchar(); walClose(pWal); + walCleanUp(); + tfCleanup(); return 0; } From 6a288d80fb0ab2dcb127ba5f2e00b4e19a381e62 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 16:55:06 +0800 Subject: [PATCH 09/50] [td-4927]:ignore the rows of expired timestamp that are not dropped yet during query according to the TTL settings. --- src/common/inc/tglobal.h | 2 +- src/common/src/tglobal.c | 2 +- src/tsdb/inc/tsdbFile.h | 4 +- src/tsdb/src/tsdbCommit.c | 10 ++--- src/tsdb/src/tsdbMemTable.c | 4 +- src/tsdb/src/tsdbRead.c | 82 ++++++++++++++++++++++++++----------- 6 files changed, 70 insertions(+), 34 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 47460a5fab..e35e805efe 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -155,7 +155,7 @@ extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; -extern int64_t tsMsPerDay[3]; +extern int64_t tsTickPerDay[3]; // system info extern char tsOsName[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index af21ef6d82..51d44add4c 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -204,7 +204,7 @@ SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; * TSDB_TIME_PRECISION_MICRO: 86400000000L * TSDB_TIME_PRECISION_NANO: 86400000000000L */ -int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L}; +int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info char tsOsName[10] = "Linux"; diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index dcb5eadfab..b9d5431de6 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -350,8 +350,8 @@ static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) { } static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) { - *minKey = fid * days * tsMsPerDay[precision]; - *maxKey = *minKey + days * tsMsPerDay[precision] - 1; + *minKey = fid * days * tsTickPerDay[precision]; + *maxKey = *minKey + days * tsTickPerDay[precision] - 1; } static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 82cc6f07f7..75b072b063 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -17,9 +17,9 @@ #define TSDB_MAX_SUBBLOCKS 8 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsMsPerDay[precision] / days - 1); + return (int)((key + 1) / tsTickPerDay[precision] / days - 1); } else { - return (int)((key / tsMsPerDay[precision] / days)); + return (int)((key / tsTickPerDay[precision] / days)); } } @@ -363,9 +363,9 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { TSKEY minKey, midKey, maxKey, now; now = taosGetTimestamp(pCfg->precision); - minKey = now - pCfg->keep * tsMsPerDay[pCfg->precision]; - midKey = now - pCfg->keep2 * tsMsPerDay[pCfg->precision]; - maxKey = now - pCfg->keep1 * tsMsPerDay[pCfg->precision]; + minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; pRtn->minKey = minKey; pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9d8b1ca7f2..50b5d32177 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -632,8 +632,8 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; TSKEY now = taosGetTimestamp(pRepo->config.precision); - TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep; - TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; + TSKEY minKey = now - tsTickPerDay[pRepo->config.precision] * pRepo->config.keep; + TSKEY maxKey = now + tsTickPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; terrno = TSDB_CODE_SUCCESS; pMsg->length = htonl(pMsg->length); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 92a0d489b3..cb50eaf896 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -280,9 +280,17 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa info.tableId.uid = info.pTableObj->tableId.uid; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(info.lastKey >= pQueryHandle->window.skey); + if (info.lastKey == INT64_MIN) { + info.lastKey = pQueryHandle->window.skey; + } + + assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); } else { - assert(info.lastKey <= pQueryHandle->window.skey); + if (info.lastKey == INT64_MIN) { + info.lastKey = pQueryHandle->window.ekey; + } + + assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); } taosArrayPush(pTableCheckInfo, &info); @@ -339,14 +347,44 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } +// Update the query time window according to the data time to live(TTL) information, in order to avoid to return +// the expired data to client, even it is queried already. +static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { + STsdbCfg* pCfg = &pTsdb->config; + + int64_t now = taosGetTimestamp(pCfg->precision); + return now - (tsTickPerDay[pCfg->precision] * pCfg->keep); +} + +static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { + pQueryHandle->window = pCond->twindow; + + int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (startTs > pQueryHandle->window.skey) { + pQueryHandle->window.skey = startTs; + } + + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + if (startTs > pQueryHandle->window.ekey) { + pQueryHandle->window.ekey = startTs; + } + + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } + + tsdbDebug("%p update the query time window, old:%"PRId64" - %"PRId64", new:%"PRId64" - %"PRId64 ", 0x%"PRIx64, + pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); +} + static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); if (pQueryHandle == NULL) { - goto out_of_memory; + goto _end; } pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->cur.fid = INT32_MIN; @@ -354,36 +392,33 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC pQueryHandle->checkFiles = true; pQueryHandle->activeIndex = 0; // current active table index pQueryHandle->qId = qId; - pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->allocSize = 0; pQueryHandle->locateStart = false; pQueryHandle->pMemRef = pMemRef; + pQueryHandle->loadType = pCond->type; + + pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->loadExternalRow = pCond->loadExternalRows; pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; - pQueryHandle->loadType = pCond->type; - if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { - goto out_of_memory; + goto _end; } assert(pCond != NULL && pMemRef != NULL); - if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); - } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); - } + setQueryTimewindow(pQueryHandle, pCond); + if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); if (pQueryHandle->statis == NULL) { - goto out_of_memory; + goto _end; } - pQueryHandle->pColumns = - taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? + // todo: use list instead of array? + pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); if (pQueryHandle->pColumns == NULL) { - goto out_of_memory; + goto _end; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { @@ -392,14 +427,16 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC colInfo.info = pCond->colList[i]; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { - goto out_of_memory; + goto _end; } + taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; } pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); } + STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -407,7 +444,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC if (pQueryHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto out_of_memory; + goto _end; } tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); @@ -415,7 +452,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC return (TsdbQueryHandleT) pQueryHandle; - out_of_memory: + _end: tsdbCleanupQueryHandle(pQueryHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -864,10 +901,10 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio } if (key < 0) { - key -= (daysPerFile * tsMsPerDay[precision]); + key -= (daysPerFile * tsTickPerDay[precision]); } - int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision])); // set the starting fileId + int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerDay[precision])); // set the starting fileId if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; } @@ -3548,7 +3585,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab taosArrayPush(group, &info); taosArrayPush(pGroupInfo->pGroupList, &group); - return TSDB_CODE_SUCCESS; _error: From 8a79e00e4dfed767ee9b0f4eb9dcac8e41ed1e22 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 16:55:59 +0800 Subject: [PATCH 10/50] [td-4791]:fix bug in nest query. --- src/client/src/tscSQLParser.c | 40 +++++----- tests/script/general/parser/nestquery.sim | 89 +++++++++++++++++++++-- 2 files changed, 100 insertions(+), 29 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cf72b7abd9..e24c063b2e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7810,18 +7810,19 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf pQueryInfo->numOfTables = 0; // parse the subquery in the first place - int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); - for(int32_t i = 0; i < numOfSub; ++i) { + int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); + for (int32_t i = 0; i < numOfSub; ++i) { code = doValidateSubquery(pSqlNode, i, pSql, pQueryInfo, tscGetErrorMsgPayload(pCmd)); if (code != TSDB_CODE_SUCCESS) { return code; } } - + int32_t timeWindowQuery = (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != TSDB_CODE_SUCCESS) { + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != + TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -7831,12 +7832,12 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } // todo NOT support yet - for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { + for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t f = pExpr->base.functionId; + int32_t f = pExpr->base.functionId; if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } + } if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) { pExpr->base.numOfParams = 1; @@ -7845,22 +7846,19 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } } - // todo derivative function requires ts column exists in subquery STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - if (numOfExprs == 1) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } else { - SExprInfo* pExpr = tscExprGet(pQueryInfo, 1); - int32_t f = pExpr->base.functionId; - if ((f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { + int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + + int32_t f = pExpr->base.functionId; + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } } } diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index fd56a91dd6..b26d163ab5 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -179,14 +179,6 @@ if $data21 != 49.500000000 then return -1 endi -#define TSDB_FUNC_APERCT 7 -#define TSDB_FUNC_TWA 14 -#define TSDB_FUNC_LEASTSQR 15 -#define TSDB_FUNC_DIFF 24 -#define TSDB_FUNC_INTERP 28 -#define TSDB_FUNC_IRATE 30 -#define TSDB_FUNC_DERIVATIVE 32 - sql_error select stddev(c1) from (select c1 from nest_tb0); sql_error select percentile(c1, 20) from (select * from nest_tb0); sql_error select interp(c1) from (select * from nest_tb0); @@ -197,9 +189,90 @@ sql_error select diff(c1), twa(c1) from (select * from nest_tb0); sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); sql select apercentile(c1, 50) from (select * from nest_tb0) interval(1d) +if $rows != 7 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data01 != 47.571428571 then + return -1 +endi + +if $data10 != @20-09-16 00:00:00.000@ then + return -1 +endi + +if $data11 != 49.666666667 then + return -1 +endi + +if $data20 != @20-09-17 00:00:00.000@ then + return -1 +endi + +if $data21 != 49.000000000 then + return -1 +endi + +if $data30 != @20-09-18 00:00:00.000@ then + return -1 +endi + +if $data31 != 48.333333333 then + return -1 +endi + sql select twa(c1) from (select * from nest_tb0); +if $rows != 1 then + return -1 +endi + +if $data00 != 49.500000000 then + return -1 +endi + sql select leastsquares(c1, 1, 1) from (select * from nest_tb0); +if $rows != 1 then + return -1 +endi + +if $data00 != @{slop:0.000100, intercept:49.000000}@ then + return -1 +endi + sql select irate(c1) from (select * from nest_tb0); +if $data00 != 0.016666667 then + return -1 +endi + +sql select derivative(c1, 1s, 0) from (select * from nest_tb0); +if $rows != 9999 then + return -1 +endi + +if $data00 != @20-09-15 00:01:00.000@ then + return -1 +endi + +if $data01 != 0.016666667 then + return -1 +endi + +if $data10 != @20-09-15 00:02:00.000@ then + return -1 +endi + +if $data11 != 0.016666667 then + return -1 +endi + +sql select diff(c1) from (select * from nest_tb0); +if $rows != 9999 then + return -1 +endi sql select avg(c1),sum(c2), max(c3), min(c4), count(*), first(c7), last(c7),spread(c6) from (select * from nest_tb0) interval(1d); if $rows != 7 then From 51dc9fa425b19584c8724837879111624387fb6e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 19:34:48 +0800 Subject: [PATCH 11/50] [td-225]fix bug by regression test. --- src/tsdb/src/tsdbRead.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index cb50eaf896..7d366ec814 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -280,16 +280,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa info.tableId.uid = info.pTableObj->tableId.uid; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (info.lastKey == INT64_MIN) { + if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { info.lastKey = pQueryHandle->window.skey; } assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); } else { - if (info.lastKey == INT64_MIN) { - info.lastKey = pQueryHandle->window.ekey; - } - assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); } From 25d2821d78d09d050a36447ed8962772cf4aaec2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 10:41:19 +0800 Subject: [PATCH 12/50] [td-225]fix bug found by regression test. --- src/query/src/qExecutor.c | 4 +- src/tsdb/src/tsdbRead.c | 61 +++++++++++++++---- .../general/parser/single_row_in_tb_query.sim | 4 ++ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e7f63a027d..6138d9deb3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4145,8 +4145,8 @@ static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, in return pFillCol; } -int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, - SArray* pOperator, void* param) { +int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, + void* param) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7d366ec814..b1c8633c02 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -343,6 +343,15 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } +static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { + assert(pQueryHandle != NULL); + + STimeWindow* w = &pQueryHandle->window; + bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + + return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); +} + // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { @@ -355,23 +364,27 @@ static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { pQueryHandle->window = pCond->twindow; + bool updateTs = false; int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (startTs > pQueryHandle->window.skey) { pQueryHandle->window.skey = startTs; + pCond->twindow.skey = startTs; + updateTs = true; } - - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); } else { if (startTs > pQueryHandle->window.ekey) { pQueryHandle->window.ekey = startTs; + pCond->twindow.ekey = startTs; + updateTs = true; } - - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); } - tsdbDebug("%p update the query time window, old:%"PRId64" - %"PRId64", new:%"PRId64" - %"PRId64 ", 0x%"PRIx64, - pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); + if (updateTs) { + tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 + ", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, + pQueryHandle->window.ekey, pQueryHandle->qId); + } } static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { @@ -456,6 +469,9 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (emptyQueryTimewindow(pQueryHandle)) { + return (TsdbQueryHandleT*) pQueryHandle; + } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -479,6 +495,15 @@ TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STable void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { STsdbQueryHandle* pQueryHandle = queryHandle; + if (emptyQueryTimewindow(pQueryHandle)) { + if (pCond->order != pQueryHandle->order) { + pQueryHandle->order = pCond->order; + SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); + } + + return; + } + pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; @@ -1204,8 +1229,9 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pQueryHandle->cur; int32_t code = TSDB_CODE_SUCCESS; + bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (asc) { // query ended in/started from current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { @@ -1226,7 +1252,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, assert(pCheckInfo->lastKey <= pBlock->keyLast); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer - cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(pBlock->numOfRows - 1); + cur->pos = asc? 0:(pBlock->numOfRows - 1); code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block @@ -1246,7 +1272,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, assert(pCheckInfo->lastKey >= pBlock->keyFirst); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { - cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(pBlock->numOfRows-1); + cur->pos = asc? 0:(pBlock->numOfRows-1); code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } } @@ -2717,6 +2743,11 @@ static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + if (emptyQueryTimewindow(pQueryHandle)) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + return false; + } + int64_t stime = taosGetTimestampUs(); int64_t elapsedTime = stime; @@ -3669,15 +3700,21 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { return; } - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle->statis); - // todo check error - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (!emptyQueryTimewindow(pQueryHandle)) { + tsdbMayUnTakeMemSnapshot(pQueryHandle); + } else { + assert(pQueryHandle->pTableCheckInfo == NULL); + } + + if (pQueryHandle->pTableCheckInfo != NULL) { + pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + } tsdbDestroyReadH(&pQueryHandle->rhelper); diff --git a/tests/script/general/parser/single_row_in_tb_query.sim b/tests/script/general/parser/single_row_in_tb_query.sim index 1f9cb8b558..acf85ea692 100644 --- a/tests/script/general/parser/single_row_in_tb_query.sim +++ b/tests/script/general/parser/single_row_in_tb_query.sim @@ -193,3 +193,7 @@ endi if $data04 != 1 then return -1 endi + +print ===============>safty check TD-4927 +sql select first(ts, c1) from sr_stb where ts<1 group by t1; +sql select first(ts, c1) from sr_stb where ts>0 and ts<1; \ No newline at end of file From 00055ce2aad7a4e1da5ac5fbab9a489c082179d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 13:16:11 +0800 Subject: [PATCH 13/50] [td-225]fix bug found by regression test. --- src/tsdb/src/tsdbRead.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b1c8633c02..2c44df0237 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -39,6 +39,12 @@ enum { TSDB_QUERY_TYPE_LAST = 2, }; +enum { + TSDB_CACHED_TYPE_NONE = 0, + TSDB_CACHED_TYPE_LASTROW = 1, + TSDB_CACHED_TYPE_LAST = 2, +}; + typedef struct SQueryFilePos { int32_t fid; int32_t slot; @@ -2751,10 +2757,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t stime = taosGetTimestampUs(); int64_t elapsedTime = stime; + // TODO refactor: remove "type" if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { - if (pQueryHandle->cachelastrow == 1) { + if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { return loadCachedLastRow(pQueryHandle); - } else if (pQueryHandle->cachelastrow == 2) { + } else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { return loadCachedLast(pQueryHandle); } } @@ -2960,7 +2967,7 @@ out: } bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { - return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > 0; + return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { @@ -2978,9 +2985,9 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g if (((STable*)pInfo->pTable)->lastRow) { code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = 0; + pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; } else { - pQueryHandle->cachelastrow = 1; + pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; } } @@ -3000,12 +3007,11 @@ int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { int32_t code = 0; if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ - pQueryHandle->cachelastrow = 2; + pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; } // update the tsdb query time range if (pQueryHandle->cachelastrow) { - pQueryHandle->window = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = false; pQueryHandle->activeIndex = -1; // start from -1 } From ccb66fca0cc4d15ee9d8db654220f5ba23909105 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 13:18:30 +0800 Subject: [PATCH 14/50] [td-4969]:enable the interval value to be 1us. --- src/client/src/tscSQLParser.c | 6 +++--- src/common/src/tglobal.c | 4 ++-- src/os/src/detail/osTime.c | 1 + tests/script/general/parser/fill_us.sim | 4 ++-- tests/script/general/parser/last_cache.sim | 2 +- tests/script/general/parser/testSuite.sim | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e24c063b2e..50c8aa0376 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -914,8 +914,8 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn } int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { + const char* msg1 = "sliding cannot be used without interval"; const char* msg2 = "interval cannot be less than 10 ms"; - const char* msg3 = "sliding cannot be used without interval"; SSqlCmd* pCmd = &pSql->cmd; @@ -924,7 +924,7 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS if (!TPARSER_HAS_TOKEN(pSqlNode->interval.interval)) { if (TPARSER_HAS_TOKEN(pSqlNode->sliding)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } return TSDB_CODE_SUCCESS; @@ -945,7 +945,7 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { // interval cannot be less than 10 milliseconds - if (convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MILLI) < tsMinIntervalTime) { + if (convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 51d44add4c..ec98915227 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -84,8 +84,8 @@ int32_t tsMaxNumOfOrderedResults = 100000; // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; -// 10 ms for interval time range, changed accordingly -int32_t tsMinIntervalTime = 10; +// 1 us for interval time range, changed accordingly +int32_t tsMinIntervalTime = 1; // 20sec, the maximum value of stream computing delay, changed accordingly int32_t tsMaxStreamComputDelay = 20000; diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 4d64e14d94..847d484d9e 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -348,6 +348,7 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec {1.0 / 1000000, 1.0 / 1000, 1.} }; return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } + static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { switch (unit) { diff --git a/tests/script/general/parser/fill_us.sim b/tests/script/general/parser/fill_us.sim index 8cd2c33347..762413d0a1 100644 --- a/tests/script/general/parser/fill_us.sim +++ b/tests/script/general/parser/fill_us.sim @@ -959,14 +959,14 @@ endi if $data31 != 9.000000000 then return -1 endi -if $data41 != null then +if $data41 != NULL then print ===== $data41 return -1 endi if $data51 != 16.000000000 then return -1 endi -if $data61 != null then +if $data61 != NULL then print ===== $data61 return -1 endi diff --git a/tests/script/general/parser/last_cache.sim b/tests/script/general/parser/last_cache.sim index 4b3285871b..9c414263ec 100644 --- a/tests/script/general/parser/last_cache.sim +++ b/tests/script/general/parser/last_cache.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 5f71138966..d7f06769a8 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -12,7 +12,7 @@ run general/parser/create_tb.sim run general/parser/dbtbnameValidate.sim run general/parser/fill.sim run general/parser/fill_stb.sim -#run general/parser/fill_us.sim # +run general/parser/fill_us.sim run general/parser/first_last.sim run general/parser/import_commit1.sim run general/parser/import_commit2.sim From 2237648e8333c801c9cb0513bcaff23d711a3d42 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 29 Jun 2021 14:20:45 +0800 Subject: [PATCH 15/50] [TD-3178] : describe command "COMPACT". --- documentation20/cn/00.index/docs.md | 1 + documentation20/cn/11.administrator/docs.md | 13 +++++++++++++ documentation20/cn/13.faq/docs.md | 1 + 3 files changed, 15 insertions(+) diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index ed7156be17..18bdc15d30 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -106,6 +106,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [数据导入](/administrator#import)：可按脚本文件导入，也可按数据文件导入 * [数据导出](/administrator#export)：从shell按表导出，也可用taosdump工具做各种导出 * [系统监控](/administrator#status)：检查系统现有的连接、查询、流式计算，日志和事件等 +* [性能优化](/administrator#optimize)：对长期运行的系统进行维护优化，保障性能表现 * [文件目录结构](/administrator#directories)：TDengine数据文件、配置文件等所在目录 * [参数限制与保留关键字](/administrator#keywords)：TDengine的参数限制与保留关键字列表 diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index dce2f50d9e..753f91f589 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -418,6 +418,19 @@ TDengine启动后，会自动创建一个监测数据库log，并自动将服务 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。 + +## 性能优化 + +因数据行 [update](https://www.taosdata.com/cn/documentation/faq#update)、表删除、数据过期等原因，TDengine 的磁盘存储文件有可能出现数据碎片，影响查询操作的性能表现。从 2.1.3.0 版本开始，新增 SQL 指令 COMPACT 来启动碎片重整过程： + +```mysql +COMPACT VNODES IN (vg_id1, vg_id2, ...) +``` + +COMPACT 命令对指定的一个或多个 VGroup 启动碎片重整，系统会通过任务队列尽快安排重整操作的具体执行。COMPACT 指令所需的 VGroup id，可以通过 `SHOW VGROUPS;` 指令的输出结果获取；而且在 `SHOW VGROUPS;` 中会有一个 compacting 列，值为 1 时表示对应的 VGroup 正在进行碎片重整，为 0 时则表示并没有处于重整状态。 + +需要注意的是，碎片重整操作会大幅消耗磁盘 I/O。因此在重整进行期间，有可能会影响节点的写入和查询性能，甚至在极端情况下导致短时间的阻写。 + ## 文件目录结构 安装TDengine后，默认会在操作系统中生成下列目录或文件： diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index 8d18ae269d..300ff27fe4 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -87,6 +87,7 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 + ## 9. 我可以删除或更新一条记录吗？ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支持。 From 80047afd7e7e2adca213284a5e59764bda5d3e6a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 29 Jun 2021 15:55:06 +0800 Subject: [PATCH 16/50] [TD-3086] : tags columns can use timestamp type now. --- documentation20/cn/12.taos-sql/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 2a1112fd16..764560dbeb 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -279,11 +279,11 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传 说明： - 1) TAGS 列的数据类型不能是 timestamp 类型； + 1) TAGS 列的数据类型不能是 timestamp 类型；（从 2.1.3.0 版本开始，TAGS 列中支持使用 timestamp 类型，但需注意在 TAGS 中的 timestamp 列写入数据时需要提供给定值，而暂不支持四则运算，例如 `NOW + 10s` 这类表达式） 2) TAGS 列名不能与其他列名相同； - 3) TAGS 列名不能为预留关键字； + 3) TAGS 列名不能为预留关键字（参见：[参数限制与保留关键字](https://www.taosdata.com/cn/documentation/administrator#keywords) 章节）； 4) TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。 From 44758e1b51238e62f01babb730260a99a720055b Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 29 Jun 2021 16:17:15 +0800 Subject: [PATCH 17/50] [TD-4906]: fix grafana dashboard import error in Grafana v6.2.x (#6661) --- src/connector/grafanaplugin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 3530c6df09..dbc5f04ebd 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa +Subproject commit dbc5f04ebd29522d2acd0636f6fc350060d15a6b From 46b3927e07f31a6aedb47753df3ea21ad184aff6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Jun 2021 16:23:45 +0800 Subject: [PATCH 18/50] Docs/sangshuduo/td 4752 python nanosecond (#6664) * [TD-4752]: python connector supports nanoseconds. * update english document too. --- documentation20/cn/08.connector/docs.md | 2 +- documentation20/en/08.connector/docs.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 454d896fc2..762d1ad480 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -557,7 +557,7 @@ c1.close() conn.close() ``` -#### 关于纳秒 (nanosecon) 在 Python 连接器中的说明 +#### 关于纳秒 (nanosecond) 在 Python 连接器中的说明 由于目前 Python 对 nanosecond 支持的不完善(参见链接 1. 2. )，目前的实现方式是在 nanosecond 精度时返回整数，而不是 ms 和 us 返回的 datetime 类型，应用开发者需要自行处理，建议使用 pandas 的 to_datetime()。未来如果 Python 正式完整支持了纳秒，涛思数据可能会修改相关接口。 diff --git a/documentation20/en/08.connector/docs.md b/documentation20/en/08.connector/docs.md index 3da9937f98..9b3940a2e8 100644 --- a/documentation20/en/08.connector/docs.md +++ b/documentation20/en/08.connector/docs.md @@ -515,6 +515,13 @@ c1.close() conn.close() ``` +#### Using nanosecond in Python connector + +So far Python still does not completely support nanosecond type. Please refer to the link 1 and 2. The implementation of the python connector is to return an integer number for nanosecond value rather than datatime type as what ms and us do. The developer needs to handle it themselves. We recommend using pandas to_datetime() function. If Python officially support nanosecond in the future, TAOS Data might be possible to change the interface accordingly, which mean the application need change too. + +1. https://stackoverflow.com/questions/10611328/parsing-datetime-strings-containing-nanoseconds +2. https://www.python.org/dev/peps/pep-0564/ + #### Helper Users can directly view the usage information of the module through Python's helper, or refer to the sample program in tests/examples/Python. The following are some common classes and methods: From bf14dba3566a0df4f4d96fa746604a9e59379ea2 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 29 Jun 2021 17:39:55 +0800 Subject: [PATCH 19/50] [TD-4528] : add Go example about reading from Kafka and writing to TDengine. --- documentation20/cn/08.connector/docs.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index aa5fa50b66..ca28a25fdf 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -900,6 +900,10 @@ go env -w GOPROXY=https://goproxy.io,direct sql.Open内置的方法，Close closes the statement. +### 其他代码示例 + +[Consume Messages from Kafka](https://github.com/taosdata/go-demo-kafka) 是一个通过 Go 语言实现消费 Kafka 队列写入 TDengine 的示例程序，也可以作为通过 Go 连接 TDengine 的写法参考。 + ## Node.js Connector Node.js连接器支持的系统有： From 0a45ee6812f3d93f1442d651e4bb23b98df83b96 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 10 Aug 2021 14:30:37 +0800 Subject: [PATCH 20/50] [TD-4967]update retention policy testcase --- tests/pytest/insert/retentionpolicy.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index e0446113d6..067fe3e5c3 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -71,13 +71,10 @@ class TDTestRetetion: tdDnodes.start(1) tdLog.info(cmd) + ttime = datetime.datetime.now() tdSql.execute(cmd) self.queryRows=tdSql.query('select * from test') - if self.queryRows==4: - self.checkRows(4,cmd) - return 0 - else: - self.checkRows(5,cmd) + self.checkRows(3,cmd) tdLog.info("=============== step3") tdDnodes.stop(1) os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) @@ -92,7 +89,7 @@ class TDTestRetetion: tdLog.info(cmd) tdSql.execute(cmd) self.queryRows=tdSql.query('select * from test') - self.checkRows(6,cmd) + self.checkRows(3,cmd) tdLog.info("=============== step4") tdDnodes.stop(1) tdDnodes.start(1) @@ -100,7 +97,7 @@ class TDTestRetetion: tdLog.info(cmd) tdSql.execute(cmd) self.queryRows=tdSql.query('select * from test') - self.checkRows(5,cmd) + self.checkRows(4,cmd) tdLog.info("=============== step5") tdDnodes.stop(1) @@ -109,6 +106,21 @@ class TDTestRetetion: self.queryRows=tdSql.query('select * from test where ts > now-1d') self.checkRows(2,cmd) + tdLog.info("=============== step6") + tdDnodes.stop(1) + os.system("date -s '%s'"%(ttime + datetime.timedelta(seconds=(72*60*60-7)))) + tdDnodes.start(1) + while datetime.datetime.now() < (ttime + datetime.timedelta(seconds=(72*60*60-1))): + time.sleep(0.001) + cmd = 'select * from test' + self.queryRows=tdSql.query(cmd) + self.checkRows(4,cmd) + while datetime.datetime.now() < (ttime + datetime.timedelta(hours=72)): + time.sleep(0.001) + cmd = 'select * from test' + self.queryRows=tdSql.query(cmd) + self.checkRows(3,cmd) + def stop(self): os.system("sudo timedatectl set-ntp true") os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) From d654a5c762b114e17ee37a77a8b4656b8e73ab0c Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 29 Jun 2021 18:22:10 +0800 Subject: [PATCH 21/50] [TD-4927] update test case --- tests/pytest/alter/alter_keep.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 72ca635ac3..b23f364fc6 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -173,8 +173,9 @@ class TDTestCase: tdSql.checkData(0,7,'10,10,10') tdSql.error('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') - tdSql.checkRows(rowNum) + tdSql.checkRows(2) + rowNum = 2 tdLog.notice('testing keep will be altered if sudden change from small to big') for i in range(30): tdSql.execute('alter database db keep 14,14,14') @@ -182,14 +183,19 @@ class TDTestCase: tdSql.execute('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') rowNum += 1 - tdSql.checkRows(rowNum ) + tdSql.checkRows(rowNum) tdLog.notice('testing keep will be altered if sudden change from big to small') tdSql.execute('alter database db keep 16,16,16') tdSql.execute('alter database db keep 14,14,14') tdSql.error('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') - tdSql.checkRows(rowNum) + tdSql.checkRows(2) + + tdLog.notice('testing data will show up again when keep is being changed to large value') + tdSql.execute('alter database db keep 40,40,40') + tdSql.query('select * from tb') + tdSql.checkRows(63) From 4233a5a5d4879c22ccf51d0919e2e1e80429da79 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Jun 2021 21:48:15 +0800 Subject: [PATCH 22/50] Hotfix/sangshuduo/td 4892 taosdemo sub fetch for develop (#6668) * fix crash if no result file. * make taosdemo same as master branch. * decrease few debug print level. --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ec4c6d8773..f6dc6d9c79 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1221,7 +1221,7 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; - debugPrint("totalLen: %"PRId64"\n", totalLen); + verbosePrint("%s() LN%d, totalLen: %"PRId64"\n", __func__, __LINE__, totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", @@ -7280,7 +7280,7 @@ static void *superSubscribe(void *sarg) { pThreadInfo->threadID); } - debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", + verbosePrint("%s() LN%d, [%d] subSqlstr: %s\n", __func__, __LINE__, pThreadInfo->threadID, subSqlstr); tsub[tsubSeq] = subscribeImpl( STABLE_CLASS, From 9fd6d0593f45745df4eab56a0b592198fb3b0d6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 22:58:00 +0800 Subject: [PATCH 23/50] [td-4954]:update some error message strings. --- src/client/src/tscSQLParser.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 50c8aa0376..3ceb10520b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1309,15 +1309,8 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - /* timestamp in tag is not allowed */ for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - //if (p->type == TSDB_DATA_TYPE_TIMESTAMP) { - // invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - // return false; - //} - if (!isValidDataType(p->type)) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; @@ -2174,7 +2167,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg3 = "illegal column name"; const char* msg4 = "invalid table name"; const char* msg5 = "parameter is out of range [0, 100]"; - const char* msg6 = "function applied to tags not allowed"; + const char* msg6 = "functions applied to tags are not allowed"; const char* msg7 = "normal table can not apply this function"; const char* msg8 = "multi-columns selection does not support alias column name"; const char* msg9 = "diff/derivative can no be applied to unsigned numeric type"; @@ -3087,9 +3080,9 @@ void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { } bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "TWA/Diff/Derivative/Irate not allowed to apply to super table directly"; + const char* msg1 = "TWA/Diff/Derivative/Irate are not allowed to apply to super table directly"; const char* msg2 = "TWA/Diff/Derivative/Irate only support group by tbname for super table query"; - const char* msg3 = "function not support for super table query"; + const char* msg3 = "functions not support for super table query"; // filter sql function not supported by metric query yet. size_t size = tscNumOfExprs(pQueryInfo); From e55053fc451995827950209bfa03266d9d136a68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 10:31:40 +0800 Subject: [PATCH 24/50] [td-4966]:add more check in group by clause. --- src/client/src/tscSQLParser.c | 72 ++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3ceb10520b..3c0c9ae6c3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3187,34 +3187,42 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd const char* msg2 = "invalid column name in group by clause"; const char* msg3 = "columns from one table allowed as group by columns"; const char* msg4 = "join query does not support group by"; + const char* msg5 = "not allowed column type for group by"; + const char* msg6 = "tags not allowed for table query"; const char* msg7 = "not support group by expression"; - const char* msg8 = "not allowed column type for group by"; - const char* msg9 = "tags not allowed for table query"; + const char* msg8 = "normal column can only locate at the end of group by clause"; // todo : handle two tables situation STableMetaInfo* pTableMetaInfo = NULL; - if (pList == NULL) { return TSDB_CODE_SUCCESS; } - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList); - if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - if (pQueryInfo->numOfTables > 1) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - STableMeta* pTableMeta = NULL; - SSchema* pSchema = NULL; + SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; + if (pGroupExpr->columnInfo == NULL) { + pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); + } - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + if (pGroupExpr->columnInfo == NULL || pQueryInfo->colList == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pGroupExpr->numOfGroupCols = (int16_t)taosArrayGetSize(pList); + if (pGroupExpr->numOfGroupCols > TSDB_MAX_TAGS) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + SSchema *pSchema = NULL; + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + int32_t numOfGroupCols = 0; size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { @@ -3235,28 +3243,20 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - pTableMeta = pTableMetaInfo->pTableMeta; + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = tGetTbnameColumnSchema(); } else { pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } - bool groupTag = false; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols) { - groupTag = true; - } + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); + bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - if (groupTag) { if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } int32_t relIndex = index.columnIndex; @@ -3273,7 +3273,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); @@ -3283,10 +3283,20 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; + numOfGroupCols++; + } + } - if (i == 0 && num > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } + // 1. only one normal column allowed in the group by clause + // 2. the normal column in the group by clause can only located in the end position + if (numOfGroupCols) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + + for(int32_t i = 0; i < num; ++i) { + SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i); + if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } } From b2802fad4493792b72a9cf58db1b65bd622cabd3 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Tue, 29 Jun 2021 20:17:57 -0700 Subject: [PATCH 25/50] Add default path for waltest --- src/wal/test/waltest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 0f54a8c53a..505728fbe4 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -37,7 +37,7 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { } int main(int argc, char *argv[]) { - char path[128] = {0}; + char path[128] = "/tmp/wal"; int level = 2; int total = 5; int rows = 10000; From 1f670fc95a637512bf91872517cc352659689853 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 11:27:17 +0800 Subject: [PATCH 26/50] [td-225]update the sim. --- tests/script/general/parser/groupby.sim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 6ae5d420d8..e47af5588e 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -781,4 +781,14 @@ if $data11 != 2 then return -1 endi +sql_error select count(*) from m1 group by tbname,k,f1; +sql_error select count(*) from m1 group by tbname,k,a; +sql_error select count(*) from m1 group by k, tbname; +sql_error select count(*) from m1 group by k,f1; +sql_error select count(*) from tm0 group by tbname; +sql_error select count(*) from tm0 group by a; +sql_error select count(*) from tm0 group by k,f1; + +sql_error select count(*),f1 from m1 group by tbname,k; + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 201f6c465e56846dfcf87e880e7b06d895394af8 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Tue, 29 Jun 2021 20:42:16 -0700 Subject: [PATCH 27/50] Fix some printf typing issue (#6659) --- src/client/tests/cliTest.cpp | 3 ++- src/client/tests/timeParseTest.cpp | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/client/tests/cliTest.cpp b/src/client/tests/cliTest.cpp index 30f248b541..c2799c7ff6 100644 --- a/src/client/tests/cliTest.cpp +++ b/src/client/tests/cliTest.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "taos.h" #include "tglobal.h" @@ -132,7 +133,7 @@ void validateResultFields() { taos_free_result(res); char sql[512] = {0}; - sprintf(sql, "insert into t1 values(%ld, 99, 'abc', 'test')", start_ts); + sprintf(sql, "insert into t1 values(%" PRId64 ", 99, 'abc', 'test')", start_ts); res = taos_query(conn, sql); ASSERT_EQ(taos_errno(res), 0); diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index 3cc6d541e0..f6de5d46a7 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -1,8 +1,10 @@ -#include "os.h" + #include #include #include +#include +#include "os.h" #include "taos.h" #include "ttoken.h" #include "tutil.h" @@ -15,10 +17,10 @@ int main(int argc, char** argv) { extern void deltaToUtcInitOnce(); /* test parse time function */ TEST(testCase, parse_time) { - + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); deltaToUtcInitOnce(); - + char t1[] = "2018-1-1 1:1:1.952798"; char t13[] = "1970-1-1 0:0:0"; @@ -77,15 +79,15 @@ TEST(testCase, parse_time) { taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, time1); - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 0); taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); deltaToUtcInitOnce(); - + char t14[] = "1970-1-1T0:0:0Z"; taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 0); @@ -135,7 +137,7 @@ TEST(testCase, parse_time) { //======================== add some case ============================// - + char b1[] = "9999-12-31 23:59:59.999"; taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0); EXPECT_EQ(time, 253402271999999); @@ -145,27 +147,27 @@ TEST(testCase, parse_time) { taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 1577811661321); - taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); char t[] = "2021-01-08T02:11:40.000+00:00"; taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); - printf("%ld\n", time); + printf("%" PRId64 "\n", time); } From d9ba8b1662788d9a93bca79e9d7fb5caac3b2a54 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 13:06:51 +0800 Subject: [PATCH 28/50] [td-225] --- src/client/src/tscSQLParser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3c0c9ae6c3..ac4a0bc9d8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3289,7 +3289,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd // 1. only one normal column allowed in the group by clause // 2. the normal column in the group by clause can only located in the end position - if (numOfGroupCols) { + if (numOfGroupCols > 1) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } @@ -6283,7 +6283,7 @@ static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { */ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg3 = "aggregation function should not be mixed up with projection"; + const char* msg2 = "aggregation function should not be mixed up with projection"; bool tagTsColExists = false; int16_t numOfSelectivity = 0; @@ -6362,7 +6362,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { } else { if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - return invalidOperationMsg(msg, msg3); + return invalidOperationMsg(msg, msg2); } if (numOfAggregation > 0 || numOfSelectivity > 0) { From d5f914857ab66755839e1c2833e3c87c45d1e049 Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 30 Jun 2021 13:54:32 +0800 Subject: [PATCH 29/50] [TD-4953]: fixed a crash bug when run taos on Windows. --- src/client/src/tscUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6219fe23c7..175ad04cdb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3526,6 +3526,7 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); + pthread_mutex_init(&pSql->subState.mutex, NULL); for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); From 3f5f0261f231c63df92fd1a36f8a21c2d2630096 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 30 Jun 2021 12:47:12 +0800 Subject: [PATCH 30/50] [TD-4951]: add test case --- tests/pytest/fulltest.sh | 1 + ...tFromCSVOurofOrder.py => insertFromCSV.py} | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) rename tests/pytest/insert/{insertFromCSVOurofOrder.py => insertFromCSV.py} (59%) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 88c5909863..17d0bbe190 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -334,6 +334,7 @@ python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py python3 ./test.py -f insert/unsignedSmallint.py python3 ./test.py -f insert/unsignedTinyint.py +python3 ./test.py -f insert/insertFromCSV.py python3 ./test.py -f query/filterAllUnsignedIntTypes.py python3 ./test.py -f tag_lite/unsignedInt.py diff --git a/tests/pytest/insert/insertFromCSVOurofOrder.py b/tests/pytest/insert/insertFromCSV.py similarity index 59% rename from tests/pytest/insert/insertFromCSVOurofOrder.py rename to tests/pytest/insert/insertFromCSV.py index d4de85b7e9..c5d3648569 100644 --- a/tests/pytest/insert/insertFromCSVOurofOrder.py +++ b/tests/pytest/insert/insertFromCSV.py @@ -28,16 +28,15 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1500074556514 + self.ts = 1500074556514 + self.csvfile = "/tmp/csvfile.csv" + self.rows = 100000 def writeCSV(self): - with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + with open(self.csvfile, 'w', encoding='utf-8', newline='') as csvFile: writer = csv.writer(csvFile, dialect='excel') - for i in range(1000000): - newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) - d = datetime.datetime.fromtimestamp(newTimestamp / 1000) - dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) - writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + for i in range(self.rows): + writer.writerow([self.ts + i, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) def removCSVHeader(self): data = pd.read_csv("ordered.csv") @@ -45,23 +44,25 @@ class TDTestCase: data.to_csv("ordered.csv", header = False, index = False) def run(self): + self.writeCSV() + tdSql.prepare() - tdSql.execute("create table t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") startTime = time.time() - tdSql.execute("insert into t1 file 'outoforder.csv'") + tdSql.execute("insert into t1 file '%s'" % self.csvfile) duration = time.time() - startTime - print("Out of Order - Insert time: %d" % duration) - tdSql.query("select count(*) from t1") - rows = tdSql.getData(0, 0) + print("Insert time: %d" % duration) + tdSql.query("select * from t1") + tdSql.checkRows(self.rows) - tdSql.execute("create table t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") - startTime = time.time() - tdSql.execute("insert into t2 file 'ordered.csv'") - duration = time.time() - startTime - print("Ordered - Insert time: %d" % duration) - tdSql.query("select count(*) from t2") - tdSql.checkData(0,0, rows) + tdSql.execute("create table stb(ts timestamp, c1 int, c2 float, c3 int, c4 int) tags(t1 int, t2 binary(20))") + tdSql.execute("insert into t2 using stb(t1) tags(1) file '%s'" % self.csvfile) + tdSql.query("select * from stb") + tdSql.checkRows(self.rows) + + tdSql.execute("insert into t3 using stb tags(1, 'test') file '%s'" % self.csvfile) + tdSql.query("select * from stb") + tdSql.checkRows(self.rows * 2) def stop(self): tdSql.close() From 1978c7729a5799e8c375792adadf5e4be92fbcb3 Mon Sep 17 00:00:00 2001 From: hope Date: Wed, 30 Jun 2021 15:29:42 +0800 Subject: [PATCH 31/50] 1. move gnu compile flags "-pipe -Wshadow" from root "CMakeLists.txt" to "cmake/define.inc"; and change compile flag "/O3" to "/O2", because "/O3" is not supported by MSVC. 2. fix compile warnings in "src/util/src/tbuffer.c", by using "union" to transform integer with float and double. 3. use "memcpy()" instead of directly type convert in "rc/util/src/tutil.c" to resolve memory alignment problem in ARM platform. 4. reduce other compile warnings. --- CMakeLists.txt | 7 ---- cmake/define.inc | 6 +++- src/os/src/detail/osSysinfo.c | 26 +++++++++----- src/plugins/http/src/httpJson.c | 2 ++ src/util/src/tbuffer.c | 62 ++++++++++++++++++++++++++++----- src/util/src/tconfig.c | 20 ++++++++--- src/util/src/tutil.c | 18 +++++++--- 7 files changed, 108 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f50aca079..fe2cc6b42e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,13 +42,6 @@ INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) -IF (CMAKE_SYSTEM_NAME MATCHES "Linux") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pipe -Wall -Wshadow -Werror") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe -Wall -Wshadow -Werror") -ENDIF () -MESSAGE(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") -MESSAGE(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") - ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(tests) diff --git a/cmake/define.inc b/cmake/define.inc index d6a3f2b915..dc3ea3a0b2 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -178,7 +178,7 @@ IF (TD_WINDOWS) MESSAGE("memory sanitizer detected as false") SET(DEBUG_FLAGS "/Zi /W3 /GL") ENDIF () - SET(RELEASE_FLAGS "/W0 /O3 /GL") + SET(RELEASE_FLAGS "/W0 /O2 /GL") # MSVC only support O2 ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) @@ -201,6 +201,10 @@ IF (TD_WINDOWS_32) MESSAGE(STATUS "windows32 is defined") ENDIF () +IF (TD_LINUX) + SET(COMMON_FLAGS "${COMMON_FLAGS} -pipe -Wshadow") +ENDIF () + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 891dccaf97..04b1efe7bf 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -74,13 +74,14 @@ bool taosGetProcMemory(float *memoryUsedMB) { return false; } + ssize_t _bytes = 0; size_t len; char * line = NULL; while (!feof(fp)) { tfree(line); len = 0; - getline(&line, &len, fp); - if (line == NULL) { + _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { break; } if (strstr(line, "VmRSS:") != NULL) { @@ -113,8 +114,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { size_t len; char * line = NULL; - getline(&line, &len, fp); - if (line == NULL) { + ssize_t _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { uError("read file:%s failed", tsSysCpuFile); fclose(fp); return false; @@ -138,8 +139,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { size_t len = 0; char * line = NULL; - getline(&line, &len, fp); - if (line == NULL) { + ssize_t _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { uError("read file:%s failed", tsProcCpuFile); fclose(fp); return false; @@ -339,6 +340,7 @@ static bool taosGetCardInfo(int64_t *bytes) { return false; } + ssize_t _bytes = 0; size_t len = 2048; char * line = calloc(1, len); @@ -357,7 +359,12 @@ static bool taosGetCardInfo(int64_t *bytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - getline(&line, &len, fp); + _bytes = getline(&line, &len, fp); + if (_bytes < 0) + { + break; + } + line[len - 1] = 0; if (strstr(line, "lo:") != NULL) { @@ -420,6 +427,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { return false; } + ssize_t _bytes = 0; size_t len; char * line = NULL; char tmp[10]; @@ -428,8 +436,8 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { while (!feof(fp)) { tfree(line); len = 0; - getline(&line, &len, fp); - if (line == NULL) { + _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { break; } if (strstr(line, "rchar:") != NULL) { diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index df4c4fbc63..10300e9367 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -297,6 +297,7 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) { } default: + fractionLen = 0; assert(false); } @@ -342,6 +343,7 @@ void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) { } default: + fractionLen = 0; assert(false); } diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c index abfa35f42c..c06d1e59bd 100644 --- a/src/util/src/tbuffer.c +++ b/src/util/src/tbuffer.c @@ -18,6 +18,24 @@ #include "exception.h" #include "taoserror.h" +typedef union Un4B { + uint32_t ui; + float f; +} Un4B; +#if __STDC_VERSION__ >= 201112L +static_assert(sizeof(Un4B) == sizeof(uint32_t), "sizeof(Un4B) must equal to sizeof(uint32_t)"); +static_assert(sizeof(Un4B) == sizeof(float), "sizeof(Un4B) must equal to sizeof(float)"); +#endif + +typedef union Un8B { + uint64_t ull; + double d; +} Un8B; +#if __STDC_VERSION__ >= 201112L +static_assert(sizeof(Un8B) == sizeof(uint64_t), "sizeof(Un8B) must equal to sizeof(uint64_t)"); +static_assert(sizeof(Un8B) == sizeof(double), "sizeof(Un8B) must equal to sizeof(double)"); +#endif + //////////////////////////////////////////////////////////////////////////////// // reader functions @@ -175,13 +193,21 @@ uint64_t tbufReadUint64( SBufferReader* buf ) { } float tbufReadFloat( SBufferReader* buf ) { - uint32_t ret = tbufReadUint32( buf ); - return *(float*)( &ret ); + Un4B _un; + tbufReadToBuffer( buf, &_un, sizeof(_un) ); + if( buf->endian ) { + _un.ui = ntohl( _un.ui ); + } + return _un.f; } double tbufReadDouble(SBufferReader* buf) { - uint64_t ret = tbufReadUint64( buf ); - return *(double*)( &ret ); + Un8B _un; + tbufReadToBuffer( buf, &_un, sizeof(_un) ); + if( buf->endian ) { + _un.ull = htobe64( _un.ull ); + } + return _un.d; } //////////////////////////////////////////////////////////////////////////////// @@ -381,17 +407,37 @@ void tbufWriteUint64At( SBufferWriter* buf, size_t pos, uint64_t data ) { } void tbufWriteFloat( SBufferWriter* buf, float data ) { - tbufWriteUint32( buf, *(uint32_t*)(&data) ); + Un4B _un; + _un.f = data; + if( buf->endian ) { + _un.ui = htonl( _un.ui ); + } + tbufWrite( buf, &_un, sizeof(_un) ); } void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data ) { - tbufWriteUint32At( buf, pos, *(uint32_t*)(&data) ); + Un4B _un; + _un.f = data; + if( buf->endian ) { + _un.ui = htonl( _un.ui ); + } + tbufWriteAt( buf, pos, &_un, sizeof(_un) ); } void tbufWriteDouble( SBufferWriter* buf, double data ) { - tbufWriteUint64( buf, *(uint64_t*)(&data) ); + Un8B _un; + _un.d = data; + if( buf->endian ) { + _un.ull = htobe64( _un.ull ); + } + tbufWrite( buf, &_un, sizeof(_un) ); } void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data ) { - tbufWriteUint64At( buf, pos, *(uint64_t*)(&data) ); + Un8B _un; + _un.d = data; + if( buf->endian ) { + _un.ull = htobe64( _un.ull ); + } + tbufWriteAt( buf, pos, &_un, sizeof(_un) ); } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 442e83bb4f..30d53af4b9 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -327,7 +327,8 @@ void taosReadGlobalLogCfg() { printf("\nconfig file:%s not found, all variables are set to default\n", fileName); return; } - + + ssize_t _bytes = 0; size_t len = 1024; line = calloc(1, len); @@ -337,7 +338,12 @@ void taosReadGlobalLogCfg() { option = value = NULL; olen = vlen = 0; - tgetline(&line, &len, fp); + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) + { + break; + } + line[len - 1] = 0; paGetToken(line, &option, &olen); @@ -373,7 +379,8 @@ bool taosReadGlobalCfg() { return false; } } - + + ssize_t _bytes = 0; size_t len = 1024; line = calloc(1, len); @@ -383,7 +390,12 @@ bool taosReadGlobalCfg() { option = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; - tgetline(&line, &len, fp); + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) + { + break; + } + line[len - 1] = 0; paGetToken(line, &option, &olen); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index bda52936f9..1a73991ade 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -427,13 +427,23 @@ char *taosIpStr(uint32_t ipInt) { } FORCE_INLINE float taos_align_get_float(const char* pBuf) { - float fv = 0; - *(int32_t*)(&fv) = *(int32_t*)pBuf; +#if __STDC_VERSION__ >= 201112L + static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); +#else + assert(sizeof(float) == sizeof(uint32_t)); +#endif + float fv = 0; + memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem return fv; } FORCE_INLINE double taos_align_get_double(const char* pBuf) { - double dv = 0; - *(int64_t*)(&dv) = *(int64_t*)pBuf; +#if __STDC_VERSION__ >= 201112L + static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); +#else + assert(sizeof(double) == sizeof(uint64_t)); +#endif + double dv = 0; + memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem return dv; } From e7cc15a86132b055ba8dbbce29376d5802eb693d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 30 Jun 2021 16:30:46 +0800 Subject: [PATCH 32/50] fix retention case error --- tests/pytest/insert/retentionpolicy.py | 2 +- tests/pytest/util/dnodes.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index 067fe3e5c3..e43321cd6e 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -115,7 +115,7 @@ class TDTestRetetion: cmd = 'select * from test' self.queryRows=tdSql.query(cmd) self.checkRows(4,cmd) - while datetime.datetime.now() < (ttime + datetime.timedelta(hours=72)): + while datetime.datetime.now() <= (ttime + datetime.timedelta(hours=72)): time.sleep(0.001) cmd = 'select * from test' self.queryRows=tdSql.query(cmd) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0f71ffd0a3..ae4ba97eb3 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -127,6 +127,7 @@ class TDDnode: "anyIp":"0", "tsEnableTelemetryReporting":"0", "dDebugFlag":"135", + "tsdbDebugFlag":"135", "mDebugFlag":"135", "sdbDebugFlag":"135", "rpcDebugFlag":"135", From fab17bf797f7eefe1b8a63d44ff57b9e78ce058f Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Wed, 30 Jun 2021 16:44:12 +0800 Subject: [PATCH 33/50] [TD-4986]:fix core when no ts returned from subquery --- src/query/src/qAggMain.c | 4 ++-- tests/script/general/parser/lastrow.sim | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 676e5b6ce6..80285f498b 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -1575,7 +1575,7 @@ static void last_function(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, data, pCtx->inputBytes); - TSKEY ts = GET_TS_DATA(pCtx, i); + TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->hasResult = DATA_SET_FLAG; @@ -1590,7 +1590,7 @@ static void last_function(SQLFunctionCtx *pCtx) { continue; } - TSKEY ts = GET_TS_DATA(pCtx, i); + TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index fea322ec16..7cdd04e2cc 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -82,5 +82,9 @@ endi if $data01 != NULL then return -1 endi +sql select last_row(*) from (select f from lr_nested) +if $rows != 1 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From 72d8302f9dc3c3c5f2169732103cf0f43a5fb5f0 Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Wed, 30 Jun 2021 12:07:37 +0800 Subject: [PATCH 34/50] [TD-4972]:fix error about double precision conversion on timestamp expr value --- src/client/src/tscSQLParser.c | 2 ++ tests/script/general/parser/where.sim | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 658fdf06bb..ade2b846e2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3341,6 +3341,7 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); + pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); } } @@ -4905,6 +4906,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t */ if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); + pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); } tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index d20f013a8e..781b2725b5 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -352,18 +352,24 @@ if $rows != 0 then return -1 endi -print ==========================> td-4783 +print ==========================> td-4783,td-4792 sql create table where_ts(ts timestamp, f int) sql insert into where_ts values('2021-06-19 16:22:00', 1); sql insert into where_ts values('2021-06-19 16:23:00', 2); sql insert into where_ts values('2021-06-19 16:24:00', 3); sql insert into where_ts values('2021-06-19 16:25:00', 1); sql select * from (select * from where_ts) where ts<'2021-06-19 16:25:00' and ts>'2021-06-19 16:22:00' -if $row != 2 then +if $rows != 2 then return -1 endi print $data00, $data01 if $data01 != 2 then return -1 endi +sql insert into where_ts values(now, 5); +sleep 10 +sql select * from (select * from where_ts) where ts Date: Wed, 30 Jun 2021 18:06:13 +0800 Subject: [PATCH 35/50] [TD-2639] : fix return type of "taos_errno". --- documentation20/cn/08.connector/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index ca28a25fdf..218356b0fa 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -259,7 +259,7 @@ typedef struct taosField { 获取最近一次API调用失败的原因,返回值为字符串。 -- `char *taos_errno(TAOS_RES *res)` +- `int taos_errno(TAOS_RES *res)` 获取最近一次API调用失败的原因，返回值为错误代码。 From adc6cbc887b08b157a42f787a806a5afb5fb592e Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 30 Jun 2021 18:17:52 +0800 Subject: [PATCH 36/50] [TD-4928] : fix typo. --- documentation20/cn/08.connector/01.java/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 546e2dbcd3..511bab8a60 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -533,7 +533,7 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | | -------------------- | ----------------- | -------- | | 2.0.31 | 2.1.3.0 及以上 | 1.8.x | -| 2.0.22 - 20.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | +| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | | 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | | 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x | From 067219376ce78080fd1446453329029694dc8d28 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 30 Jun 2021 20:16:22 +0800 Subject: [PATCH 37/50] [TD-4641]: fix nodejs query result error (#6448) Update Node.js connector version to v2.0.8 --- src/connector/nodejs/nodetaos/cinterface.js | 30 +++++++-------------- src/connector/nodejs/package.json | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index f3961e3787..1c2685b8cb 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -126,28 +126,18 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { - data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); - let res = []; - let currOffset = 0; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); - if (dataEntry[0] == FieldTypes.C_BINARY_NULL) { - res.push(null); - } - else { - res.push(ref.readCString(dataEntry)); - } - currOffset += nbytes; - } - return res; -} + function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; - //TODO: should use the correct character encoding - res.push(dataEntry.toString("utf-8")); + + let currOffset = 0; + while (currOffset < data.length) { + let len = data.readIntLE(currOffset, 2); + let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column; + res.push(dataEntry.toString("utf-8")); + currOffset += nbytes; + } return res; } @@ -160,7 +150,7 @@ let convertFunctions = { [FieldTypes.C_BIGINT]: convertBigint, [FieldTypes.C_FLOAT]: convertFloat, [FieldTypes.C_DOUBLE]: convertDouble, - [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_BINARY]: convertNchar, [FieldTypes.C_TIMESTAMP]: convertTimestamp, [FieldTypes.C_NCHAR]: convertNchar } diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index d21b62108b..b57d4c635c 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.7", + "version": "2.0.8", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { From f7c6347c50e95133ce621db0039feede2adfaade Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Jun 2021 21:35:26 +0800 Subject: [PATCH 38/50] Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume for develop (#6689) * [TD-4918]: taosdemo subscribe endAfterResume. * fix heap use-after-free issue. Co-authored-by: Shuduo Sang --- src/client/src/tscServer.c | 2 +- src/kit/taosdemo/taosdemo.c | 8 +++++--- src/mnode/src/mnodeTable.c | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7fc47947c8..8181898ec7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -484,8 +484,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - taosRemoveRef(tscObjRef, handle); tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); } taosReleaseRef(tscObjRef, handle); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f6dc6d9c79..93fac54233 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -636,6 +636,9 @@ static FILE * g_fpOfInsertResult = NULL; #define errorPrint(fmt, ...) \ do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) +// for strncpy buffer overflow +#define min(a, b) (((a) < (b)) ? (a) : (b)) + /////////////////////////////////////////////////// @@ -2574,7 +2577,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->tags[tagIndex].note, @@ -2587,7 +2590,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->columns[columnIndex].note, @@ -7472,7 +7475,6 @@ static void *specifiedSubscribe(void *sarg) { } } taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); - taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0); taos_close(pThreadInfo->taos); return NULL; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 6c96f1f0d2..189c4b6005 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2075,7 +2075,9 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } else { if (pTable->info.type == TSDB_SUPER_TABLE) { int64_t us = taosGetTimestampUs(); - pTable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + uint64_t x = (us&0x000000FFFFFFFFFF); + x = x<<24; + pTable->uid = x + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); } else { pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->tid) & ((1ul << 24) - 1ul)) << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); From d80c0c2804df32dab44bbf6c30d36bae528312f4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:43:03 +0800 Subject: [PATCH 39/50] [TD-4991]: fix build failure on MacOS with apple m1 chip (#6688) * Fix build failure on MacOS with apple m1 chip * Fix build failure on MacOS with apple m1 chip --- cmake/define.inc | 6 +++++- cmake/platform.inc | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/define.inc b/cmake/define.inc index d6a3f2b915..468489c1cc 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -147,7 +147,11 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + IF ("${CPUTYPE}" STREQUAL "apple_m1") + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () IF (TD_MEMORY_SANITIZER) SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -O0 -g3 -DDEBUG") ELSE () diff --git a/cmake/platform.inc b/cmake/platform.inc index 5f7391c996..236732e452 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -108,6 +108,10 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_MIPS_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + SET(CPUTYPE "apple_m1") + MESSAGE(STATUS "Set CPUTYPE to apple silicon m1") + SET(TD_ARM_64 TRUE) ENDIF () ELSE () From a46e18d95b782d379c682b70a5d6c6855cd52608 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 10:15:45 +0800 Subject: [PATCH 40/50] fix retention policy error --- tests/pytest/insert/retentionpolicy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index e43321cd6e..af933aafab 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -119,6 +119,7 @@ class TDTestRetetion: time.sleep(0.001) cmd = 'select * from test' self.queryRows=tdSql.query(cmd) + print(tdSql.queryResult) self.checkRows(3,cmd) def stop(self): From 41ab4a5f9bad0d4d677230a5b75e999bccbb3298 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 10:41:35 +0800 Subject: [PATCH 41/50] [td-4973]:fix the conversion bug in outer query when handling the data of nchar type. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 10 +++++----- src/query/inc/qTableMeta.h | 1 - 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 950d81fb61..7038ae004c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -217,7 +217,7 @@ void tscColumnListDestroy(SArray* pColList); void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); void tscColumnListCopyAll(SArray* dst, const SArray* src); -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId); +void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar); void tscDequoteAndTrimToken(SStrToken* pToken); int32_t tscValidateName(SStrToken* pToken); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ec4bf52527..c0a1afda77 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -319,7 +319,7 @@ void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock); +void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar); void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); void destroyTableNameList(SInsertStatementParam* pInsertParam); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7fc47947c8..1ec006746e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1703,7 +1703,7 @@ int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { uint64_t localQueryId = pSql->self; qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 52d1092107..c74c0fa6cf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -645,7 +645,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { } } -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock) { +void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar) { assert(pRes->numOfCols > 0); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -678,7 +678,7 @@ void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBloc } } - } else if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { + } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { // convert unicode to native code in a temporary buffer extra one byte for terminated symbol pRes->buffer[i] = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); @@ -1072,14 +1072,14 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUp return pOperator; } -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId) { +void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar) { // set the correct result SSDataBlock* p = pQueryInfo->pQInfo->runtimeEnv.outputBuf; pRes->numOfRows = (p != NULL)? p->info.rows: 0; if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { tscCreateResPointerInfo(pRes, pQueryInfo); - tscSetResRawPtrRv(pRes, pQueryInfo, p); + tscSetResRawPtrRv(pRes, pQueryInfo, p, convertNchar); } tscDebug("0x%"PRIx64" retrieve result in pRes, numOfRows:%d", objId, pRes->numOfRows); @@ -1199,7 +1199,7 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue uint64_t qId = pSql->self; qTableQuery(px->pQInfo, &qId); - convertQueryResult(pOutput, px, pSql->self); + convertQueryResult(pOutput, px, pSql->self, false); } static void tscDestroyResPointerInfo(SSqlRes* pRes) { diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 4fc252b644..7ec6dfbcf9 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -118,7 +118,6 @@ typedef struct SQueryInfo { int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - int16_t resColumnId; // result column id bool distinctTag; // distinct tag or not int32_t round; // 0/1/.... int32_t bufLen; From 9150fb18890bb18beb2bf70ce166c8363e909384 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 10:43:12 +0800 Subject: [PATCH 42/50] [td-4970]: limit/offset is not active in the outer query. --- src/client/src/tscSQLParser.c | 4 ++-- src/client/src/tscUtil.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ac4a0bc9d8..329c9eee8f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5953,8 +5953,8 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlN if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* - * the offset value should be removed during retrieve data from virtual node, since the - * global order are done in client side, so the offset is applied at the client side + * The offset value should be removed during retrieve data from virtual node, since the + * global order are done at the client side, so the offset is applied at the client side. * However, note that the maximum allowed number of result for each table should be less * than or equal to the value of limit. */ diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c74c0fa6cf..d366f4c4c7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2938,6 +2938,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { pQueryInfo->pTableMetaInfo = NULL; pQueryInfo->bufLen = pSrc->bufLen; + pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery; pQueryInfo->buf = malloc(pSrc->bufLen); if (pQueryInfo->buf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; From ee120bc1848b8cb4920458da64f62d58f68527a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 1 Jul 2021 10:58:39 +0800 Subject: [PATCH 43/50] [TD-4987]: issues memory sanitizier detected. (#6677) * [TD-4987]: issues memory sanitizier detected. * fix mask width issue. From a34b5785ddfafd7f57a63d353cbf134c0b4323c4 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 11:59:07 +0800 Subject: [PATCH 44/50] fix jenkins error --- tests/Jenkinsfile | 4 ++-- tests/pytest/crash_gen/valgrind_taos.supp | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 0c1f651059..a93b412ab9 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -110,8 +110,8 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single -DskipTests >/dev/null - java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + mvn clean package >/dev/null + java -jar target/JdbcRestfulDemo-jar-with-dependencies.jar ''' } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index b42015a053..376567b7e8 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17722,4 +17722,24 @@ fun:_PyEval_EvalCodeWithName fun:_PyFunction_Vectorcall fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:__libc_alloc_buffer_allocate + fun:alloc_buffer_allocate + fun:__resolv_conf_allocate + fun:__resolv_conf_load + fun:__resolv_conf_get_current + fun:__res_vinit + fun:maybe_init + fun:context_get + fun:__resolv_context_get + fun:gaih_inet.constprop.7 + fun:getaddrinfo + fun:taosGetFqdn + fun:taosCheckGlobalCfg + fun:taos_init_imp } \ No newline at end of file From 824a7c40225ba338d7271f8b126a3e5928a370a0 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Thu, 1 Jul 2021 15:33:12 +0800 Subject: [PATCH 45/50] [TD-4989]: fix stmt_bind_param error for unsigned types (#6692) --- src/client/src/tscPrepare.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 8bb776ffee..b75088ba28 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -745,19 +745,23 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, switch(param->type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: size = 1; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: size = 2; break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_FLOAT: size = 4; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: size = 8; From 9b01458bd9345b8b05e4a23f3854f39e608a304f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 16:43:09 +0800 Subject: [PATCH 46/50] [td-255]fix the bug in recording the elapsed time. --- cmake/platform.inc | 2 +- src/query/src/qExecutor.c | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 236732e452..bdc54c2384 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -157,5 +157,5 @@ ELSEIF (${OSTYPE} MATCHES "Alpine") MESSAGE(STATUS "input osType: Alpine") SET(TD_APLHINE TRUE) ELSE () - MESSAGE(STATUS "input osType unknown: " ${OSTYPE}) + MESSAGE(STATUS "The user specified osType is unknown: " ${OSTYPE}) ENDIF () diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5989bd810b..0fc75481a6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3847,14 +3847,17 @@ int32_t doFillTimeIntervalGapsInResults(SFillInfo* pFillInfo, SSDataBlock *pOutp } void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { - SQueryProfEvent event; - event.eventType = eventType; - event.eventTime = taosGetTimestampUs(); + SQueryProfEvent event = {0}; + + event.eventType = eventType; + event.eventTime = taosGetTimestampUs(); event.operatorType = operatorInfo->operatorType; - SQInfo* qInfo = operatorInfo->pRuntimeEnv->qinfo; - if (qInfo->summary.queryProfEvents) { - taosArrayPush(qInfo->summary.queryProfEvents, &event); + if (operatorInfo->pRuntimeEnv) { + SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } } } From 255d531f929448055cf3ea73857aa76680a322b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 16:53:29 +0800 Subject: [PATCH 47/50] [td-4988]: check for having clause in outer query. --- src/client/src/tscSQLParser.c | 9 ++++++++- tests/script/general/parser/nestquery.sim | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 253529c5d1..7ed27ed94e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -73,7 +73,6 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); - static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength); static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult); @@ -7916,6 +7915,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + if (isTimeWindowQuery(pQueryInfo)) { // check if the first column of the nest query result is timestamp column SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); @@ -7929,6 +7929,13 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } } + // parse the having clause in the first place + int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); + if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != + TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + // set order by info if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMeta)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index b26d163ab5..ce48138f21 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -424,4 +424,6 @@ if $data01 != 1 then return -1 endi +sql_error select last_row(*) from (select * from nest_tb0) having c1 > 0 + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 788f797ad2b1378d75af2a345d08a61fa0c10905 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 16:53:40 +0800 Subject: [PATCH 48/50] fix jenkins err --- tests/Jenkinsfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index a93b412ab9..c75427b5f4 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -114,14 +114,6 @@ pipeline { java -jar target/JdbcRestfulDemo-jar-with-dependencies.jar ''' } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/src/connector/jdbc - mvn clean package -Dmaven.test.skip=true >/dev/null - cd ${WKC}/tests/examples/JDBC/JDBCDemo/ - java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - ''' - } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/ From 40301083039ab00eb5e1659fadecf78d225e99b2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 1 Jul 2021 18:16:07 +0800 Subject: [PATCH 49/50] Feature/sangshuduo/td 3973 use jemalloc (#6703) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. Co-authored-by: Shuduo Sang --- deps/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 99152c6ce3..e3b91efc25 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -20,7 +20,7 @@ IF (TD_DARWIN AND TD_MQTT) ENDIF () IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - MESSAGE("setup dpes/jemalloc, current source dir:" ${CMAKE_CURRENT_SOURCE_DIR}) + MESSAGE("setup deps/jemalloc, current source dir:" ${CMAKE_CURRENT_SOURCE_DIR}) MESSAGE("binary dir:" ${CMAKE_BINARY_DIR}) include(ExternalProject) ExternalProject_Add(jemalloc From 71367b78367004a1abb6b8e8a118fd22078d362e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 19:08:43 +0800 Subject: [PATCH 50/50] [td-255] fix a typo --- tests/script/unique/http/admin.sim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index 1d67a7f86b..ae206744c4 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -34,6 +34,7 @@ print =============== step1 - login system_content curl 127.0.0.1:7111/admin/ print 1-> $system_content if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then + print actual: $system_content return -1 endi @@ -149,6 +150,8 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' 127.0.0.1:7111/admin/all print curl 127.0.0.1:7111/admin/all -----> $system_content if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then + print actual: $system_content + print expect =======> {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11} return -1 endi