From 3626a2de181af79b9bf13adce6102b0ad7941e8f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 23 Jun 2021 10:12:45 +0800 Subject: [PATCH 01/29] update docker cluster script --- .../OneMnodeMultipleVnodesTest.py | 5 ++-- tests/pytest/dockerCluster/basic.py | 27 ++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py b/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py index ee663f89b0..43e281f437 100644 --- a/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py +++ b/tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py @@ -12,9 +12,6 @@ # -*- coding: utf-8 -*- from basic import * -from util.sql import tdSql - - class TDTestCase: @@ -36,4 +33,6 @@ td = TDTestCase() td.init() +## usage: python3 OneMnodeMultipleVnodesTest.py + diff --git a/tests/pytest/dockerCluster/basic.py b/tests/pytest/dockerCluster/basic.py index 50914b0be9..871d69790d 100644 --- a/tests/pytest/dockerCluster/basic.py +++ b/tests/pytest/dockerCluster/basic.py @@ -44,7 +44,16 @@ class BuildDockerCluser: "jnidebugFlag":"135", "qdebugFlag":"135", "maxSQLLength":"1048576" - } + } + cmd = "mkdir -p %s" % self.dockerDir + self.execCmd(cmd) + + cmd = "cp *.yml %s" % self.dockerDir + self.execCmd(cmd) + + cmd = "cp Dockerfile %s" % self.dockerDir + self.execCmd(cmd) + # execute command, and return the output # ref: https://blog.csdn.net/wowocpp/article/details/80775650 @@ -81,7 +90,7 @@ class BuildDockerCluser: def removeFile(self, rootDir, index, dir): cmd = "rm -rf %s/node%d/%s/*" % (rootDir, index, dir) self.execCmd(cmd) - + def clearEnv(self): cmd = "cd %s && docker-compose down --remove-orphans" % self.dockerDir self.execCmd(cmd) @@ -108,10 +117,14 @@ class BuildDockerCluser: self.execCmd(cmd) def updateLocalhosts(self): - cmd = "grep '172.27.0.7 *tdnode1' /etc/hosts" + cmd = "grep '172.27.0.7 *tdnode1' /etc/hosts | sed 's: ::g'" result = self.execCmdAndGetOutput(cmd) - if result and not result.isspace(): + print(result) + if result is None or result.isspace(): + print("==========") cmd = "echo '172.27.0.7 tdnode1' >> /etc/hosts" + display = "echo %s" % cmd + self.execCmd(display) self.execCmd(cmd) def deploy(self): @@ -138,13 +151,13 @@ class BuildDockerCluser: if self.numOfNodes < 2 or self.numOfNodes > 10: print("the number of nodes must be between 2 and 10") exit(0) - self.clearEnv() - self.createDirs() self.updateLocalhosts() self.deploy() def run(self): - cmd = "./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.getTaosdVersion(), self.dockerDir) + cmd = "./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.getTaosdVersion(), self.dockerDir) + display = "echo %s" % cmd + self.execCmd(display) self.execCmd(cmd) self.getConnection() self.createDondes() From d02ce1e2dfb6a885afef0b13b0b601bc75cb7e75 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 15:23:33 +0800 Subject: [PATCH 02/29] Hotfix/sangshuduo/td 4823 taosdemo gettablename (#6606) * [TD-4823]: taosdemo getTableName return empty. * fix typo. * check table name is empty in early stage. --- src/kit/taosdemo/taosdemo.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6b12e66cb9..6314aa6a00 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2497,6 +2497,13 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* pTblName = childTblName; while((row = taos_fetch_row(res)) != NULL) { int32_t* len = taos_fetch_lengths(res); + + if (0 == strlen((char *)row[0])) { + errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", + __func__, __LINE__, count); + exit(-1); + } + tstrncpy(pTblName, (char *)row[0], len[0]+1); //printf("==== sub table name: %s\n", pTblName); count++; @@ -6296,16 +6303,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - // read sample data from file first - if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample")))) { - if (0 != prepareSampleDataForSTable(superTblInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", - __func__, __LINE__); - exit(-1); - } - } - TAOS* taos0 = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); From 44621d8a0f8b1da94bae86b9e3b145ab513baed9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 22:44:46 +0800 Subject: [PATCH 03/29] Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6614) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. --- src/kit/taosdemo/taosdemo.c | 127 ++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6314aa6a00..abf3c74436 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -79,10 +79,9 @@ enum TEST_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) -#define COND_BUF_LEN BUFFER_SIZE - 30 +#define COND_BUF_LEN (BUFFER_SIZE - 30) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 -#define MAX_DB_NAME_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*1024)+20 // max record len: 16*1024, timestamp string and ,('') need extra space @@ -90,7 +89,7 @@ enum TEST_MODE { #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -#define MAX_FILE_NAME_LEN 256 +#define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_DATATYPE 10 @@ -195,13 +194,6 @@ enum _describe_table_index { TSDB_MAX_DESCRIBE_METRIC }; -typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[128]; -} SColDes; - /* Used by main to communicate with parse_opt. */ static char *g_dupstr = NULL; @@ -247,16 +239,16 @@ typedef struct SArguments_S { } SArguments; typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN + 1]; - char dataType[MAX_TB_NAME_SIZE]; + char field[TSDB_COL_NAME_LEN]; + char dataType[16]; uint32_t dataLen; char note[128]; } StrColumn; typedef struct SSuperTable_S { - char sTblName[MAX_TB_NAME_SIZE+1]; - char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - char childTblPrefix[MAX_TB_NAME_SIZE]; + char sTblName[TSDB_TABLE_NAME_LEN]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest uint16_t childTblExists; int64_t childTblCount; @@ -277,8 +269,8 @@ typedef struct SSuperTable_S { int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN+1]; - char tagsFile[MAX_FILE_NAME_LEN+1]; + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; @@ -305,7 +297,7 @@ typedef struct SSuperTable_S { } SSuperTable; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; + char name[TSDB_DB_NAME_LEN]; char create_time[32]; int64_t ntables; int32_t vgroups; @@ -341,11 +333,11 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[MAX_TB_NAME_SIZE]; + char precision[8]; } SDbCfg; typedef struct SDataBase_S { - char dbName[MAX_DB_NAME_SIZE]; + char dbName[TSDB_DB_NAME_LEN]; bool drop; // 0: use exists, 1: if exists, drop then new create SDbCfg dbCfg; uint64_t superTblCount; @@ -353,14 +345,14 @@ typedef struct SDataBase_S { } SDataBase; typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; struct sockaddr_in serv_addr; uint16_t port; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char resultFile[MAX_FILE_NAME_LEN+1]; + char resultFile[MAX_FILE_NAME_LEN]; bool use_metric; bool insert_only; bool do_aggreFunc; @@ -387,7 +379,7 @@ typedef struct SpecifiedQueryInfo_S { bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; int endAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -398,7 +390,7 @@ typedef struct SpecifiedQueryInfo_S { } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { - char sTblName[MAX_TB_NAME_SIZE+1]; + char sTblName[TSDB_TABLE_NAME_LEN]; uint64_t queryInterval; // 0: unlimit > 0 loop/s uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async @@ -407,10 +399,10 @@ typedef struct SuperQueryInfo_S { int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; - char childTblPrefix[MAX_TB_NAME_SIZE]; + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq int sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume; int endAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -420,13 +412,13 @@ typedef struct SuperQueryInfo_S { } SuperQueryInfo; typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; uint16_t port; struct sockaddr_in serv_addr; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char dbName[MAX_DB_NAME_SIZE+1]; + char dbName[TSDB_DB_NAME_LEN]; char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; @@ -438,11 +430,11 @@ typedef struct SThreadInfo_S { TAOS * taos; TAOS_STMT *stmt; int threadID; - char db_name[MAX_DB_NAME_SIZE+1]; + char db_name[TSDB_DB_NAME_LEN]; uint32_t time_precision; char filePath[4096]; FILE *fp; - char tb_prefix[MAX_TB_NAME_SIZE]; + char tb_prefix[TSDB_TABLE_NAME_LEN]; uint64_t start_table_from; uint64_t end_table_to; int64_t ntables; @@ -608,7 +600,7 @@ SArguments g_args = { 1, // query_times 0, // interlace_rows; 30000, // num_of_RPR - (1024*1024), // max_sql_len + (1024*1024), // max_sql_len 10000, // num_of_tables 10000, // num_of_DPT 0, // abort @@ -3038,7 +3030,7 @@ static int startMultiThreadCreateChildTable( for (int64_t i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); pThreadInfo->taos = taos_connect( @@ -3329,7 +3321,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(column, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3344,7 +3336,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, MAX_TB_NAME_SIZE); + columnCase.dataType, strlen(columnCase.dataType) + 1); superTbls->columns[index].dataLen = columnCase.dataLen; index++; } @@ -3400,7 +3392,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3415,7 +3407,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - MAX_TB_NAME_SIZE); + strlen(columnCase.dataType) + 1); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3638,7 +3630,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, db name not found\n"); goto PARSE_OVER; } - tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { @@ -3659,10 +3651,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - MAX_DB_NAME_SIZE); + 8); } else if (!precision) { - //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, 8); } else { printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; @@ -3839,7 +3830,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { @@ -3847,7 +3838,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - MAX_DB_NAME_SIZE); + TSDB_TABLE_NAME_LEN - 20); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl @@ -3915,9 +3906,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, MAX_DB_NAME_SIZE); + dataSource->valuestring, TSDB_DB_NAME_LEN); } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); @@ -3975,10 +3966,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - ts->valuestring, MAX_DB_NAME_SIZE); + ts->valuestring, TSDB_DB_NAME_LEN); } else if (!ts) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - "now", MAX_DB_NAME_SIZE); + "now", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, start_timestamp not found\n"); goto PARSE_OVER; @@ -3998,9 +3989,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, MAX_DB_NAME_SIZE); + sampleFormat->valuestring, TSDB_DB_NAME_LEN); } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; @@ -4245,7 +4236,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { - tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); } else if (!dbs) { printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; @@ -4495,7 +4486,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, super table name input error\n", __func__, __LINE__); @@ -6408,7 +6399,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->time_precision = timePrec; pThreadInfo->superTblInfo = superTblInfo; @@ -6852,7 +6843,7 @@ static void *specifiedTableQuery(void *sarg) { } } - char sqlStr[MAX_DB_NAME_SIZE + 5]; + char sqlStr[TSDB_DB_NAME_LEN + 5]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); @@ -7328,12 +7319,6 @@ static void *superSubscribe(void *sarg) { performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); if (res) { - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->filePath, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - fetchResult(res, pThreadInfo); - } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], @@ -7440,10 +7425,10 @@ static void *specifiedSubscribe(void *sarg) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo); } + fetchResult( + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], + pThreadInfo); g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) @@ -7680,9 +7665,9 @@ static void setParaFromArg(){ g_Dbs.dbCount = 1; g_Dbs.db[0].drop = true; - tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); @@ -7704,7 +7689,7 @@ static void setParaFromArg(){ if (g_args.use_metric) { g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; @@ -7715,7 +7700,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, MAX_TB_NAME_SIZE); + g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].iface = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, @@ -7732,7 +7717,7 @@ static void setParaFromArg(){ } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], MAX_TB_NAME_SIZE); + data_type[i], strlen(data_type[i]) + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -7743,18 +7728,18 @@ static void setParaFromArg(){ for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", MAX_TB_NAME_SIZE); + "BINARY", strlen("BINARY") + 1); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -7890,11 +7875,11 @@ static void queryResult() { pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); } else { pThreadInfo->ntables = g_args.num_of_tables; pThreadInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN); } pThreadInfo->taos = taos_connect( From 5628cb20913c50e2607325742fb6f5c764391e36 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 25 Jun 2021 10:16:18 +0800 Subject: [PATCH 04/29] [TD-4895]: more checking for creating mnode --- src/balance/src/bnMain.c | 21 ++++++++++++++------- src/inc/tbn.h | 1 + src/mnode/src/mnodeDnode.c | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index f022fff6d8..9997d44ca5 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -637,6 +637,19 @@ int32_t bnDropDnode(SDnodeObj *pDnode) { return TSDB_CODE_SUCCESS; } +int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode) { + if (pDnode == NULL) + return 0; + + if (pDnode->isMgmt || pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE + || pDnode->status == TAOS_DN_STATUS_DROPPING + || pDnode->status == TAOS_DN_STATUS_OFFLINE) { + return 0; + } else { + return 1; + } +} + static void bnMonitorDnodeModule() { int32_t numOfMnodes = mnodeGetMnodesNum(); if (numOfMnodes >= tsNumOfMnodes) return; @@ -645,13 +658,7 @@ static void bnMonitorDnodeModule() { SDnodeObj *pDnode = tsBnDnodes.list[i]; if (pDnode == NULL) break; - if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - continue; - } - - if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE) { - continue; - } + if (!bnDnodeCanCreateMnode(pDnode)) continue; mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes); mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true); diff --git a/src/inc/tbn.h b/src/inc/tbn.h index b9f4e3c608..b35f90eb15 100644 --- a/src/inc/tbn.h +++ b/src/inc/tbn.h @@ -31,6 +31,7 @@ void bnReset(); int32_t bnAllocVnodes(struct SVgObj *pVgroup); int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); int32_t bnDropDnode(struct SDnodeObj *pDnode); +int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index ce21af49c2..2325122830 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tgrant.h" -#include "tbn.h" #include "tglobal.h" #include "tconfig.h" #include "tutil.h" @@ -632,7 +631,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum() && !pDnode->isMgmt) { + if (numOfMnodes < tsNumOfMnodes && numOfMnodes < mnodeGetOnlineDnodesNum() + && bnDnodeCanCreateMnode(pDnode)) { bnNotify(); } From 163c58ec1496795c1f7002b9a650e030276300e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:04:17 +0800 Subject: [PATCH 05/29] [td-4894]:fix bug in group by normal columns. --- src/query/src/qExecutor.c | 18 ++++------- tests/script/general/parser/groupby.sim | 40 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index a130a6e7ca..814908a9b3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1282,11 +1282,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn return; } - int64_t* tsList = NULL; SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - if (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - tsList = (int64_t*) pFirstColData->pData; - } + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; STimeWindow w = TSWINDOW_INITIALIZER; @@ -1319,12 +1316,10 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1340,17 +1335,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn memcpy(pInfo->prevData, val, bytes); if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + tfree(pInfo->prevData); } } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 507431f536..6ae5d420d8 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -741,4 +741,44 @@ if $data14 != 2 then return -1 endi +sql create table m1 (ts timestamp, k int, f1 int) tags(a int); +sql create table tm0 using m1 tags(0); +sql create table tm1 using m1 tags(1); + +sql insert into tm0 values('2020-1-1 1:1:1', 1, 10); +sql insert into tm0 values('2020-1-1 1:1:2', 1, 20); +sql insert into tm1 values('2020-2-1 1:1:1', 2, 10); +sql insert into tm1 values('2020-2-1 1:1:2', 2, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 100 +system sh/exec.sh -n dnode1 -s start +sleep 100 + +sql connect +sleep 100 +sql use group_db0; + +print =========================>TD-4894 +sql select count(*),k from m1 group by k; +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 2 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From a2c1754168c35840a2c31c765c674d5465f1a9e7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Jun 2021 22:34:30 +0800 Subject: [PATCH 06/29] Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6634) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. * fix crash if no result file. --- src/kit/taosdemo/taosdemo.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index abf3c74436..b5781c6070 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1204,23 +1204,24 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { return ; } - int totalLen = 0; - char temp[16000]; + int64_t totalLen = 0; // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((strlen(pThreadInfo->filePath) > 0) - && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, pThreadInfo); + if (totalLen >= 100*1024*1024 - 32000) { + if (strlen(pThreadInfo->filePath) > 0) + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; + char temp[16000] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; + debugPrint("totalLen: %"PRId64"\n", totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", From 55d3e7a2ea682a682b5d81ac55f9bb8dafd58ee7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Jun 2021 21:45:18 +0800 Subject: [PATCH 07/29] Hotfix/sangshuduo/td 4902 for mastere (#6623) * [TD-4902] FIX with 1 char * Fix prompt of -m @ taosdemo Co-authored-by: SunShine Chan Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b5781c6070..743c4ac668 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -982,9 +982,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->len_of_binary = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + (isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-m need a number following!\n"); + errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); exit(EXIT_FAILURE); } arguments->tb_prefix = argv[++i]; From 1faccd4208d9b87f70b5957403df64696fb1af14 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Sun, 27 Jun 2021 22:41:08 -0700 Subject: [PATCH 08/29] Add init to waltest and make it work --- src/wal/test/waltest.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 9a52a2ca83..0f54a8c53a 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -19,6 +19,7 @@ #include "tglobal.h" #include "tlog.h" #include "twal.h" +#include "tfile.h" int64_t ver = 0; void *pWal = NULL; @@ -36,7 +37,7 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { } int main(int argc, char *argv[]) { - char path[128] = "/home/jhtao/test/wal"; + char path[128] = {0}; int level = 2; int total = 5; int rows = 10000; @@ -72,9 +73,11 @@ int main(int argc, char *argv[]) { printf(" [-h help]: print out this help\n\n"); exit(0); } - } + } taosInitLog("wal.log", 100000, 10); + tfInit(); + walInit(); SWalCfg walCfg = {0}; walCfg.walLevel = level; @@ -122,13 +125,13 @@ int main(int argc, char *argv[]) { printf("index:%" PRId64 " wal:%s\n", index, name); if (code == 0) break; - - index++; } getchar(); walClose(pWal); + walCleanUp(); + tfCleanup(); return 0; } From 9fd6d0593f45745df4eab56a0b592198fb3b0d6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Jun 2021 22:58:00 +0800 Subject: [PATCH 09/29] [td-4954]:update some error message strings. --- src/client/src/tscSQLParser.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 50c8aa0376..3ceb10520b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1309,15 +1309,8 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - /* timestamp in tag is not allowed */ for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - //if (p->type == TSDB_DATA_TYPE_TIMESTAMP) { - // invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - // return false; - //} - if (!isValidDataType(p->type)) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; @@ -2174,7 +2167,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg3 = "illegal column name"; const char* msg4 = "invalid table name"; const char* msg5 = "parameter is out of range [0, 100]"; - const char* msg6 = "function applied to tags not allowed"; + const char* msg6 = "functions applied to tags are not allowed"; const char* msg7 = "normal table can not apply this function"; const char* msg8 = "multi-columns selection does not support alias column name"; const char* msg9 = "diff/derivative can no be applied to unsigned numeric type"; @@ -3087,9 +3080,9 @@ void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { } bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "TWA/Diff/Derivative/Irate not allowed to apply to super table directly"; + const char* msg1 = "TWA/Diff/Derivative/Irate are not allowed to apply to super table directly"; const char* msg2 = "TWA/Diff/Derivative/Irate only support group by tbname for super table query"; - const char* msg3 = "function not support for super table query"; + const char* msg3 = "functions not support for super table query"; // filter sql function not supported by metric query yet. size_t size = tscNumOfExprs(pQueryInfo); From e55053fc451995827950209bfa03266d9d136a68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 10:31:40 +0800 Subject: [PATCH 10/29] [td-4966]:add more check in group by clause. --- src/client/src/tscSQLParser.c | 72 ++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3ceb10520b..3c0c9ae6c3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3187,34 +3187,42 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd const char* msg2 = "invalid column name in group by clause"; const char* msg3 = "columns from one table allowed as group by columns"; const char* msg4 = "join query does not support group by"; + const char* msg5 = "not allowed column type for group by"; + const char* msg6 = "tags not allowed for table query"; const char* msg7 = "not support group by expression"; - const char* msg8 = "not allowed column type for group by"; - const char* msg9 = "tags not allowed for table query"; + const char* msg8 = "normal column can only locate at the end of group by clause"; // todo : handle two tables situation STableMetaInfo* pTableMetaInfo = NULL; - if (pList == NULL) { return TSDB_CODE_SUCCESS; } - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList); - if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - if (pQueryInfo->numOfTables > 1) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - STableMeta* pTableMeta = NULL; - SSchema* pSchema = NULL; + SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; + if (pGroupExpr->columnInfo == NULL) { + pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); + } - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + if (pGroupExpr->columnInfo == NULL || pQueryInfo->colList == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pGroupExpr->numOfGroupCols = (int16_t)taosArrayGetSize(pList); + if (pGroupExpr->numOfGroupCols > TSDB_MAX_TAGS) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + SSchema *pSchema = NULL; + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + int32_t numOfGroupCols = 0; size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { @@ -3235,28 +3243,20 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - pTableMeta = pTableMetaInfo->pTableMeta; + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = tGetTbnameColumnSchema(); } else { pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } - bool groupTag = false; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols) { - groupTag = true; - } + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); + bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - if (groupTag) { if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } int32_t relIndex = index.columnIndex; @@ -3273,7 +3273,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); @@ -3283,10 +3283,20 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; + numOfGroupCols++; + } + } - if (i == 0 && num > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } + // 1. only one normal column allowed in the group by clause + // 2. the normal column in the group by clause can only located in the end position + if (numOfGroupCols) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + + for(int32_t i = 0; i < num; ++i) { + SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i); + if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } } From b2802fad4493792b72a9cf58db1b65bd622cabd3 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Tue, 29 Jun 2021 20:17:57 -0700 Subject: [PATCH 11/29] Add default path for waltest --- src/wal/test/waltest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 0f54a8c53a..505728fbe4 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -37,7 +37,7 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { } int main(int argc, char *argv[]) { - char path[128] = {0}; + char path[128] = "/tmp/wal"; int level = 2; int total = 5; int rows = 10000; From 1f670fc95a637512bf91872517cc352659689853 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 11:27:17 +0800 Subject: [PATCH 12/29] [td-225]update the sim. --- tests/script/general/parser/groupby.sim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 6ae5d420d8..e47af5588e 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -781,4 +781,14 @@ if $data11 != 2 then return -1 endi +sql_error select count(*) from m1 group by tbname,k,f1; +sql_error select count(*) from m1 group by tbname,k,a; +sql_error select count(*) from m1 group by k, tbname; +sql_error select count(*) from m1 group by k,f1; +sql_error select count(*) from tm0 group by tbname; +sql_error select count(*) from tm0 group by a; +sql_error select count(*) from tm0 group by k,f1; + +sql_error select count(*),f1 from m1 group by tbname,k; + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 201f6c465e56846dfcf87e880e7b06d895394af8 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Tue, 29 Jun 2021 20:42:16 -0700 Subject: [PATCH 13/29] Fix some printf typing issue (#6659) --- src/client/tests/cliTest.cpp | 3 ++- src/client/tests/timeParseTest.cpp | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/client/tests/cliTest.cpp b/src/client/tests/cliTest.cpp index 30f248b541..c2799c7ff6 100644 --- a/src/client/tests/cliTest.cpp +++ b/src/client/tests/cliTest.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "taos.h" #include "tglobal.h" @@ -132,7 +133,7 @@ void validateResultFields() { taos_free_result(res); char sql[512] = {0}; - sprintf(sql, "insert into t1 values(%ld, 99, 'abc', 'test')", start_ts); + sprintf(sql, "insert into t1 values(%" PRId64 ", 99, 'abc', 'test')", start_ts); res = taos_query(conn, sql); ASSERT_EQ(taos_errno(res), 0); diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index 3cc6d541e0..f6de5d46a7 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -1,8 +1,10 @@ -#include "os.h" + #include #include #include +#include +#include "os.h" #include "taos.h" #include "ttoken.h" #include "tutil.h" @@ -15,10 +17,10 @@ int main(int argc, char** argv) { extern void deltaToUtcInitOnce(); /* test parse time function */ TEST(testCase, parse_time) { - + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); deltaToUtcInitOnce(); - + char t1[] = "2018-1-1 1:1:1.952798"; char t13[] = "1970-1-1 0:0:0"; @@ -77,15 +79,15 @@ TEST(testCase, parse_time) { taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, time1); - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 0); taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); deltaToUtcInitOnce(); - + char t14[] = "1970-1-1T0:0:0Z"; taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 0); @@ -135,7 +137,7 @@ TEST(testCase, parse_time) { //======================== add some case ============================// - + char b1[] = "9999-12-31 23:59:59.999"; taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0); EXPECT_EQ(time, 253402271999999); @@ -145,27 +147,27 @@ TEST(testCase, parse_time) { taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 1577811661321); - taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); deltaToUtcInitOnce(); - + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); char t[] = "2021-01-08T02:11:40.000+00:00"; taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); - printf("%ld\n", time); + printf("%" PRId64 "\n", time); } From d9ba8b1662788d9a93bca79e9d7fb5caac3b2a54 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Jun 2021 13:06:51 +0800 Subject: [PATCH 14/29] [td-225] --- src/client/src/tscSQLParser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3c0c9ae6c3..ac4a0bc9d8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3289,7 +3289,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd // 1. only one normal column allowed in the group by clause // 2. the normal column in the group by clause can only located in the end position - if (numOfGroupCols) { + if (numOfGroupCols > 1) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } @@ -6283,7 +6283,7 @@ static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { */ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg3 = "aggregation function should not be mixed up with projection"; + const char* msg2 = "aggregation function should not be mixed up with projection"; bool tagTsColExists = false; int16_t numOfSelectivity = 0; @@ -6362,7 +6362,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { } else { if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - return invalidOperationMsg(msg, msg3); + return invalidOperationMsg(msg, msg2); } if (numOfAggregation > 0 || numOfSelectivity > 0) { From d5f914857ab66755839e1c2833e3c87c45d1e049 Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 30 Jun 2021 13:54:32 +0800 Subject: [PATCH 15/29] [TD-4953]: fixed a crash bug when run taos on Windows. --- src/client/src/tscUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6219fe23c7..175ad04cdb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3526,6 +3526,7 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); + pthread_mutex_init(&pSql->subState.mutex, NULL); for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); From 3f5f0261f231c63df92fd1a36f8a21c2d2630096 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 30 Jun 2021 12:47:12 +0800 Subject: [PATCH 16/29] [TD-4951]: add test case --- tests/pytest/fulltest.sh | 1 + ...tFromCSVOurofOrder.py => insertFromCSV.py} | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) rename tests/pytest/insert/{insertFromCSVOurofOrder.py => insertFromCSV.py} (59%) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 88c5909863..17d0bbe190 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -334,6 +334,7 @@ python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py python3 ./test.py -f insert/unsignedSmallint.py python3 ./test.py -f insert/unsignedTinyint.py +python3 ./test.py -f insert/insertFromCSV.py python3 ./test.py -f query/filterAllUnsignedIntTypes.py python3 ./test.py -f tag_lite/unsignedInt.py diff --git a/tests/pytest/insert/insertFromCSVOurofOrder.py b/tests/pytest/insert/insertFromCSV.py similarity index 59% rename from tests/pytest/insert/insertFromCSVOurofOrder.py rename to tests/pytest/insert/insertFromCSV.py index d4de85b7e9..c5d3648569 100644 --- a/tests/pytest/insert/insertFromCSVOurofOrder.py +++ b/tests/pytest/insert/insertFromCSV.py @@ -28,16 +28,15 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1500074556514 + self.ts = 1500074556514 + self.csvfile = "/tmp/csvfile.csv" + self.rows = 100000 def writeCSV(self): - with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + with open(self.csvfile, 'w', encoding='utf-8', newline='') as csvFile: writer = csv.writer(csvFile, dialect='excel') - for i in range(1000000): - newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) - d = datetime.datetime.fromtimestamp(newTimestamp / 1000) - dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) - writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + for i in range(self.rows): + writer.writerow([self.ts + i, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) def removCSVHeader(self): data = pd.read_csv("ordered.csv") @@ -45,23 +44,25 @@ class TDTestCase: data.to_csv("ordered.csv", header = False, index = False) def run(self): + self.writeCSV() + tdSql.prepare() - tdSql.execute("create table t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") startTime = time.time() - tdSql.execute("insert into t1 file 'outoforder.csv'") + tdSql.execute("insert into t1 file '%s'" % self.csvfile) duration = time.time() - startTime - print("Out of Order - Insert time: %d" % duration) - tdSql.query("select count(*) from t1") - rows = tdSql.getData(0, 0) + print("Insert time: %d" % duration) + tdSql.query("select * from t1") + tdSql.checkRows(self.rows) - tdSql.execute("create table t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") - startTime = time.time() - tdSql.execute("insert into t2 file 'ordered.csv'") - duration = time.time() - startTime - print("Ordered - Insert time: %d" % duration) - tdSql.query("select count(*) from t2") - tdSql.checkData(0,0, rows) + tdSql.execute("create table stb(ts timestamp, c1 int, c2 float, c3 int, c4 int) tags(t1 int, t2 binary(20))") + tdSql.execute("insert into t2 using stb(t1) tags(1) file '%s'" % self.csvfile) + tdSql.query("select * from stb") + tdSql.checkRows(self.rows) + + tdSql.execute("insert into t3 using stb tags(1, 'test') file '%s'" % self.csvfile) + tdSql.query("select * from stb") + tdSql.checkRows(self.rows * 2) def stop(self): tdSql.close() From e7cc15a86132b055ba8dbbce29376d5802eb693d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 30 Jun 2021 16:30:46 +0800 Subject: [PATCH 17/29] fix retention case error --- tests/pytest/insert/retentionpolicy.py | 2 +- tests/pytest/util/dnodes.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index 067fe3e5c3..e43321cd6e 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -115,7 +115,7 @@ class TDTestRetetion: cmd = 'select * from test' self.queryRows=tdSql.query(cmd) self.checkRows(4,cmd) - while datetime.datetime.now() < (ttime + datetime.timedelta(hours=72)): + while datetime.datetime.now() <= (ttime + datetime.timedelta(hours=72)): time.sleep(0.001) cmd = 'select * from test' self.queryRows=tdSql.query(cmd) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0f71ffd0a3..ae4ba97eb3 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -127,6 +127,7 @@ class TDDnode: "anyIp":"0", "tsEnableTelemetryReporting":"0", "dDebugFlag":"135", + "tsdbDebugFlag":"135", "mDebugFlag":"135", "sdbDebugFlag":"135", "rpcDebugFlag":"135", From 72d8302f9dc3c3c5f2169732103cf0f43a5fb5f0 Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Wed, 30 Jun 2021 12:07:37 +0800 Subject: [PATCH 18/29] [TD-4972]:fix error about double precision conversion on timestamp expr value --- src/client/src/tscSQLParser.c | 2 ++ tests/script/general/parser/where.sim | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 658fdf06bb..ade2b846e2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3341,6 +3341,7 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); + pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); } } @@ -4905,6 +4906,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t */ if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); + pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); } tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index d20f013a8e..781b2725b5 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -352,18 +352,24 @@ if $rows != 0 then return -1 endi -print ==========================> td-4783 +print ==========================> td-4783,td-4792 sql create table where_ts(ts timestamp, f int) sql insert into where_ts values('2021-06-19 16:22:00', 1); sql insert into where_ts values('2021-06-19 16:23:00', 2); sql insert into where_ts values('2021-06-19 16:24:00', 3); sql insert into where_ts values('2021-06-19 16:25:00', 1); sql select * from (select * from where_ts) where ts<'2021-06-19 16:25:00' and ts>'2021-06-19 16:22:00' -if $row != 2 then +if $rows != 2 then return -1 endi print $data00, $data01 if $data01 != 2 then return -1 endi +sql insert into where_ts values(now, 5); +sleep 10 +sql select * from (select * from where_ts) where ts Date: Wed, 30 Jun 2021 18:06:13 +0800 Subject: [PATCH 19/29] [TD-2639] : fix return type of "taos_errno". --- documentation20/cn/08.connector/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index ca28a25fdf..218356b0fa 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -259,7 +259,7 @@ typedef struct taosField { 获取最近一次API调用失败的原因,返回值为字符串。 -- `char *taos_errno(TAOS_RES *res)` +- `int taos_errno(TAOS_RES *res)` 获取最近一次API调用失败的原因，返回值为错误代码。 From adc6cbc887b08b157a42f787a806a5afb5fb592e Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 30 Jun 2021 18:17:52 +0800 Subject: [PATCH 20/29] [TD-4928] : fix typo. --- documentation20/cn/08.connector/01.java/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 546e2dbcd3..511bab8a60 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -533,7 +533,7 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | | -------------------- | ----------------- | -------- | | 2.0.31 | 2.1.3.0 及以上 | 1.8.x | -| 2.0.22 - 20.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | +| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | | 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | | 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x | From 067219376ce78080fd1446453329029694dc8d28 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 30 Jun 2021 20:16:22 +0800 Subject: [PATCH 21/29] [TD-4641]: fix nodejs query result error (#6448) Update Node.js connector version to v2.0.8 --- src/connector/nodejs/nodetaos/cinterface.js | 30 +++++++-------------- src/connector/nodejs/package.json | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index f3961e3787..1c2685b8cb 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -126,28 +126,18 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { - data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); - let res = []; - let currOffset = 0; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); - if (dataEntry[0] == FieldTypes.C_BINARY_NULL) { - res.push(null); - } - else { - res.push(ref.readCString(dataEntry)); - } - currOffset += nbytes; - } - return res; -} + function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; - //TODO: should use the correct character encoding - res.push(dataEntry.toString("utf-8")); + + let currOffset = 0; + while (currOffset < data.length) { + let len = data.readIntLE(currOffset, 2); + let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column; + res.push(dataEntry.toString("utf-8")); + currOffset += nbytes; + } return res; } @@ -160,7 +150,7 @@ let convertFunctions = { [FieldTypes.C_BIGINT]: convertBigint, [FieldTypes.C_FLOAT]: convertFloat, [FieldTypes.C_DOUBLE]: convertDouble, - [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_BINARY]: convertNchar, [FieldTypes.C_TIMESTAMP]: convertTimestamp, [FieldTypes.C_NCHAR]: convertNchar } diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index d21b62108b..b57d4c635c 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.7", + "version": "2.0.8", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { From f7c6347c50e95133ce621db0039feede2adfaade Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Jun 2021 21:35:26 +0800 Subject: [PATCH 22/29] Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume for develop (#6689) * [TD-4918]: taosdemo subscribe endAfterResume. * fix heap use-after-free issue. Co-authored-by: Shuduo Sang --- src/client/src/tscServer.c | 2 +- src/kit/taosdemo/taosdemo.c | 8 +++++--- src/mnode/src/mnodeTable.c | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7fc47947c8..8181898ec7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -484,8 +484,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - taosRemoveRef(tscObjRef, handle); tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); } taosReleaseRef(tscObjRef, handle); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f6dc6d9c79..93fac54233 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -636,6 +636,9 @@ static FILE * g_fpOfInsertResult = NULL; #define errorPrint(fmt, ...) \ do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) +// for strncpy buffer overflow +#define min(a, b) (((a) < (b)) ? (a) : (b)) + /////////////////////////////////////////////////// @@ -2574,7 +2577,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->tags[tagIndex].note, @@ -2587,7 +2590,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->columns[columnIndex].note, @@ -7472,7 +7475,6 @@ static void *specifiedSubscribe(void *sarg) { } } taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); - taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0); taos_close(pThreadInfo->taos); return NULL; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 6c96f1f0d2..189c4b6005 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2075,7 +2075,9 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } else { if (pTable->info.type == TSDB_SUPER_TABLE) { int64_t us = taosGetTimestampUs(); - pTable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + uint64_t x = (us&0x000000FFFFFFFFFF); + x = x<<24; + pTable->uid = x + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); } else { pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->tid) & ((1ul << 24) - 1ul)) << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); From d80c0c2804df32dab44bbf6c30d36bae528312f4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:43:03 +0800 Subject: [PATCH 23/29] [TD-4991]: fix build failure on MacOS with apple m1 chip (#6688) * Fix build failure on MacOS with apple m1 chip * Fix build failure on MacOS with apple m1 chip --- cmake/define.inc | 6 +++++- cmake/platform.inc | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/define.inc b/cmake/define.inc index d6a3f2b915..468489c1cc 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -147,7 +147,11 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + IF ("${CPUTYPE}" STREQUAL "apple_m1") + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () IF (TD_MEMORY_SANITIZER) SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -O0 -g3 -DDEBUG") ELSE () diff --git a/cmake/platform.inc b/cmake/platform.inc index 5f7391c996..236732e452 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -108,6 +108,10 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_MIPS_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + SET(CPUTYPE "apple_m1") + MESSAGE(STATUS "Set CPUTYPE to apple silicon m1") + SET(TD_ARM_64 TRUE) ENDIF () ELSE () From a46e18d95b782d379c682b70a5d6c6855cd52608 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 10:15:45 +0800 Subject: [PATCH 24/29] fix retention policy error --- tests/pytest/insert/retentionpolicy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index e43321cd6e..af933aafab 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -119,6 +119,7 @@ class TDTestRetetion: time.sleep(0.001) cmd = 'select * from test' self.queryRows=tdSql.query(cmd) + print(tdSql.queryResult) self.checkRows(3,cmd) def stop(self): From 41ab4a5f9bad0d4d677230a5b75e999bccbb3298 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 10:41:35 +0800 Subject: [PATCH 25/29] [td-4973]:fix the conversion bug in outer query when handling the data of nchar type. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 10 +++++----- src/query/inc/qTableMeta.h | 1 - 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 950d81fb61..7038ae004c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -217,7 +217,7 @@ void tscColumnListDestroy(SArray* pColList); void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); void tscColumnListCopyAll(SArray* dst, const SArray* src); -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId); +void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar); void tscDequoteAndTrimToken(SStrToken* pToken); int32_t tscValidateName(SStrToken* pToken); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ec4bf52527..c0a1afda77 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -319,7 +319,7 @@ void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock); +void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar); void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); void destroyTableNameList(SInsertStatementParam* pInsertParam); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7fc47947c8..1ec006746e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1703,7 +1703,7 @@ int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { uint64_t localQueryId = pSql->self; qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 52d1092107..c74c0fa6cf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -645,7 +645,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { } } -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock) { +void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar) { assert(pRes->numOfCols > 0); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -678,7 +678,7 @@ void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBloc } } - } else if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { + } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { // convert unicode to native code in a temporary buffer extra one byte for terminated symbol pRes->buffer[i] = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); @@ -1072,14 +1072,14 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUp return pOperator; } -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId) { +void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar) { // set the correct result SSDataBlock* p = pQueryInfo->pQInfo->runtimeEnv.outputBuf; pRes->numOfRows = (p != NULL)? p->info.rows: 0; if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { tscCreateResPointerInfo(pRes, pQueryInfo); - tscSetResRawPtrRv(pRes, pQueryInfo, p); + tscSetResRawPtrRv(pRes, pQueryInfo, p, convertNchar); } tscDebug("0x%"PRIx64" retrieve result in pRes, numOfRows:%d", objId, pRes->numOfRows); @@ -1199,7 +1199,7 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue uint64_t qId = pSql->self; qTableQuery(px->pQInfo, &qId); - convertQueryResult(pOutput, px, pSql->self); + convertQueryResult(pOutput, px, pSql->self, false); } static void tscDestroyResPointerInfo(SSqlRes* pRes) { diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 4fc252b644..7ec6dfbcf9 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -118,7 +118,6 @@ typedef struct SQueryInfo { int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - int16_t resColumnId; // result column id bool distinctTag; // distinct tag or not int32_t round; // 0/1/.... int32_t bufLen; From 9150fb18890bb18beb2bf70ce166c8363e909384 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jul 2021 10:43:12 +0800 Subject: [PATCH 26/29] [td-4970]: limit/offset is not active in the outer query. --- src/client/src/tscSQLParser.c | 4 ++-- src/client/src/tscUtil.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ac4a0bc9d8..329c9eee8f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5953,8 +5953,8 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlN if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* - * the offset value should be removed during retrieve data from virtual node, since the - * global order are done in client side, so the offset is applied at the client side + * The offset value should be removed during retrieve data from virtual node, since the + * global order are done at the client side, so the offset is applied at the client side. * However, note that the maximum allowed number of result for each table should be less * than or equal to the value of limit. */ diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c74c0fa6cf..d366f4c4c7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2938,6 +2938,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { pQueryInfo->pTableMetaInfo = NULL; pQueryInfo->bufLen = pSrc->bufLen; + pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery; pQueryInfo->buf = malloc(pSrc->bufLen); if (pQueryInfo->buf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; From ee120bc1848b8cb4920458da64f62d58f68527a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 1 Jul 2021 10:58:39 +0800 Subject: [PATCH 27/29] [TD-4987]: issues memory sanitizier detected. (#6677) * [TD-4987]: issues memory sanitizier detected. * fix mask width issue. From a34b5785ddfafd7f57a63d353cbf134c0b4323c4 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 11:59:07 +0800 Subject: [PATCH 28/29] fix jenkins error --- tests/Jenkinsfile | 4 ++-- tests/pytest/crash_gen/valgrind_taos.supp | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 0c1f651059..a93b412ab9 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -110,8 +110,8 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single -DskipTests >/dev/null - java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + mvn clean package >/dev/null + java -jar target/JdbcRestfulDemo-jar-with-dependencies.jar ''' } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index b42015a053..376567b7e8 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17722,4 +17722,24 @@ fun:_PyEval_EvalCodeWithName fun:_PyFunction_Vectorcall fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:__libc_alloc_buffer_allocate + fun:alloc_buffer_allocate + fun:__resolv_conf_allocate + fun:__resolv_conf_load + fun:__resolv_conf_get_current + fun:__res_vinit + fun:maybe_init + fun:context_get + fun:__resolv_context_get + fun:gaih_inet.constprop.7 + fun:getaddrinfo + fun:taosGetFqdn + fun:taosCheckGlobalCfg + fun:taos_init_imp } \ No newline at end of file From 824a7c40225ba338d7271f8b126a3e5928a370a0 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Thu, 1 Jul 2021 15:33:12 +0800 Subject: [PATCH 29/29] [TD-4989]: fix stmt_bind_param error for unsigned types (#6692) --- src/client/src/tscPrepare.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 8bb776ffee..b75088ba28 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -745,19 +745,23 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, switch(param->type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: size = 1; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: size = 2; break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_FLOAT: size = 4; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: size = 8;