From 0913d4553e1e33026807a04e2f82e2d31c6e8378 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 11 Apr 2021 09:55:20 +0800 Subject: [PATCH 01/35] [TD-3741]: python test case on arm32. (#5767) * [TD-3741]: python test case on arm32. select * with lots data lead timeout. * [TD-3741]: python test case on arm32. change importDataLastS.py select * to count. --- tests/pytest/import_merge/importDataLastS.py | 8 ++++---- tests/pytest/insert/boundary2.py | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py index 2c1559d290..4e5701deb1 100644 --- a/tests/pytest/import_merge/importDataLastS.py +++ b/tests/pytest/import_merge/importDataLastS.py @@ -54,8 +54,8 @@ class TDTestCase: tdSql.execute(" ".join(sqlcmd)) tdLog.info("================= step3") - tdSql.query('select * from tb1') - tdSql.checkRows(205) + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, 205) tdLog.info("================= step4") tdDnodes.stop(1) @@ -71,8 +71,8 @@ class TDTestCase: tdSql.execute(" ".join(sqlcmd)) tdLog.info("================= step6") - tdSql.query('select * from tb1') - tdSql.checkRows(250) + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, 250) def stop(self): tdSql.close() diff --git a/tests/pytest/insert/boundary2.py b/tests/pytest/insert/boundary2.py index 99784b7cd9..8a6fd1e6a1 100644 --- a/tests/pytest/insert/boundary2.py +++ b/tests/pytest/insert/boundary2.py @@ -50,14 +50,16 @@ class TDTestCase: sql += "'%s')" % self.get_random_string(22) tdSql.execute(sql % (self.ts + i)) - tdSql.query("select * from stb") - tdSql.checkRows(4096) + time.sleep(10) + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 4096) tdDnodes.stop(1) tdDnodes.start(1) - - tdSql.query("select * from stb") - tdSql.checkRows(4096) + + time.sleep(1) + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 4096) endTime = time.time() From fd5cebdf341c569362284e01863691ccb17b5820 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 11 Apr 2021 09:55:59 +0800 Subject: [PATCH 02/35] Hotfix/sangshuduo/td 3733 for develop (#5768) * [TD-3733]: taosdemo buffer processing refactor. * [TD-3733]: taosdemo generate SQL head buffer process. * [TD-3733]: taosdemo generate sql head buffer processing. fix compile issue for windows. * [TD-3733]: taosdemo generate sql head buffer processing. fix max_sql_len. generate data buffer refactor. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 208 ++++++++++++++++++++++-------------- 1 file changed, 127 insertions(+), 81 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c6d5933dc7..ed2b74c973 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2949,9 +2949,6 @@ static int readSampleFromCsvFileToMem( continue; } - verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen, - superTblInfo->lenOfOneRow, getRows); - memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); getRows++; @@ -3214,9 +3211,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // rows per table need be less than insert batch if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n", + printf("NOTICE: interlace rows value %d > num_of_records_per_req %d\n\n", g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + printf(" interlace rows value will be set to num_of_records_per_req %d\n\n", g_args.num_of_RPR); printf(" press Enter key to continue or Ctrl-C to stop."); (void)getchar(); @@ -3527,6 +3524,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + /* cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; @@ -3536,6 +3534,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } + */ cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists @@ -3683,14 +3682,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { int32_t len = maxSqlLen->valueint; if (len > TSDB_MAX_ALLOWED_SQL_LEN) { len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < TSDB_MAX_SQL_LEN) { - len = TSDB_MAX_SQL_LEN; + } else if (len < 5) { + len = 5; } g_Dbs.db[i].superTbls[j].maxSqlLen = len; } else if (!maxSqlLen) { g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; } else { - printf("ERROR: failed to read json, maxSqlLen not found\n"); + errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3716,9 +3716,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint; // rows per table need be less than insert batch if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n", + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_req %d\n\n", i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + printf(" interlace rows value will be set to num_of_records_per_req %d\n\n", g_args.num_of_RPR); printf(" press Enter key to continue or Ctrl-C to stop."); (void)getchar(); @@ -4512,13 +4512,15 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) } } -static int generateDataTail(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, +static int generateDataTail( + SSuperTable* superTblInfo, int batch, char* buffer, int remainderBufLen, int64_t insertRows, int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) { int len = 0; int ncols_per_record = 1; // count first col ts + char *pstr = buffer; + if (superTblInfo == NULL) { int datatypeSeq = 0; while(g_args.datatype[datatypeSeq]) { @@ -4547,15 +4549,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq, pSamplePos); } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - int rand_num = taosRandom() % 100; - int randTail; - if (0 != superTblInfo->disorderRatio - && rand_num < superTblInfo->disorderRatio) { - randTail = (superTblInfo->timeStampStep * k - + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); - debugPrint("rand data generated, back %d\n", randTail); - } else { - randTail = superTblInfo->timeStampStep * k; + + int randTail = superTblInfo->timeStampStep * k; + if (superTblInfo->disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if(rand_num < superTblInfo->disorderRatio) { + randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %d\n", randTail); + } } uint64_t d = startTime @@ -4570,7 +4571,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq, break; } - buffer += snprintf(buffer, retLen + 1, "%s", data); + pstr += snprintf(pstr , retLen + 1, "%s", data); k++; len += retLen; remainderBufLen -= retLen; @@ -4598,7 +4599,7 @@ static int generateDataTail(char *tableName, int32_t tableSeq, if (len > remainderBufLen) break; - buffer += sprintf(buffer, " %s", data); + pstr += sprintf(pstr, " %s", data); k++; len += retLen; remainderBufLen -= retLen; @@ -4619,9 +4620,14 @@ static int generateDataTail(char *tableName, int32_t tableSeq, } static int generateSQLHead(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer) + threadInfo* pThreadInfo, SSuperTable* superTblInfo, + char *buffer, int remainderBufLen) { int len; + +#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. + char headBuf[HEAD_BUFF_LEN]; + if (superTblInfo) { if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; @@ -4638,8 +4644,9 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, return -1; } - len = snprintf(buffer, - superTblInfo->maxSqlLen, + len = snprintf( + headBuf, + HEAD_BUFF_LEN, "insert into %s.%s using %s.%s tags %s values", pThreadInfo->db_name, tableName, @@ -4648,30 +4655,93 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, tagsValBuf); tmfree(tagsValBuf); } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - len = snprintf(buffer, - superTblInfo->maxSqlLen, + len = snprintf( + headBuf, + HEAD_BUFF_LEN, "insert into %s.%s values", pThreadInfo->db_name, tableName); } else { - len = snprintf(buffer, - superTblInfo->maxSqlLen, + len = snprintf( + headBuf, + HEAD_BUFF_LEN, "insert into %s.%s values", pThreadInfo->db_name, tableName); } } else { - len = snprintf(buffer, - g_args.max_sql_len, + len = snprintf( + headBuf, + HEAD_BUFF_LEN, "insert into %s.%s values", pThreadInfo->db_name, tableName); } + if (len > remainderBufLen) + return -1; + + tstrncpy(buffer, headBuf, len + 1); + return len; } -static int generateProgressiveDataBuffer(char *pTblName, +static int generateInterlaceDataBuffer( + char *tableName, int batchPerTbl, int i, int batchPerTblTimes, + int32_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + int64_t startTime, + int *pRemainderBufLen) +{ + char *pstr = buffer; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + + int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, + superTblInfo, pstr, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, i, buffer); + + pstr += headLen; + *pRemainderBufLen -= headLen; + + int dataLen = 0; + + verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", + pThreadInfo->threadID, __func__, __LINE__, + i, batchPerTblTimes, batchPerTbl); + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } + } else { + startTime = 1500000000000; + } + int k = generateDataTail( + superTblInfo, + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); + + if (k > 0) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + pstr -= headLen; + pstr[0] = '\0'; + } + + return k; +} + +static int generateProgressiveDataBuffer( + char *tableName, int32_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, @@ -4691,6 +4761,7 @@ static int generateProgressiveDataBuffer(char *pTblName, assert(buffer != NULL); + int k = 0; int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; int remainderBufLen = maxSqlLen; @@ -4698,14 +4769,17 @@ static int generateProgressiveDataBuffer(char *pTblName, char *pstr = buffer; - int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo, - buffer); + int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, + buffer, remainderBufLen); + + if (headLen <= 0) { + return 0; + } pstr += headLen; remainderBufLen -= headLen; - int k; int dataLen; - k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, + k = generateDataTail(superTblInfo, g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); @@ -4722,8 +4796,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; - if (interlaceRows > insertRows) - interlaceRows = insertRows; + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; int insertMode; @@ -4788,8 +4862,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { bool flagSleep = true; int sleepTimeTotal = 0; - int remainderBufLen; - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); @@ -4797,7 +4869,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } // generate data memset(buffer, 0, maxSqlLen); - remainderBufLen = maxSqlLen; + int remainderBufLen = maxSqlLen; char *pstr = buffer; int recOfBatch = 0; @@ -4811,50 +4883,23 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { return NULL; } - int headLen; - if (i == 0) { - headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, - superTblInfo, pstr); - } else { - headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values", - pThreadInfo->db_name, - tableName); - } - - // generate data buffer - verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n", - pThreadInfo->threadID, __func__, __LINE__, i, buffer); - - pstr += headLen; - remainderBufLen -= headLen; - - int dataLen = 0; - - verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", - pThreadInfo->threadID, __func__, __LINE__, - i, batchPerTblTimes, batchPerTbl); - - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); - } - } else { - startTime = 1500000000000; - } - int generated = generateDataTail( - tableName, tableSeq, pThreadInfo, superTblInfo, - batchPerTbl, pstr, remainderBufLen, insertRows, 0, + int generated = generateInterlaceDataBuffer( + tableName, batchPerTbl, i, batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, startTime, - &(pThreadInfo->samplePos), &dataLen); + &remainderBufLen); if (generated < 0) { debugPrint("[%d] %s() LN%d, generated data is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); goto free_and_statistics_interlace; + } else if (generated == 0) { + break; } - pstr += dataLen; - remainderBufLen -= dataLen; + tableSeq ++; recOfBatch += batchPerTbl; // startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; @@ -4862,7 +4907,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - tableSeq ++; if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table @@ -5071,11 +5115,13 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { */ } // num_of_DPT - if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && + if (g_args.verbose_print) { + if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && (0 == strncasecmp( superTblInfo->dataSource, "sample", strlen("sample")))) { - printf("%s() LN%d samplePos=%d\n", + verbosePrint("%s() LN%d samplePos=%d\n", __func__, __LINE__, pThreadInfo->samplePos); + } } } // tableSeq From 76db6804e4ea28f44cf896a5ad8204d937a938fc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 11 Apr 2021 20:44:12 +0800 Subject: [PATCH 03/35] Feature/sangshuduo/td 3317 taosdemo interlace (#5773) * [TD-3316] : add testcase for taosdemo limit and offset. check offset 0. * [TD-3316] : add testcase for taosdemo limit and offset. fix sample file import bug. * [TD-3316] : add test case for limit and offset. fix sample data issue. * [TD-3327] : fix taosdemo segfault when import data from sample data file. * [TD-3317] : make taosdemo support interlace mode. json parameter rows_per_tbl support. * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode. refactor * [TD-3317] : support interlace mode insertion. refactor. * [TD-3317] : support interlace mode insertion. change json file. * [TD-3317] : support interlace mode insertion. fix multithread create table regression. * [TD-3317] : support interlace mode insertion. working but not perfect. * [TD-3317] : support interlace mode insertion. rename lowaTest with taosdemoTestWithJson * [TD-3317] : support interlace mode insertion. perfect * [TD-3317] : support interlace mode insertion. cleanup. * [TD-3317] : support interlace mode insertion. adjust algorithm of loop times. * [TD-3317] : support interlace mode insertion. fix delay time bug. * [TD-3317] : support interlace mode insertion. fix progressive timestamp bug. * [TD-3317] : support interlace mode insertion. add an option for performance print. * [TD-3317] : support interlace mode insertion. change json test case with less table for acceleration. * [TD-3317] : support interlace mode insertion. change progressive mode timestamp step and testcase. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3317] : support interlace mode insertion. add time shift for no sleep time. * [TD-3317] : support interlace insert. rework timestamp. * [TD-3317] : support interlace mode insertion. change rows_per_tbl to interlace_rows. * [TD-3317] : taosdemo suppoert interlace mode. remove trailing spaces. * [TD-3317] : taosdemo support interlace insertion. prompt if interlace > num_of_records_per_req * fill insert-into early to buffer. * fix buffer overflow issue. * change rows_per_tbl to interlace_rows to align with taosdemo. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 74 +++++++++++++++-------- tests/pytest/tools/taosdemoPerformance.py | 4 +- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ed2b74c973..49e550cc01 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -18,6 +18,7 @@ when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. */ +#include #define _GNU_SOURCE #define CURL_STATICLIB @@ -3242,7 +3243,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (numRecPerReq && numRecPerReq->type == cJSON_Number) { g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_args.num_of_RPR = 0xffff; + g_args.num_of_RPR = INT32_MAX; } else { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); @@ -4647,7 +4648,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, len = snprintf( headBuf, HEAD_BUFF_LEN, - "insert into %s.%s using %s.%s tags %s values", + "%s.%s using %s.%s tags %s values", pThreadInfo->db_name, tableName, pThreadInfo->db_name, @@ -4658,14 +4659,14 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, len = snprintf( headBuf, HEAD_BUFF_LEN, - "insert into %s.%s values", + "%s.%s values", pThreadInfo->db_name, tableName); } else { len = snprintf( headBuf, HEAD_BUFF_LEN, - "insert into %s.%s values", + "%s.%s values", pThreadInfo->db_name, tableName); } @@ -4673,7 +4674,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, len = snprintf( headBuf, HEAD_BUFF_LEN, - "insert into %s.%s values", + "%s.%s values", pThreadInfo->db_name, tableName); } @@ -4694,6 +4695,7 @@ static int generateInterlaceDataBuffer( int64_t startTime, int *pRemainderBufLen) { + assert(buffer); char *pstr = buffer; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -4723,18 +4725,20 @@ static int generateInterlaceDataBuffer( } else { startTime = 1500000000000; } + int k = generateDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - if (k > 0) { + if (k == batchPerTbl) { pstr += dataLen; *pRemainderBufLen -= dataLen; } else { pstr -= headLen; pstr[0] = '\0'; + k = 0; } return k; @@ -4745,7 +4749,8 @@ static int generateProgressiveDataBuffer( int32_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, - int64_t startFrom, int64_t startTime, int *pSamplePos) + int64_t startFrom, int64_t startTime, int *pSamplePos, + int *pRemainderBufLen) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -4760,27 +4765,24 @@ static int generateProgressiveDataBuffer( } assert(buffer != NULL); - - int k = 0; - int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - int remainderBufLen = maxSqlLen; - - memset(buffer, 0, maxSqlLen); - char *pstr = buffer; + int k = 0; + + memset(buffer, 0, *pRemainderBufLen); + int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, - buffer, remainderBufLen); + buffer, *pRemainderBufLen); if (headLen <= 0) { return 0; } pstr += headLen; - remainderBufLen -= headLen; + *pRemainderBufLen -= headLen; int dataLen; k = generateDataTail(superTblInfo, - g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, + g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); @@ -4842,18 +4844,17 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int64_t startTime = pThreadInfo->start_time; - int batchPerTblTimes; - int batchPerTbl; - assert(pThreadInfo->ntables > 0); if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; - batchPerTbl = interlaceRows; + int batchPerTbl = interlaceRows; + + int batchPerTblTimes; if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { batchPerTblTimes = - (g_args.num_of_RPR / (interlaceRows * pThreadInfo->ntables)) + 1; + g_args.num_of_RPR / interlaceRows; } else { batchPerTblTimes = 1; } @@ -4862,6 +4863,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { bool flagSleep = true; int sleepTimeTotal = 0; + char *strInsertInto = "insert into "; + int nInsertBufLen = strlen(strInsertInto); + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); @@ -4872,6 +4876,11 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int remainderBufLen = maxSqlLen; char *pstr = buffer; + + int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); + pstr += len; + remainderBufLen -= len; + int recOfBatch = 0; for (int i = 0; i < batchPerTblTimes; i ++) { @@ -4883,6 +4892,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { return NULL; } + int oldRemainderLen = remainderBufLen; int generated = generateInterlaceDataBuffer( tableName, batchPerTbl, i, batchPerTblTimes, tableSeq, @@ -4901,6 +4911,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { tableSeq ++; recOfBatch += batchPerTbl; + pstr += (oldRemainderLen - remainderBufLen); // startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", @@ -5012,11 +5023,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); + char* buffer = calloc(maxSqlLen, 1); if (NULL == buffer) { errorPrint( "Failed to alloc %d Bytes, reason:%s\n", - superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, + maxSqlLen, strerror(errno)); return NULL; } @@ -5059,10 +5071,20 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); + int remainderBufLen = maxSqlLen; + char *pstr = buffer; + int nInsertBufLen = strlen("insert into "); + + int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); + + pstr += len; + remainderBufLen -= len; + int generated = generateProgressiveDataBuffer( - tableName, tableSeq, pThreadInfo, buffer, insertRows, + tableName, tableSeq, pThreadInfo, pstr, insertRows, i, start_time, - &(pThreadInfo->samplePos)); + &(pThreadInfo->samplePos), + &remainderBufLen); if (generated > 0) i += generated; else diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 1156cc2484..8ce99d0969 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -51,7 +51,7 @@ class taosdemoPerformace: "insert_rows": 100000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 100, + "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, @@ -159,4 +159,4 @@ if __name__ == '__main__': perftest = taosdemoPerformace(args.commit_id, args.database_name) perftest.insertData() - perftest.createTablesAndStoreData() \ No newline at end of file + perftest.createTablesAndStoreData() From f1f6a572452cf3bda2daba178246fbe34ecb2639 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Mon, 12 Apr 2021 10:14:06 +0800 Subject: [PATCH 04/35] Hotfix/td 3689 (#5754) * [TD-3689]: fix setNull exception in JDBC * change * change --- .../taosdata/jdbc/TSDBPreparedStatement.java | 137 +----------------- .../jdbc/TSDBPreparedStatementTest.java | 47 +++++- .../com/taosdata/jdbc/TSDBResultSetTest.java | 1 + .../jdbc/cases/DriverAutoloadTest.java | 2 + .../jdbc/rs/RestfulPreparedStatementTest.java | 47 +++++- .../jdbc/rs/RestfulResultSetTest.java | 1 + 6 files changed, 102 insertions(+), 133 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 728b537f71..e545bbc8f2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -33,17 +33,9 @@ import java.util.regex.Pattern; public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { private String rawSql; - private String sql; - // private ArrayList parameters = new ArrayList<>(); private Object[] parameters; private boolean isPrepared; - //start with insert or import and is case-insensitive - private static Pattern savePattern = Pattern.compile("(?i)^\\s*(insert|import)"); - // is insert or import - private boolean isSaved; - - // private SavedPreparedStatement savedPreparedStatement; private volatile TSDBParameterMetaData parameterMetaData; TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connecter, String sql) { @@ -65,35 +57,11 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private void init(String sql) { this.rawSql = sql; preprocessSql(); -// this.isSaved = isSavedSql(this.rawSql); -// if (this.isSaved) { -// try { -// this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); -// } catch (SQLException e) { -// e.printStackTrace(); -// } -// } - - } - - /** - * if the precompiled sql is insert or import - * - * @param sql - * @return - */ - private boolean isSavedSql(String sql) { - Matcher matcher = savePattern.matcher(sql); - return matcher.find(); } @Override public int[] executeBatch() throws SQLException { -// if (isSaved) { -// return this.savedPreparedStatement.executeBatch(); -// } else { return super.executeBatch(); -// } } /* @@ -157,23 +125,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat * * @return a string of the native sql statement for TSDB */ -// private String getNativeSql(String rawSql) { -// for (int i = 0; i < parameters.length; i++) { -// Object para = parameters[i]; -// if (para != null) { -// String paraStr = para.toString(); -// if (para instanceof Timestamp || para instanceof String) { -// paraStr = "'" + paraStr + "'"; -// } -// this.sql = this.sql.replaceFirst("[?]", paraStr); -// } else { -// this.sql = this.sql.replaceFirst("[?]", "NULL"); -// } -// } -// parameters = new Object[parameters.length]; -// return sql; -// } - private String getNativeSql(String rawSql) throws SQLException { String sql = rawSql; for (int i = 0; i < parameters.length; ++i) { @@ -201,108 +152,58 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public ResultSet executeQuery() throws SQLException { -// if (isSaved) { -// this.savedPreparedStatement.executeBatchInternal(); -// return null; -// } else { - if (!isPrepared) return executeQuery(this.rawSql); final String sql = getNativeSql(this.rawSql); return executeQuery(sql); -// } } @Override public int executeUpdate() throws SQLException { -// if (isSaved) { -// return this.savedPreparedStatement.executeBatchInternal(); -// } else { if (!isPrepared) return executeUpdate(this.rawSql); String sql = getNativeSql(this.rawSql); return executeUpdate(sql); -// } - } - - private boolean isSupportedSQLType(int sqlType) { - switch (sqlType) { - case Types.TIMESTAMP: - case Types.INTEGER: - case Types.BIGINT: - case Types.FLOAT: - case Types.DOUBLE: - case Types.SMALLINT: - case Types.TINYINT: - case Types.BOOLEAN: - case Types.BINARY: - case Types.NCHAR: - return true; - default: - return false; - } } @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - if (!isSupportedSQLType(sqlType) || parameterIndex < 0) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); -// if (parameterIndex >= parameters.size()) -// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_BOUNDARY); - - setObject(parameterIndex, "NULL"); + setObject(parameterIndex, null); } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - setObject(parameterIndex, x); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - setObject(parameterIndex,x); + setObject(parameterIndex, x); } @Override public void setShort(int parameterIndex, short x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setInt(int parameterIndex, int x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setLong(int parameterIndex, long x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @@ -315,17 +216,12 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setString(int parameterIndex, String x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - - setObject(parameterIndex,x); + setObject(parameterIndex, x); } @Override @@ -344,8 +240,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @@ -360,7 +254,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @@ -375,8 +268,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void clearParameters() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - -// parameters.clear(); parameters = new Object[parameters.length]; } @@ -384,43 +275,29 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + setObject(parameterIndex,x); } @Override public void setObject(int parameterIndex, Object x) throws SQLException { -// if (isSaved) { -// this.savedPreparedStatement.setParam(parameterIndex, x); -// } else { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); if (parameterIndex < 1 && parameterIndex >= parameters.length) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); - parameters[parameterIndex - 1] = x; -// parameters.add(x); -// } } @Override public boolean execute() throws SQLException { -// if (isSaved) { -// int result = this.savedPreparedStatement.executeBatchInternal(); -// return result > 0; -// } else { if (!isPrepared) return execute(this.rawSql); final String sql = getNativeSql(this.rawSql); - return execute(sql); -// } } @Override public void addBatch() throws SQLException { -// if (isSaved) { -// this.savedPreparedStatement.addBatch(); -// } else { if (this.batchedArgs == null) { batchedArgs = new ArrayList<>(); } @@ -431,7 +308,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat String sql = this.getConnection().nativeSQL(this.rawSql); addBatch(sql); } -// } } @Override @@ -475,7 +351,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat public ResultSetMetaData getMetaData() throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); -// return this.getResultSet().getMetaData(); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 434095efa2..dc6fd4c501 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -50,6 +50,51 @@ public class TSDBPreparedStatementTest { pstmt_insert.setNull(2, Types.INTEGER); int result = pstmt_insert.executeUpdate(); Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(3, Types.BIGINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(4, Types.FLOAT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(5, Types.DOUBLE); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(6, Types.SMALLINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(7, Types.TINYINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(8, Types.BOOLEAN); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(9, Types.BINARY); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(10, Types.NCHAR); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(10, Types.OTHER); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); } @Test @@ -129,7 +174,7 @@ public class TSDBPreparedStatementTest { Assert.assertFalse(pstmt_insert.execute()); } - class Person implements Serializable { + class Person { String name; int age; boolean sex; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java index 374b1335fc..c5c6f7bca5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java @@ -160,6 +160,7 @@ public class TSDBResultSetTest { @Test public void getTime() throws SQLException { Time f1 = rs.getTime("f1"); + Assert.assertNotNull(f1); Assert.assertEquals("00:00:00", f1.toString()); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java index 9826e6ed76..6c8aed1b06 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -20,6 +20,7 @@ public class DriverAutoloadTest { final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); Assert.assertNotNull(conn); + conn.close(); } @Test @@ -27,6 +28,7 @@ public class DriverAutoloadTest { final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); Assert.assertNotNull(conn); + conn.close(); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java index 094dff8c8d..40956a601f 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -50,6 +50,51 @@ public class RestfulPreparedStatementTest { pstmt_insert.setNull(2, Types.INTEGER); int result = pstmt_insert.executeUpdate(); Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(3, Types.BIGINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(4, Types.FLOAT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(5, Types.DOUBLE); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(6, Types.SMALLINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(7, Types.TINYINT); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(8, Types.BOOLEAN); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(9, Types.BINARY); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(10, Types.NCHAR); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(10, Types.OTHER); + result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); } @Test @@ -129,7 +174,7 @@ public class RestfulPreparedStatementTest { Assert.assertFalse(pstmt_insert.execute()); } - class Person implements Serializable { + private class Person { String name; int age; boolean sex; diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index d6b2a58127..9bfe9a04ff 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -160,6 +160,7 @@ public class RestfulResultSetTest { @Test public void getTime() throws SQLException { Time f1 = rs.getTime("f1"); + Assert.assertNotNull(f1); Assert.assertEquals("00:00:00", f1.toString()); } From 74487212fe13f49493e85d25e562fc99ab1d7037 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 12 Apr 2021 10:49:56 +0800 Subject: [PATCH 05/35] [TD-3606] : locale & charset options will not be checked when dnode add to cluster. --- documentation20/cn/10.cluster/docs.md | 9 ++++----- documentation20/cn/11.administrator/docs.md | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/documentation20/cn/10.cluster/docs.md b/documentation20/cn/10.cluster/docs.md index a430ce8277..62d709c279 100644 --- a/documentation20/cn/10.cluster/docs.md +++ b/documentation20/cn/10.cluster/docs.md @@ -55,12 +55,11 @@ arbitrator ha.taosdata.com:6042 | 4 | statusInterval | dnode向mnode报告状态时长 | | 5 | arbitrator | 系统中裁决器的end point | | 6 | timezone | 时区 | -| 7 | locale | 系统区位信息及编码格式 | -| 8 | charset | 字符集编码 | -| 9 | balance | 是否启动负载均衡 | -| 10 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 | -| 11 | maxVgroupsPerDb | 每个DB中能够使用的最大vgroup个数 | +| 7 | balance | 是否启动负载均衡 | +| 8 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 | +| 9 | maxVgroupsPerDb | 每个DB中能够使用的最大vgroup个数 | +备注：在 2.0.19.0 及更早的版本中，除以上 9 项参数外，dnode 加入集群时，还会要求 locale 和 charset 参数的取值也一致。 ## 启动第一个数据节点 diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index cc8689786d..961a3801e7 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -150,7 +150,7 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 - maxVgroupsPerDb: 每个数据库中能够使用的最大vgroup个数。 - arbitrator: 系统中裁决器的end point，缺省为空。 -- timezone、locale、charset 的配置见客户端配置。 +- timezone、locale、charset 的配置见客户端配置。（2.0.19.1 及以上的版本里，集群中加入新节点已不要求 locale 和 charset 参数取值一致） 为方便调试，可通过SQL语句临时调整每个dnode的日志配置，系统重启后会失效： From 16e145b73ddcb7e820cfbf2ce7af0e843e55bc01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 12 Apr 2021 11:19:52 +0800 Subject: [PATCH 06/35] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5776) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49e550cc01..7037f1015b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -703,7 +703,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { if (strcmp(argv[i], "-f") == 0) { arguments->metaFile = argv[++i]; } else if (strcmp(argv[i], "-c") == 0) { - tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN); + tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); } else if (strcmp(argv[i], "-h") == 0) { arguments->host = argv[++i]; From 72ab7e8abc6ce3acfa83d71c56c416e4a7394d9f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 12 Apr 2021 11:37:48 +0800 Subject: [PATCH 07/35] [TD-3751]update Jenkins mail notification --- tests/Jenkinsfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 178a0446c3..afd0b25338 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -227,11 +227,11 @@ pipeline { } } - post { + post { success { emailext ( - subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ @@ -247,29 +247,29 @@ pipeline {
    -
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建名称>>分支：${env.BRANCH_NAME}
  • 构建结果： Successful
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用户：${CAUSE}
    • -
  • 变更概要：${CHANGES}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
  • 构建地址：${BUILD_URL}
  • 构建日志：${BUILD_URL}console
    • -
  • 变更集：${JELLY_SCRIPT}
    • +
- ''', + """, to: "yqliu@taosdata.com,pxiao@taosdata.com", from: "support@taosdata.com" ) } failure { emailext ( - subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ @@ -285,21 +285,21 @@ pipeline {
    -
  • 构建名称>>分支：${PROJECT_NAME}
    • -
  • 构建结果： Successful
    • +
  • 构建名称>>分支：${env.BRANCH_NAME}
    • +
  • 构建结果： Failure
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用户：${CAUSE}
    • -
  • 变更概要：${CHANGES}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
  • 构建地址：${BUILD_URL}
  • 构建日志：${BUILD_URL}console
    • -
  • 变更集：${JELLY_SCRIPT}
    • +
- ''', + """, to: "yqliu@taosdata.com,pxiao@taosdata.com", from: "support@taosdata.com" ) From 802869e5b9154e2b495740e3d0a488f88cab6e2f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 12 Apr 2021 13:19:17 +0800 Subject: [PATCH 08/35] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5779) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. * [TD-3197]: coverity scan issue. taosdump argument validation. Co-authored-by: Shuduo Sang --- src/kit/taosdump/taosdump.c | 41 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index fd6ee9f2fc..092374cef1 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -483,28 +483,33 @@ static int queryDbImpl(TAOS *taos, char *command) { static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-E") == 0) { - char *tmp = strdup(argv[++i]); + if (argv[i+1]) { + char *tmp = strdup(argv[++i]); - if (tmp) { - int64_t tmpEpoch; - if (strchr(tmp, ':') && strchr(tmp, '-')) { - if (TSDB_CODE_SUCCESS != taosParseTime( - tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { - fprintf(stderr, "Input end time error!\n"); - free(tmp); - return; + if (tmp) { + int64_t tmpEpoch; + if (strchr(tmp, ':') && strchr(tmp, '-')) { + if (TSDB_CODE_SUCCESS != taosParseTime( + tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { + fprintf(stderr, "Input end time error!\n"); + free(tmp); + return; + } + } else { + tmpEpoch = atoll(tmp); } - } else { - tmpEpoch = atoll(tmp); - } - - sprintf(argv[i], "%"PRId64"", tmpEpoch); - debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", - __func__, __LINE__, tmp, i, argv[i]); - free(tmp); + sprintf(argv[i], "%"PRId64"", tmpEpoch); + debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", + __func__, __LINE__, tmp, i, argv[i]); + + free(tmp); + } else { + errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); + exit(-1); + } } else { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); + errorPrint("%s() LN%d, -E need a valid value following!\n", __func__, __LINE__); exit(-1); } } else if (strcmp(argv[i], "-g") == 0) { From 4876811b9de1dd9bdacc18f342ac96d6a8d6f2f7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 12 Apr 2021 18:12:21 +0800 Subject: [PATCH 09/35] Hotfix/sangshuduo/td 3683 taosdemo lots tables (#5783) * [TD-3683]: reduce buffer size for more stable table creation. * [TD-3683]: taosdemo create lots of tables. clean up. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/insert-interlace.json | 1 - src/kit/taosdemo/insert.json | 1 - src/kit/taosdemo/taosdemo.c | 6 ++---- tests/examples/JDBC/taosdemo/src/main/resources/insert.json | 2 -- tests/pytest/cluster/clusterEnvSetup/insert.json | 2 -- tests/pytest/perfbenchmark/bug3433.py | 4 ---- tests/pytest/query/query1970YearsAf.py | 4 +--- tests/pytest/tools/insert-interlace.json | 1 - tests/pytest/tools/insert-tblimit-tboffset-createdb.json | 2 -- tests/pytest/tools/insert-tblimit-tboffset-insertrec.json | 2 -- tests/pytest/tools/insert-tblimit-tboffset.json | 2 -- tests/pytest/tools/insert-tblimit-tboffset0.json | 2 -- tests/pytest/tools/insert-tblimit1-tboffset.json | 2 -- tests/pytest/tools/taosdemo-sampledata.json | 2 -- tests/pytest/tools/taosdemoPerformance.py | 2 -- 15 files changed, 3 insertions(+), 32 deletions(-) diff --git a/src/kit/taosdemo/insert-interlace.json b/src/kit/taosdemo/insert-interlace.json index 344db4fd00..cf3e1de2f4 100644 --- a/src/kit/taosdemo/insert-interlace.json +++ b/src/kit/taosdemo/insert-interlace.json @@ -40,7 +40,6 @@ "data_source": "rand", "insert_mode": "taosc", "insert_rows": 1000, - "multi_thread_write_one_tbl": "no", "interlace_rows": 20, "max_sql_len": 1024000, "disorder_ratio": 0, diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index f0e3ab1d50..5f152a2d0c 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -40,7 +40,6 @@ "data_source": "rand", "insert_mode": "taosc", "insert_rows": 100000, - "multi_thread_write_one_tbl": "no", "interlace_rows": 0, "max_sql_len": 1024000, "disorder_ratio": 0, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7037f1015b..f4158e6332 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3525,7 +3525,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - /* cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; @@ -3535,7 +3534,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } - */ cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists @@ -3694,7 +3692,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } - +/* cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes if (multiThreadWriteOneTbl @@ -3711,7 +3709,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); goto PARSE_OVER; } - +*/ cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); if (interlaceRows && interlaceRows->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint; diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json index 35c7773175..7578083d33 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json +++ b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json @@ -36,8 +36,6 @@ "insert_mode": "taosc", "insert_rate": 0, "insert_rows": 100, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "interlace_rows": 3, "max_sql_len": 1024, "disorder_ratio": 0, diff --git a/tests/pytest/cluster/clusterEnvSetup/insert.json b/tests/pytest/cluster/clusterEnvSetup/insert.json index 4548ef74e6..2f3cf0f0d9 100644 --- a/tests/pytest/cluster/clusterEnvSetup/insert.json +++ b/tests/pytest/cluster/clusterEnvSetup/insert.json @@ -37,8 +37,6 @@ "insert_mode": "taosc", "insert_rate": 0, "insert_rows": 100000, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 1, "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index 80e9cce0dc..e4480df6b6 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -91,8 +91,6 @@ class TDTestCase: "data_source": "rand", "insert_mode": "taosc", "insert_rows": 1, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, @@ -135,8 +133,6 @@ class TDTestCase: "data_source": "rand", "insert_mode": "taosc", "insert_rows": 1, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index e66a79f5e6..93404afd59 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -82,8 +82,6 @@ class TDTestCase: "data_source": "rand", "insert_mode": "taosc", "insert_rows": 5000, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, @@ -255,4 +253,4 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/tools/insert-interlace.json b/tests/pytest/tools/insert-interlace.json index d4767ad064..a5c545d159 100644 --- a/tests/pytest/tools/insert-interlace.json +++ b/tests/pytest/tools/insert-interlace.json @@ -41,7 +41,6 @@ "data_source": "rand", "insert_mode": "taosc", "insert_rows": 250, - "multi_thread_write_one_tbl": "no", "interlace_rows": 80, "max_sql_len": 1024000, "disorder_ratio": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json index 8c3b39fb0b..9220bc1d17 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json @@ -40,8 +40,6 @@ "data_source": "rand", "insert_mode": "taosc", "insert_rows": 0, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json index a9efa7c31c..164d4fe8be 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json @@ -42,8 +42,6 @@ "insert_rows": 1000, "childtable_limit": 33, "childtable_offset": 33, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/insert-tblimit-tboffset.json b/tests/pytest/tools/insert-tblimit-tboffset.json index f3d3e864ba..0b8e0bd6c5 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset.json +++ b/tests/pytest/tools/insert-tblimit-tboffset.json @@ -42,8 +42,6 @@ "insert_rows": 1000, "childtable_limit": 33, "childtable_offset": 33, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json index 6bf3783cb2..55d9e19055 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset0.json +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -42,8 +42,6 @@ "insert_rows": 1000, "childtable_limit": 20, "childtable_offset": 0, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json index 292902093d..3a88665661 100644 --- a/tests/pytest/tools/insert-tblimit1-tboffset.json +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -42,8 +42,6 @@ "insert_rows": 1000, "childtable_limit": 1, "childtable_offset": 50, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/tools/taosdemo-sampledata.json b/tests/pytest/tools/taosdemo-sampledata.json index d543b7e086..ce1c68b393 100644 --- a/tests/pytest/tools/taosdemo-sampledata.json +++ b/tests/pytest/tools/taosdemo-sampledata.json @@ -22,8 +22,6 @@ "insert_mode": "taosc", "insert_rate": 0, "insert_rows": 20, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "max_sql_len": 1048000, "timestamp_step": 1000, "start_timestamp": "2020-1-1 00:00:00", diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 8ce99d0969..b50180e2b3 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -49,8 +49,6 @@ class taosdemoPerformace: "batch_create_tbl_num": 10, "insert_mode": "taosc", "insert_rows": 100000, - "multi_thread_write_one_tbl": "no", - "number_of_tbl_in_one_sql": 0, "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, From eed1961dc2d3844e6bfd46f2df06d5a7d2e34b44 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 12 Apr 2021 18:58:30 +0800 Subject: [PATCH 10/35] [TD-3754] : correct the meaning of slimit option. --- documentation20/cn/12.taos-sql/docs.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 58191e0bd8..864169b898 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -626,7 +626,8 @@ Query OK, 1 row(s) in set (0.001091s) - WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串 - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。 - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 -- 参数 SLIMIT 控制由 GROUP BY 指令划分的每个分组中的输出条数。 + * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 +- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。 - 通过”>>"输出结果可以导出到指定文件 ### 支持的条件过滤操作 From c344a2b435346db7563eb9649698a893cf3642d1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 12 Apr 2021 21:38:38 +0800 Subject: [PATCH 11/35] Hotfix/sangshuduo/td 3683 for master (#5786) * [TD-3752]: taosdemo columns+tags+ts must less then 1024. (#5782) Co-authored-by: Shuduo Sang * [TD-3683]: taosdemo create lots of tables. clean up. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f4158e6332..02a1ec2b6d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2980,7 +2980,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int columnSize = cJSON_GetArraySize(columns); - if (columnSize > MAX_COLUMN_COUNT) { + if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) { errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", __func__, __LINE__, MAX_COLUMN_COUNT); goto PARSE_OVER; @@ -3038,8 +3038,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } - if (index > MAX_COLUMN_COUNT) { - errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", + if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) { + errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", __func__, __LINE__, MAX_COLUMN_COUNT); goto PARSE_OVER; } @@ -3110,16 +3110,16 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } if (index > MAX_TAG_COUNT) { - errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", + errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", __func__, __LINE__, MAX_TAG_COUNT); goto PARSE_OVER; } superTbls->tagCount = index; - if ((superTbls->columnCount + superTbls->tagCount) > MAX_COLUMN_COUNT) { - errorPrint("%s() LN%d, columns + tags is more than max columns count: %d\n", - __func__, __LINE__, MAX_TAG_COUNT); + if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) { + errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", + __func__, __LINE__, MAX_COLUMN_COUNT); goto PARSE_OVER; } ret = true; From ec582d3996951f93b3a44b828b3cc74ab07b5a8e Mon Sep 17 00:00:00 2001 From: skye0207 <45970022+skye0207@users.noreply.github.com> Date: Tue, 13 Apr 2021 14:18:33 +0800 Subject: [PATCH 12/35] add docker doc --- .../cn/02.getting-started/01.docker/docs.md | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 documentation20/cn/02.getting-started/01.docker/docs.md diff --git a/documentation20/cn/02.getting-started/01.docker/docs.md b/documentation20/cn/02.getting-started/01.docker/docs.md new file mode 100644 index 0000000000..8021a54536 --- /dev/null +++ b/documentation20/cn/02.getting-started/01.docker/docs.md @@ -0,0 +1,226 @@ +# 通过Docker快速体验TDengine + +由于目前TDengine2.0的服务器仅能在Linux系统上安装和运行，如果开发者使用的是Mac或Windows系统，可能需要安装虚拟机或远程连接Linux服务器，来使用TDengine。Docker轻巧快速，可以作为一个可行，经济，高效的替代方案。下文准备一步一步介绍，如何通过Docker快速使用TDengine，可用于开发，测试，和备份数据。 + +  +### 下载Docker + +docker的下载可以参考[docker官网文档Get Docker部分](https://docs.docker.com/get-docker/)。 + +安装成功后可以查看下docker版本。 + +```bash +$ docker -v +Docker version 20.10.5, build 55c4c88 +``` + +  +  +### 在docker容器中运行TDengine + ++ 使用命令拉取tdengine镜像，并使它在后台运行。 + +```bash +$ docker run -d tdengine/tdengine +cdf548465318c6fc2ad97813f89cc60006393392401cae58a27b15ca9171f316 +``` + +  **run**：和前面的docker命令组合起来，运行一个容器。 + +  **-d**：让容器在后台运行。 + +  **tdengine/tdengine**：拉取的TDengine官方发布的应用镜像 + +  **cdf548465318c6fc2ad97813f89cc60006393392401cae58a27b15ca9171f316**： + +  返回的长字符是容器ID，我们可以通过容器ID来查看对应的容器。 + +  ++ 确认容器是否已经运行起来了。 + +```bash +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS ··· +cdf548465318 tdengine/tdengine "taosd" 14 minutes ago Up 14 minutes ··· +``` + +  **docker ps**：列出所有在运行中的容器信息。 + +  **CONTAINER ID**：容器ID。 + +  **IMAGE**：使用的镜像。 + +  **COMMAND**：启动容器时运行的命令。 + +  **CREATED**：容器创建时间。 + +  **STATUS**：容器状态。UP表示运行中。 + +  ++ 进入docker容器内，使用tdengine。 + +```bash +$ docker exec -it cdf548465318 /bin/bash +root@cdf548465318:~/TDengine-server-2.0.13.0# +``` + +  **exec**：通过docker exec命令进入容器，如果退出容器不会停止。 + +  **-i**：进入交互模式。 + +  **-t**：指定一个终端。 + +  **cdf548465318**：容器ID，需要根据具体情况进行修改。 + +  **/bin/bash**：载入容器后运行bash来进行交互。 + +  ++ 进入容器后，执行TDengine命令行程序。 + +```bash +$ root@cdf548465318:~/TDengine-server-2.0.13.0# taos + +Welcome to the TDengine shell from Linux, Client Version:2.0.13.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> +``` + +TDengine终端连接服务成功，打印出了欢迎消息和版本信息。如果失败，会有错误信息打印出来。 + +在TDengine终端中，可以通过SQL命令来创建/删除数据库，表，超级表等，并可以进行插入查询操作。具体可以参考[TDengine官网文档的TAOS SQL部分](https://www.taosdata.com/cn/documentation/taos-sql)。 + +  +  +### 通过taosdemo进一步了解TDengine + ++ 接上面的步骤，先退出TDengine命令行程序。 + +```bash +$ taos> q +root@cdf548465318:~/TDengine-server-2.0.13.0# +``` + +  ++ 在Linux终端执行taosdemo。 + +```bash +$ root@cdf548465318:~/TDengine-server-2.0.13.0# taosdemo +################################################################### +# Server IP: localhost:0 +# User: root +# Password: taosdata +# Use metric: true +# Datatype of Columns: int int int int int int int float +# Binary Length(If applicable): -1 +# Number of Columns per record: 3 +# Number of Threads: 10 +# Number of Tables: 10000 +# Number of Data per Table: 100000 +# Records/Request: 1000 +# Database name: test +# Table prefix: t +# Delete method: 0 +# Test time: 2021-04-13 02:05:20 +################################################################### +``` + +回车后，该命令将新建一个数据库test，并且自动创建一张超级表meters，并以超级表meters为模版创建了1万张表，表名从"t0"到"t9999"。每张表有10万条记录，每条记录有f1，f2，f3三个字段，时间戳ts字段从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"。每张表带有areaid和loc两个标签TAG， areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 + +  ++ 运行taodemo命令的结果，可以进入TDengine命令行，进行验证。 + +  **进入命令行。** + +```bash +$ root@cdf548465318:~/TDengine-server-2.0.13.0# taos + +Welcome to the TDengine shell from Linux, Client Version:2.0.13.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> +``` + +  **查看数据库。** + +```bash +$ taos> show databases; + name | created_time | ntables | vgroups | ··· + test | 2021-04-13 02:14:15.950 | 10000 | 6 | ··· + log | 2021-04-12 09:36:37.549 | 4 | 1 | ··· + +``` + +  **查看超级表。** + +```bash +$ taos> use test; +Database changed. + +$ taos> show stables; + name | created_time | columns | tags | tables | +===================================================================================== + meters | 2021-04-13 02:14:15.955 | 4 | 2 | 10000 | +Query OK, 1 row(s) in set (0.001737s) + +``` + +  **查看表，限制输出十条。** + +```bash +$ taos> select * from test.t0 limit 10; + ts | f1 | f2 | f3 | +==================================================================== + 2017-07-14 02:40:01.000 | 3 | 9 | 0 | + 2017-07-14 02:40:02.000 | 0 | 1 | 2 | + 2017-07-14 02:40:03.000 | 7 | 2 | 3 | + 2017-07-14 02:40:04.000 | 9 | 4 | 5 | + 2017-07-14 02:40:05.000 | 1 | 2 | 5 | + 2017-07-14 02:40:06.000 | 6 | 3 | 2 | + 2017-07-14 02:40:07.000 | 4 | 7 | 8 | + 2017-07-14 02:40:08.000 | 4 | 6 | 6 | + 2017-07-14 02:40:09.000 | 5 | 7 | 7 | + 2017-07-14 02:40:10.000 | 1 | 5 | 0 | +Query OK, 10 row(s) in set (0.003638s) + +``` + +  **查看t0表的标签值。** + +```bash +$ taos> select areaid, loc from test.t0; + areaid | loc | +=========================== + 10 | shanghai | +Query OK, 1 row(s) in set (0.002904s) + +``` + +  +  +### 开发时连接上通过docker启动的TDengine + +连接并且使用在docker容器中的启动的TDengine，有以下两个思路： + + +1. 通过端口映射(-p)，将容器内部开放的网络端口映射到宿主机的指定端口上。通过挂载本地目录(-v)，可以实现宿主机与容器内部的数据同步，防止容器删除后，数据丢失。 + +```bash +$ docker run -d -v /etc/taos:/etc/taos -p 6041:6041 tdengine/tdengine +526aa188da767ae94b244226a2b2eec2b5f17dd8eff592893d9ec0cd0f3a1ccd + +$ curl -u root:taosdata -d 'show databases' 127.0.0.1:6041/rest/sql +{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0} +``` + +  第一条命令，启动了一个运行了TDengine的docker容器，并且将容器的6041端口映射到宿主机的6041端口上。 + +  第二条命令，通过RESTful接口访问TDengine，这时连接的是本机的6041端口，可见连接成功。 + +  +2. 直接通过exec命令，进入到docker容器中去做开发。 + +```bash +$ docker exec -it 526aa188da /bin/bash +``` + From 40a534fb66cb1e35539ad9af6af6f040892c6a27 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 13 Apr 2021 17:21:09 +0800 Subject: [PATCH 13/35] [TD-3412] : create docker version for "getting start". --- .../cn/02.getting-started/01.docker/docs.md | 127 ++++++++---------- documentation20/cn/02.getting-started/docs.md | 4 +- 2 files changed, 59 insertions(+), 72 deletions(-) diff --git a/documentation20/cn/02.getting-started/01.docker/docs.md b/documentation20/cn/02.getting-started/01.docker/docs.md index 8021a54536..8fe37a5306 100644 --- a/documentation20/cn/02.getting-started/01.docker/docs.md +++ b/documentation20/cn/02.getting-started/01.docker/docs.md @@ -1,42 +1,35 @@ -# 通过Docker快速体验TDengine +# 通过 Docker 快速体验 TDengine -由于目前TDengine2.0的服务器仅能在Linux系统上安装和运行，如果开发者使用的是Mac或Windows系统，可能需要安装虚拟机或远程连接Linux服务器，来使用TDengine。Docker轻巧快速，可以作为一个可行，经济，高效的替代方案。下文准备一步一步介绍，如何通过Docker快速使用TDengine，可用于开发，测试，和备份数据。 +虽然并不推荐在生产环境中通过 Docker 来部署 TDengine 服务，但 Docker 工具能够很好地屏蔽底层操作系统的环境差异，很适合在开发测试或初次体验时用于安装运行 TDengine 的工具集。特别是，借助 Docker，能够比较方便地在 Mac OSX 和 Windows 系统上尝试 TDengine，而无需安装虚拟机或额外租用 Linux 服务器。 -  -### 下载Docker +下文通过 Step by Step 风格的介绍，讲解如何通过 Docker 快速建立 TDengine 的单节点运行环境，以支持开发和测试。 -docker的下载可以参考[docker官网文档Get Docker部分](https://docs.docker.com/get-docker/)。 +## 下载 Docker -安装成功后可以查看下docker版本。 +Docker 工具自身的下载请参考 [Docker官网文档](https://docs.docker.com/get-docker/)。 + +安装完毕后可以在命令行终端查看 Docker 版本。如果版本号正常输出，则说明 Docker 环境已经安装成功。 ```bash $ docker -v Docker version 20.10.5, build 55c4c88 ``` -  -  -### 在docker容器中运行TDengine +## 在 Docker 容器中运行 TDengine -+ 使用命令拉取tdengine镜像，并使它在后台运行。 +1，使用命令拉取 TDengine 镜像，并使它在后台运行。 ```bash $ docker run -d tdengine/tdengine cdf548465318c6fc2ad97813f89cc60006393392401cae58a27b15ca9171f316 ``` -  **run**：和前面的docker命令组合起来，运行一个容器。 +- **docker run**：通过 Docker 运行一个容器。 +- **-d**：让容器在后台运行。 +- **tdengine/tdengine**：拉取的 TDengine 官方发布的应用镜像。 +- **cdf548465318c6fc2ad97813f89cc60006393392401cae58a27b15ca9171f316**：这个返回的长字符是容器 ID，我们可以通过容器 ID 来查看对应的容器。 -  **-d**：让容器在后台运行。 - -  **tdengine/tdengine**：拉取的TDengine官方发布的应用镜像 - -  **cdf548465318c6fc2ad97813f89cc60006393392401cae58a27b15ca9171f316**： - -  返回的长字符是容器ID，我们可以通过容器ID来查看对应的容器。 - -  -+ 确认容器是否已经运行起来了。 +2，确认容器是否已经正确运行。 ```bash $ docker ps @@ -44,38 +37,27 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS · cdf548465318 tdengine/tdengine "taosd" 14 minutes ago Up 14 minutes ··· ``` -  **docker ps**：列出所有在运行中的容器信息。 +- **docker ps**：列出所有正在运行状态的容器信息。 +- **CONTAINER ID**：容器 ID。 +- **IMAGE**：使用的镜像。 +- **COMMAND**：启动容器时运行的命令。 +- **CREATED**：容器创建时间。 +- **STATUS**：容器状态。UP 表示运行中。 -  **CONTAINER ID**：容器ID。 - -  **IMAGE**：使用的镜像。 - -  **COMMAND**：启动容器时运行的命令。 - -  **CREATED**：容器创建时间。 - -  **STATUS**：容器状态。UP表示运行中。 - -  -+ 进入docker容器内，使用tdengine。 +3，进入 Docker 容器内，使用 TDengine。 ```bash $ docker exec -it cdf548465318 /bin/bash root@cdf548465318:~/TDengine-server-2.0.13.0# ``` -  **exec**：通过docker exec命令进入容器，如果退出容器不会停止。 +- **docker exec**：通过 docker exec 命令进入容器，如果退出，容器不会停止。 +- **-i**：进入交互模式。 +- **-t**：指定一个终端。 +- **cdf548465318**：容器 ID，需要根据 docker ps 指令返回的值进行修改。 +- **/bin/bash**：载入容器后运行 bash 来进行交互。 -  **-i**：进入交互模式。 - -  **-t**：指定一个终端。 - -  **cdf548465318**：容器ID，需要根据具体情况进行修改。 - -  **/bin/bash**：载入容器后运行bash来进行交互。 - -  -+ 进入容器后，执行TDengine命令行程序。 +4，进入容器后，执行 taos shell 客户端程序。 ```bash $ root@cdf548465318:~/TDengine-server-2.0.13.0# taos @@ -86,23 +68,20 @@ Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. taos> ``` -TDengine终端连接服务成功，打印出了欢迎消息和版本信息。如果失败，会有错误信息打印出来。 +TDengine 终端成功连接服务端，打印出了欢迎消息和版本信息。如果失败，会有错误信息打印出来。 -在TDengine终端中，可以通过SQL命令来创建/删除数据库，表，超级表等，并可以进行插入查询操作。具体可以参考[TDengine官网文档的TAOS SQL部分](https://www.taosdata.com/cn/documentation/taos-sql)。 +在 TDengine 终端中，可以通过 SQL 命令来创建/删除数据库、表、超级表等，并可以进行插入和查询操作。具体可以参考 [TAOS SQL 说明文档](https://www.taosdata.com/cn/documentation/taos-sql)。 -  -  -### 通过taosdemo进一步了解TDengine +## 通过 taosdemo 进一步了解 TDengine -+ 接上面的步骤，先退出TDengine命令行程序。 +1，接上面的步骤，先退出 TDengine 终端程序。 ```bash $ taos> q root@cdf548465318:~/TDengine-server-2.0.13.0# ``` -  -+ 在Linux终端执行taosdemo。 +2，在命令行界面执行 taosdemo。 ```bash $ root@cdf548465318:~/TDengine-server-2.0.13.0# taosdemo @@ -125,12 +104,11 @@ $ root@cdf548465318:~/TDengine-server-2.0.13.0# taosdemo ################################################################### ``` -回车后，该命令将新建一个数据库test，并且自动创建一张超级表meters，并以超级表meters为模版创建了1万张表，表名从"t0"到"t9999"。每张表有10万条记录，每条记录有f1，f2，f3三个字段，时间戳ts字段从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"。每张表带有areaid和loc两个标签TAG， areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 +回车后，该命令将新建一个数据库 test，并且自动创建一张超级表 meters，并以超级表 meters 为模版创建了 1 万张表，表名从 "t0" 到 "t9999"。每张表有 10 万条记录，每条记录有 f1，f2，f3 三个字段，时间戳 ts 字段从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:41:39 999"。每张表带有 areaid 和 loc 两个标签 TAG，areaid 被设置为 1 到 10，loc 被设置为 "beijing" 或 "shanghai"。 -  -+ 运行taodemo命令的结果，可以进入TDengine命令行，进行验证。 +3，运行 taodemo 命令的结果，可以进入 TDengine 终端，进行验证。 -  **进入命令行。** +- **进入命令行。** ```bash $ root@cdf548465318:~/TDengine-server-2.0.13.0# taos @@ -141,17 +119,17 @@ Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. taos> ``` -  **查看数据库。** +- **查看数据库。** ```bash $ taos> show databases; name | created_time | ntables | vgroups | ··· test | 2021-04-13 02:14:15.950 | 10000 | 6 | ··· log | 2021-04-12 09:36:37.549 | 4 | 1 | ··· - + ``` -  **查看超级表。** +- **查看超级表。** ```bash $ taos> use test; @@ -165,7 +143,7 @@ Query OK, 1 row(s) in set (0.001737s) ``` -  **查看表，限制输出十条。** +- **查看表，限制输出十条。** ```bash $ taos> select * from test.t0 limit 10; @@ -185,7 +163,7 @@ Query OK, 10 row(s) in set (0.003638s) ``` -  **查看t0表的标签值。** +- **查看 t0 表的标签值。** ```bash $ taos> select areaid, loc from test.t0; @@ -196,14 +174,21 @@ Query OK, 1 row(s) in set (0.002904s) ``` -  -  -### 开发时连接上通过docker启动的TDengine +## 停止正在 Docker 中运行的 TDengine 服务 -连接并且使用在docker容器中的启动的TDengine，有以下两个思路： +```bash +$ docker stop cdf548465318 +cdf548465318 +``` +- **docker stop**：通过 docker stop 停止指定的正在运行中的 docker 镜像。 +- **cdf548465318**：容器 ID，根据 docker ps 指令返回的结果进行修改。 -1. 通过端口映射(-p)，将容器内部开放的网络端口映射到宿主机的指定端口上。通过挂载本地目录(-v)，可以实现宿主机与容器内部的数据同步，防止容器删除后，数据丢失。 +## 编程开发时连接在 Docker 中的 TDengine + +从 Docker 之外连接使用在 Docker 容器内运行的 TDengine 服务，有以下两个思路： + +1，通过端口映射(-p)，将容器内部开放的网络端口映射到宿主机的指定端口上。通过挂载本地目录(-v)，可以实现宿主机与容器内部的数据同步，防止容器删除后，数据丢失。 ```bash $ docker run -d -v /etc/taos:/etc/taos -p 6041:6041 tdengine/tdengine @@ -213,12 +198,12 @@ $ curl -u root:taosdata -d 'show databases' 127.0.0.1:6041/rest/sql {"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0} ``` -  第一条命令，启动了一个运行了TDengine的docker容器，并且将容器的6041端口映射到宿主机的6041端口上。 +- 第一条命令，启动一个运行了 TDengine 的 docker 容器，并且将容器的 6041 端口映射到宿主机的 6041 端口上。 +- 第二条命令，通过 RESTful 接口访问 TDengine，这时连接的是本机的 6041 端口，可见连接成功。 -  第二条命令，通过RESTful接口访问TDengine，这时连接的是本机的6041端口，可见连接成功。 +注意：在这个示例中，出于方便性考虑，只映射了 RESTful 需要的 6041 端口。如果希望以非 RESTful 方式连接 TDengine 服务，则需要映射从 6030 开始的共 11 个端口。在例子中，挂载本地目录也只是处理了配置文件所在的 /etc/taos 目录，而没有挂载数据存储目录。 -  -2. 直接通过exec命令，进入到docker容器中去做开发。 +2，直接通过 exec 命令，进入到 docker 容器中去做开发。也即，把程序代码放在 TDengine 服务端所在的同一个 Docker 容器中，连接容器本地的 TDengine 服务。 ```bash $ docker exec -it 526aa188da /bin/bash diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index b46322cef2..a98159d8c4 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -10,7 +10,9 @@ TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版 ### 通过Docker容器运行 -请参考[TDengine官方Docker镜像的发布、下载和使用](https://www.taosdata.com/blog/2020/05/13/1509.html) +暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OSX 和 Windows 环境下尝试 TDengine。 + +详细操作方法请参照 [通过Docker快速体验TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。 ### 通过安装包安装 From b8f6f2653da25fe31fea38c4cb2dacd4b140a8ea Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 13 Apr 2021 17:56:25 +0800 Subject: [PATCH 14/35] [TD-3758]: fix setnull throw exception when jdbcType is OTHER (#5797) --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 5823ef743e..cb571b1620 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.26-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index eb158b1f76..83f7563dc3 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.26-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index eb8c92575c..611b1c608a 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.25 + 2.0.26 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 1f75754b0c..719cd40938 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.25 + 2.0.26 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc From d53e74bfc120a5f8d2cc8724177d84d8443c50da Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 13 Apr 2021 20:12:39 +0800 Subject: [PATCH 15/35] Hotfix/sangshuduo/td 3401 query statistic (#5795) (#5802) * [TD-3401]: taosdemo query statistic. refactor func name. * [TD-3401]: taosdemo query statistic. refactor func name 2. * [TD-3401]: taosdemo support query statistic. implementation. * cleanup Co-authored-by: Shuduo Sang Co-authored-by: Shuduo Sang --- src/kit/taosdemo/query.json | 37 +++-- src/kit/taosdemo/taosdemo.c | 261 +++++++++++++++++++++--------------- 2 files changed, 180 insertions(+), 118 deletions(-) diff --git a/src/kit/taosdemo/query.json b/src/kit/taosdemo/query.json index 33ac120bda..d84f997c32 100644 --- a/src/kit/taosdemo/query.json +++ b/src/kit/taosdemo/query.json @@ -1,5 +1,5 @@ { - "filetype":"query", + "filetype": "query", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, @@ -7,13 +7,30 @@ "password": "taosdata", "confirm_parameter_prompt": "yes", "databases": "dbx", - "specified_table_query": - {"query_interval":1, "concurrent":4, - "sqls": [{"sql": "select last_row(*) from stb where color='red'", "result": "./query_res0.txt"}, - {"sql": "select count(*) from stb_01", "result": "./query_res1.txt"}] - }, - "super_table_query": - {"stblname": "stb", "query_interval":1, "threads":4, - "sqls": [{"sql": "select last_row(*) from xxxx", "result": "./query_res2.txt"}] - } + "query_times": 1, + "specified_table_query": { + "query_interval": 1, + "concurrent": 4, + "sqls": [ + { + "sql": "select last_row(*) from stb where color='red'", + "result": "./query_res0.txt" + }, + { + "sql": "select count(*) from stb_01", + "result": "./query_res1.txt" + } + ] + }, + "super_table_query": { + "stblname": "stb", + "query_interval": 1, + "threads": 4, + "sqls": [ + { + "sql": "select last_row(*) from xxxx", + "result": "./query_res2.txt" + } + ] + } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 02a1ec2b6d..30b78a9b21 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -366,6 +366,7 @@ typedef struct SpecifiedQueryInfo_S { char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + int totalQueried; } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { @@ -385,6 +386,7 @@ typedef struct SuperQueryInfo_S { TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; + int totalQueried; } SuperQueryInfo; typedef struct SQueryMetaInfo_S { @@ -398,6 +400,7 @@ typedef struct SQueryMetaInfo_S { SpecifiedQueryInfo specifiedQueryInfo; SuperQueryInfo superQueryInfo; + int totalQueried; } SQueryMetaInfo; typedef struct SThreadInfo_S { @@ -2602,8 +2605,8 @@ static int createDatabasesAndStables() { static void* createTable(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; int64_t lastPrintTime = taosGetTimestampMs(); @@ -2621,15 +2624,15 @@ static void* createTable(void *sarg) verbosePrint("%s() LN%d: Creating table from %d to %d\n", __func__, __LINE__, - winfo->start_table_from, winfo->end_table_to); + pThreadInfo->start_table_from, pThreadInfo->end_table_to); - for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) { + for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { if (0 == g_Dbs.use_metric) { snprintf(buffer, buff_len, "create table if not exists %s.%s%d %s;", - winfo->db_name, + pThreadInfo->db_name, g_args.tb_prefix, i, - winfo->cols); + pThreadInfo->cols); } else { if (superTblInfo == NULL) { errorPrint("%s() LN%d, use metric, but super table info is NULL\n", @@ -2658,8 +2661,8 @@ static void* createTable(void *sarg) len += snprintf(buffer + len, buff_len - len, "if not exists %s.%s%d using %s.%s tags %s ", - winfo->db_name, superTblInfo->childTblPrefix, - i, winfo->db_name, + pThreadInfo->db_name, superTblInfo->childTblPrefix, + i, pThreadInfo->db_name, superTblInfo->sTblName, tagsValBuf); free(tagsValBuf); batchNum++; @@ -2673,7 +2676,7 @@ static void* createTable(void *sarg) len = 0; verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); - if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)){ + if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){ errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); free(buffer); return NULL; @@ -2682,14 +2685,14 @@ static void* createTable(void *sarg) int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] already create %d - %d tables\n", - winfo->threadID, winfo->start_table_from, i); + pThreadInfo->threadID, pThreadInfo->start_table_from, i); lastPrintTime = currentPrintTime; } } if (0 != len) { verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer); - if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) { + if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) { errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); } } @@ -5157,45 +5160,45 @@ free_and_statistics_2: static void* syncWrite(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; if (interlaceRows > 0) { // interlace mode - return syncWriteInterlace(winfo); + return syncWriteInterlace(pThreadInfo); } else { // progressive mode - return syncWriteProgressive(winfo); + return syncWriteProgressive(pThreadInfo); } } static void callBack(void *param, TAOS_RES *res, int code) { - threadInfo* winfo = (threadInfo*)param; - SSuperTable* superTblInfo = winfo->superTblInfo; + threadInfo* pThreadInfo = (threadInfo*)param; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; if (insert_interval) { - winfo->et = taosGetTimestampUs(); - if (((winfo->et - winfo->st)/1000) < insert_interval) { - taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms + pThreadInfo->et = taosGetTimestampUs(); + if (((pThreadInfo->et - pThreadInfo->st)/1000) < insert_interval) { + taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)/1000); // ms } } - char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); + char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen); char data[MAX_DATA_SIZE]; char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, - winfo->start_table_from); -// if (winfo->counter >= winfo->superTblInfo->insertRows) { - if (winfo->counter >= g_args.num_of_RPR) { - winfo->start_table_from++; - winfo->counter = 0; + pstr += sprintf(pstr, "insert into %s.%s%d values", pThreadInfo->db_name, pThreadInfo->tb_prefix, + pThreadInfo->start_table_from); +// if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { + if (pThreadInfo->counter >= g_args.num_of_RPR) { + pThreadInfo->start_table_from++; + pThreadInfo->counter = 0; } - if (winfo->start_table_from > winfo->end_table_to) { - tsem_post(&winfo->lock_sem); + if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) { + tsem_post(&pThreadInfo->lock_sem); free(buffer); taos_free_result(res); return; @@ -5203,46 +5206,46 @@ static void callBack(void *param, TAOS_RES *res, int code) { for (int i = 0; i < g_args.num_of_RPR; i++) { int rand_num = taosRandom() % 100; - if (0 != winfo->superTblInfo->disorderRatio - && rand_num < winfo->superTblInfo->disorderRatio) { - int64_t d = winfo->lastTs - (taosRandom() % winfo->superTblInfo->disorderRange + 1); - generateRowData(data, d, winfo->superTblInfo); + if (0 != pThreadInfo->superTblInfo->disorderRatio + && rand_num < pThreadInfo->superTblInfo->disorderRatio) { + int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + generateRowData(data, d, pThreadInfo->superTblInfo); } else { - generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); + generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); } pstr += sprintf(pstr, "%s", data); - winfo->counter++; + pThreadInfo->counter++; - if (winfo->counter >= winfo->superTblInfo->insertRows) { + if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) { break; } } if (insert_interval) { - winfo->st = taosGetTimestampUs(); + pThreadInfo->st = taosGetTimestampUs(); } - taos_query_a(winfo->taos, buffer, callBack, winfo); + taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo); free(buffer); taos_free_result(res); } static void *asyncWrite(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - winfo->st = 0; - winfo->et = 0; - winfo->lastTs = winfo->start_time; + pThreadInfo->st = 0; + pThreadInfo->et = 0; + pThreadInfo->lastTs = pThreadInfo->start_time; int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; if (insert_interval) { - winfo->st = taosGetTimestampUs(); + pThreadInfo->st = taosGetTimestampUs(); } - taos_query_a(winfo->taos, "show databases", callBack, winfo); + taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo); - tsem_wait(&(winfo->lock_sem)); + tsem_wait(&(pThreadInfo->lock_sem)); return NULL; } @@ -5772,10 +5775,10 @@ static int insertTestProcess() { return 0; } -static void *specifiedQueryProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; +static void *specifiedTableQuery(void *sarg) { + threadInfo *pThreadInfo = (threadInfo *)sarg; - if (winfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -5784,17 +5787,17 @@ static void *specifiedQueryProcess(void *sarg) { g_queryInfo.port); if (taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - winfo->threadID, taos_errstr(NULL)); + pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } else { - winfo->taos = taos; + pThreadInfo->taos = taos; } } char sqlStr[MAX_DB_NAME_SIZE + 5]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(winfo->taos); + if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(pThreadInfo->taos); errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); return NULL; @@ -5805,11 +5808,15 @@ static void *specifiedQueryProcess(void *sarg) { int queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + int totalQueried = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + int64_t startTs = taosGetTimestampMs(); + while(queryTimes --) { if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.rate*1000) { taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); } st = taosGetTimestampUs(); @@ -5817,13 +5824,13 @@ static void *specifiedQueryProcess(void *sarg) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { int64_t t1 = taosGetTimestampUs(); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[winfo->querySeq][0] != 0) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[winfo->querySeq], - winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } - selectAndGetResult(winfo->taos, - g_queryInfo.specifiedQueryInfo.sql[winfo->querySeq], tmpFile); + selectAndGetResult(pThreadInfo->taos, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); int64_t t2 = taosGetTimestampUs(); printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); @@ -5831,25 +5838,37 @@ static void *specifiedQueryProcess(void *sarg) { int64_t t1 = taosGetTimestampUs(); int retCode = postProceSql(g_queryInfo.host, g_queryInfo.port, - g_queryInfo.specifiedQueryInfo.sql[winfo->querySeq]); + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); + return NULL; + } int64_t t2 = taosGetTimestampUs(); printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", taosGetSelfPthreadId(), (t2 - t1)/1000000.0); - if (0 != retCode) { - printf("====restful return fail, threadID[%d]\n", winfo->threadID); - return NULL; - } } + totalQueried ++; + g_queryInfo.specifiedQueryInfo.totalQueried ++; et = taosGetTimestampUs(); printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t endTs = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently completed queries: %d, QPS: %10.2f\n", + pThreadInfo->threadID, + totalQueried, + totalQueried/((endTs-startTs)/1000.0)); + } + lastPrintTime = currentPrintTime; } return NULL; } -static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { +static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { char sourceString[32] = "xxxx"; char subTblName[MAX_TB_NAME_SIZE*3]; sprintf(subTblName, "%s.%s", @@ -5871,11 +5890,11 @@ static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { //printf("3: %s\n", outSql); } -static void *superQueryProcess(void *sarg) { +static void *superTableQuery(void *sarg) { char sqlstr[1024]; - threadInfo *winfo = (threadInfo *)sarg; + threadInfo *pThreadInfo = (threadInfo *)sarg; - if (winfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -5884,10 +5903,10 @@ static void *superQueryProcess(void *sarg) { g_queryInfo.port); if (taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - winfo->threadID, taos_errstr(NULL)); + pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } else { - winfo->taos = taos; + pThreadInfo->taos = taos; } } @@ -5895,33 +5914,49 @@ static void *superQueryProcess(void *sarg) { int64_t et = (int64_t)g_queryInfo.superQueryInfo.rate*1000; int queryTimes = g_queryInfo.superQueryInfo.queryTimes; + int totalQueried = 0; + int64_t startTs = taosGetTimestampMs(); + int64_t lastPrintTime = taosGetTimestampMs(); while(queryTimes --) { if (g_queryInfo.superQueryInfo.rate && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) { taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); } st = taosGetTimestampUs(); - for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) { + for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); - replaceSubTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[j], - winfo->threadID); + pThreadInfo->threadID); } - selectAndGetResult(winfo->taos, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo->taos, sqlstr, tmpFile); + + totalQueried++; + g_queryInfo.superQueryInfo.totalQueried ++; + + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t endTs = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently completed queries: %d, QPS: %10.2f\n", + pThreadInfo->threadID, + totalQueried, + totalQueried/((endTs-startTs)/1000.0)); + } + lastPrintTime = currentPrintTime; } } et = taosGetTimestampUs(); printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", taosGetSelfPthreadId(), - winfo->start_table_from, - winfo->end_table_to, + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, (double)(et - st)/1000000.0); } @@ -5967,6 +6002,8 @@ static int queryTestProcess() { int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + int64_t startTs = taosGetTimestampMs(); + if ((nSqlCount > 0) && (nConcurrent > 0)) { pids = malloc(nConcurrent * nSqlCount * sizeof(pthread_t)); @@ -6000,7 +6037,7 @@ static int queryTestProcess() { t_info->taos = NULL;// TODO: workaround to use separate taos connection; - pthread_create(pids + i * nSqlCount + j, NULL, specifiedQueryProcess, + pthread_create(pids + i * nSqlCount + j, NULL, specifiedTableQuery, t_info); } } @@ -6049,7 +6086,7 @@ static int queryTestProcess() { t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superQueryProcess, t_info); + pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -6076,6 +6113,14 @@ static int queryTestProcess() { tmfree((char*)infosOfSub); // taos_close(taos);// TODO: workaround to use separate taos connection; + int64_t endTs = taosGetTimestampMs(); + + int totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + + g_queryInfo.superQueryInfo.totalQueried; + + printf("==== completed total queries: %d, the QPS of all threads: %10.2f====\n", + totalQueried, + totalQueried/((endTs-startTs)/1000.0)); return 0; } @@ -6112,12 +6157,12 @@ static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultF return tsub; } -static void *subSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; +static void *superSubscribe(void *sarg) { + threadInfo *pThreadInfo = (threadInfo *)sarg; char subSqlstr[1024]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - if (winfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -6126,17 +6171,17 @@ static void *subSubscribeProcess(void *sarg) { g_queryInfo.port); if (taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - winfo->threadID, taos_errstr(NULL)); + pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } else { - winfo->taos = taos; + pThreadInfo->taos = taos; } } char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(winfo->taos); + if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(pThreadInfo->taos); errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); return NULL; @@ -6147,7 +6192,7 @@ static void *subSubscribeProcess(void *sarg) { do { //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); //} //st = taosGetTimestampMs(); @@ -6155,15 +6200,15 @@ static void *subSubscribeProcess(void *sarg) { for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceSubTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], winfo->threadID); + g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); } - tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + tsub[i] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); if (NULL == tsub[i]) { - taos_close(winfo->taos); + taos_close(pThreadInfo->taos); return NULL; } } @@ -6185,7 +6230,7 @@ static void *subSubscribeProcess(void *sarg) { if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], - winfo->threadID); + pThreadInfo->threadID); } getResult(res, tmpFile); } @@ -6197,15 +6242,15 @@ static void *subSubscribeProcess(void *sarg) { taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); } - taos_close(winfo->taos); + taos_close(pThreadInfo->taos); return NULL; } -static void *superSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; +static void *specifiedSubscribe(void *sarg) { + threadInfo *pThreadInfo = (threadInfo *)sarg; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - if (winfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, g_queryInfo.user, @@ -6214,18 +6259,18 @@ static void *superSubscribeProcess(void *sarg) { g_queryInfo.port); if (taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", - winfo->threadID, taos_errstr(NULL)); + pThreadInfo->threadID, taos_errstr(NULL)); return NULL; } else { - winfo->taos = taos; + pThreadInfo->taos = taos; } } char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(winfo->taos); + if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(pThreadInfo->taos); return NULL; } @@ -6234,7 +6279,7 @@ static void *superSubscribeProcess(void *sarg) { do { //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); //} //st = taosGetTimestampMs(); @@ -6244,12 +6289,12 @@ static void *superSubscribeProcess(void *sarg) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); } - tsub[i] = subscribeImpl(winfo->taos, + tsub[i] = subscribeImpl(pThreadInfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { - taos_close(winfo->taos); + taos_close(pThreadInfo->taos); return NULL; } } @@ -6270,7 +6315,7 @@ static void *superSubscribeProcess(void *sarg) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); } getResult(res, tmpFile); } @@ -6283,7 +6328,7 @@ static void *superSubscribeProcess(void *sarg) { g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } - taos_close(winfo->taos); + taos_close(pThreadInfo->taos); return NULL; } @@ -6341,7 +6386,7 @@ static int subscribeTestProcess() { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + i, NULL, superSubscribeProcess, t_info); + pthread_create(pids + i, NULL, specifiedSubscribe, t_info); } //==== create sub threads for query from sub table @@ -6384,7 +6429,7 @@ static int subscribeTestProcess() { t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); + pthread_create(pidsOfSub + i, NULL, superSubscribe, t_info); } g_queryInfo.superQueryInfo.threadCnt = threads; From 2e46ae3f76263830a6211e6a1b33193f3e48b0b0 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Apr 2021 10:03:53 +0800 Subject: [PATCH 16/35] [TD-3412] : fix minor typo. --- documentation20/cn/02.getting-started/01.docker/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/02.getting-started/01.docker/docs.md b/documentation20/cn/02.getting-started/01.docker/docs.md index 8fe37a5306..d41a7a9d89 100644 --- a/documentation20/cn/02.getting-started/01.docker/docs.md +++ b/documentation20/cn/02.getting-started/01.docker/docs.md @@ -106,7 +106,7 @@ $ root@cdf548465318:~/TDengine-server-2.0.13.0# taosdemo 回车后，该命令将新建一个数据库 test，并且自动创建一张超级表 meters，并以超级表 meters 为模版创建了 1 万张表，表名从 "t0" 到 "t9999"。每张表有 10 万条记录，每条记录有 f1，f2，f3 三个字段，时间戳 ts 字段从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:41:39 999"。每张表带有 areaid 和 loc 两个标签 TAG，areaid 被设置为 1 到 10，loc 被设置为 "beijing" 或 "shanghai"。 -3，运行 taodemo 命令的结果，可以进入 TDengine 终端，进行验证。 +3，进入 TDengine 终端，查看 taosdemo 生成的数据。 - **进入命令行。** From 2658da451c417d03119cb9fe263a9fd0e8dcaa96 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Apr 2021 16:43:45 +0800 Subject: [PATCH 17/35] [TD-2577] : "GROUP BY" add "HAVING" support. --- documentation20/cn/11.administrator/docs.md | 4 ++-- documentation20/cn/12.taos-sql/docs.md | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index 961a3801e7..5eccc67ff8 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -463,7 +463,7 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | 关键字列表 | | | | | | ---------- | ----------- | ------------ | ---------- | --------- | -| ABLOCKS | CONNECTIONS | GT | MNODES | SLIDING | +| ABLOCKS | CONNECTIONS | HAVING | MNODES | SLIDING | | ABORT | COPY | ID | MODULES | SLIMIT | | ACCOUNT | COUNT | IF | NCHAR | SMALLINT | | ACCOUNTS | CREATE | IGNORE | NE | SPREAD | @@ -499,5 +499,5 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | CONCAT | GLOB | METRICS | SET | VIEW | | CONFIGS | GRANTS | MIN | SHOW | WAVG | | CONFLICT | GROUP | MINUS | SLASH | WHERE | -| CONNECTION | | | | | +| CONNECTION | GT | | | | diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 864169b898..b6d81720cf 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -407,10 +407,10 @@ SELECT select_expr [, select_expr ...] [INTERVAL (interval_val [, interval_offset])] [SLIDING sliding_val] [FILL fill_val] - [GROUP BY col_list] + [GROUP BY col_list [HAVING having_condition]] [ORDER BY col_list { DESC | ASC }] - [SLIMIT limit_val [, SOFFSET offset_val]] - [LIMIT limit_val [, OFFSET offset_val]] + [SLIMIT limit_val [SOFFSET offset_val]] + [LIMIT limit_val [OFFSET offset_val]] [>> export_file]; ``` @@ -648,6 +648,15 @@ Query OK, 1 row(s) in set (0.001091s) 2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。 3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 +### GROUP BY 之后的 HAVING 过滤 + +从 2.0.20 版本开始，GROUP BY 之后允许再跟一个 HAVING 子句，对成组后的各组数据再做筛选。HAVING 子句可以使用聚合函数和选择函数作为过滤条件（但暂时不支持 LEASTSQUARES、TOP、BOTTOM、LAST_ROW）。 + +例如，如下语句只会输出 `AVG(f1) > 0` 的分组： +```mysql +SELECT AVG(f1), SPREAD(f1, f2, st2.f1) FROM st2 WHERE f1 > 0 GROUP BY f1 HAVING AVG(f1) > 0; +``` + ### SQL 示例 - 对于下面的例子，表tb1用以下语句创建 From 63273b9fb419481fc1a615829643b7d8856b3d62 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Apr 2021 17:24:22 +0800 Subject: [PATCH 18/35] [TD-2639] : add keyword "topic". --- documentation20/cn/11.administrator/docs.md | 74 ++++++++++----------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index 5eccc67ff8..b89016fafb 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -463,41 +463,41 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | 关键字列表 | | | | | | ---------- | ----------- | ------------ | ---------- | --------- | -| ABLOCKS | CONNECTIONS | HAVING | MNODES | SLIDING | -| ABORT | COPY | ID | MODULES | SLIMIT | -| ACCOUNT | COUNT | IF | NCHAR | SMALLINT | -| ACCOUNTS | CREATE | IGNORE | NE | SPREAD | -| ADD | CTIME | IMMEDIATE | NONE | STABLE | -| AFTER | DATABASE | IMPORT | NOT | STABLES | -| ALL | DATABASES | IN | NOTNULL | STAR | -| ALTER | DAYS | INITIALLY | NOW | STATEMENT | -| AND | DEFERRED | INSERT | OF | STDDEV | -| AS | DELIMITERS | INSTEAD | OFFSET | STREAM | -| ASC | DESC | INTEGER | OR | STREAMS | -| ATTACH | DESCRIBE | INTERVAL | ORDER | STRING | -| AVG | DETACH | INTO | PASS | SUM | -| BEFORE | DIFF | IP | PERCENTILE | TABLE | -| BEGIN | DISTINCT | IS | PLUS | TABLES | -| BETWEEN | DIVIDE | ISNULL | PRAGMA | TAG | -| BIGINT | DNODE | JOIN | PREV | TAGS | -| BINARY | DNODES | KEEP | PRIVILEGE | TBLOCKS | -| BITAND | DOT | KEY | QUERIES | TBNAME | -| BITNOT | DOUBLE | KILL | QUERY | TIMES | -| BITOR | DROP | LAST | RAISE | TIMESTAMP | -| BOOL | EACH | LE | REM | TINYINT | -| BOTTOM | END | LEASTSQUARES | REPLACE | TOP | -| BY | EQ | LIKE | REPLICA | TRIGGER | -| CACHE | EXISTS | LIMIT | RESET | UMINUS | -| CASCADE | EXPLAIN | LINEAR | RESTRICT | UPLUS | -| CHANGE | FAIL | LOCAL | ROW | USE | -| CLOG | FILL | LP | ROWS | USER | -| CLUSTER | FIRST | LSHIFT | RP | USERS | -| COLON | FLOAT | LT | RSHIFT | USING | -| COLUMN | FOR | MATCH | SCORES | VALUES | -| COMMA | FROM | MAX | SELECT | VARIABLE | -| COMP | GE | METRIC | SEMI | VGROUPS | -| CONCAT | GLOB | METRICS | SET | VIEW | -| CONFIGS | GRANTS | MIN | SHOW | WAVG | -| CONFLICT | GROUP | MINUS | SLASH | WHERE | -| CONNECTION | GT | | | | +| ABLOCKS | CONNECTIONS | HAVING | MODULES | SLIMIT | +| ABORT | COPY | ID | NCHAR | SMALLINT | +| ACCOUNT | COUNT | IF | NE | SPREAD | +| ACCOUNTS | CREATE | IGNORE | NONE | STABLE | +| ADD | CTIME | IMMEDIATE | NOT | STABLES | +| AFTER | DATABASE | IMPORT | NOTNULL | STAR | +| ALL | DATABASES | IN | NOW | STATEMENT | +| ALTER | DAYS | INITIALLY | OF | STDDEV | +| AND | DEFERRED | INSERT | OFFSET | STREAM | +| AS | DELIMITERS | INSTEAD | OR | STREAMS | +| ASC | DESC | INTEGER | ORDER | STRING | +| ATTACH | DESCRIBE | INTERVAL | PASS | SUM | +| AVG | DETACH | INTO | PERCENTILE | TABLE | +| BEFORE | DIFF | IP | PLUS | TABLES | +| BEGIN | DISTINCT | IS | PRAGMA | TAG | +| BETWEEN | DIVIDE | ISNULL | PREV | TAGS | +| BIGINT | DNODE | JOIN | PRIVILEGE | TBLOCKS | +| BINARY | DNODES | KEEP | QUERIES | TBNAME | +| BITAND | DOT | KEY | QUERY | TIMES | +| BITNOT | DOUBLE | KILL | RAISE | TIMESTAMP | +| BITOR | DROP | LAST | REM | TINYINT | +| BOOL | EACH | LE | REPLACE | TOP | +| BOTTOM | END | LEASTSQUARES | REPLICA | TOPIC | +| BY | EQ | LIKE | RESET | TRIGGER | +| CACHE | EXISTS | LIMIT | RESTRICT | UMINUS | +| CASCADE | EXPLAIN | LINEAR | ROW | UPLUS | +| CHANGE | FAIL | LOCAL | ROWS | USE | +| CLOG | FILL | LP | RP | USER | +| CLUSTER | FIRST | LSHIFT | RSHIFT | USERS | +| COLON | FLOAT | LT | SCORES | USING | +| COLUMN | FOR | MATCH | SELECT | VALUES | +| COMMA | FROM | MAX | SEMI | VARIABLE | +| COMP | GE | METRIC | SET | VGROUPS | +| CONCAT | GLOB | METRICS | SHOW | VIEW | +| CONFIGS | GRANTS | MIN | SLASH | WAVG | +| CONFLICT | GROUP | MINUS | SLIDING | WHERE | +| CONNECTION | GT | MNODES | | | From 8713f96d2fa35887e50f1e845bc35face096bfda Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 14 Apr 2021 17:39:56 +0800 Subject: [PATCH 19/35] [TD-3204]: support usigned integer types for C# connector (#5808) Added unsigned integer types and change taosdemo insert and select queries for these types. taos define | taos sql | c# --- | --- | --- UTINYINT | TINYINT UNSIGNED | `byte` USMALLINT | SMALLINT UNSIGNED | `ushort` UINT | INT UNSIGNED | `uint` UBIGINT | BIGINT UNSIGNED | `ulong` TINYINT | TINYINT | `sbyte` Note: also change the tinyint value type to C# `sbyte`(the right signed one byte type) in this pr. --- tests/examples/C#/taosdemo/TDengineDriver.cs | 22 +++-- tests/examples/C#/taosdemo/taosdemo.cs | 89 ++++++++++++-------- 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/tests/examples/C#/taosdemo/TDengineDriver.cs b/tests/examples/C#/taosdemo/TDengineDriver.cs index 205269501d..2c150341f6 100644 --- a/tests/examples/C#/taosdemo/TDengineDriver.cs +++ b/tests/examples/C#/taosdemo/TDengineDriver.cs @@ -31,7 +31,11 @@ namespace TDengineDriver TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes TSDB_DATA_TYPE_BINARY = 8, // string TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes - TSDB_DATA_TYPE_NCHAR = 10 // unicode string + TSDB_DATA_TYPE_NCHAR = 10, // unicode string + TSDB_DATA_TYPE_UTINYINT = 11,// 1 byte + TSDB_DATA_TYPE_USMALLINT= 12,// 2 bytes + TSDB_DATA_TYPE_UINT = 13, // 4 bytes + TSDB_DATA_TYPE_UBIGINT= 14 // 8 bytes } enum TDengineInitOption @@ -53,15 +57,23 @@ namespace TDengineDriver switch ((TDengineDataType)type) { case TDengineDataType.TSDB_DATA_TYPE_BOOL: - return "BOOLEAN"; + return "BOOL"; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - return "BYTE"; + return "TINYINT"; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - return "SHORT"; + return "SMALLINT"; case TDengineDataType.TSDB_DATA_TYPE_INT: return "INT"; case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - return "LONG"; + return "BIGINT"; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + return "TINYINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + return "SMALLINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + return "INT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + return "BIGINT UNSIGNED"; case TDengineDataType.TSDB_DATA_TYPE_FLOAT: return "FLOAT"; case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs index 2d78418e0a..7c34f1ec04 100644 --- a/tests/examples/C#/taosdemo/taosdemo.cs +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -63,7 +63,7 @@ namespace TDengineDriver static void HelpPrint(string arg, string desc) { string indent = " "; - Console.WriteLine("{0}{1}", indent, arg.PadRight(25)+desc); + Console.WriteLine("{0}{1}", indent, arg.PadRight(25) + desc); } static void PrintHelp(String[] argv) @@ -142,33 +142,33 @@ namespace TDengineDriver verbose = this.GetArgumentAsFlag(argv, "-v", true); debug = this.GetArgumentAsFlag(argv, "-g", true); - VerbosePrint ("###################################################################\n"); - VerbosePrintFormat ("# Server IP: {0}\n", host); - VerbosePrintFormat ("# User: {0}\n", user); - VerbosePrintFormat ("# Password: {0}\n", password); - VerbosePrintFormat ("# Number of Columns per record: {0}\n", colsPerRecord); - VerbosePrintFormat ("# Number of Threads: {0}\n", numOfThreads); - VerbosePrintFormat ("# Number of Tables: {0}\n", numOfTables); - VerbosePrintFormat ("# Number of records per Table: {0}\n", recordsPerTable); - VerbosePrintFormat ("# Records/Request: {0}\n", recordsPerRequest); - VerbosePrintFormat ("# Database name: {0}\n", dbName); - VerbosePrintFormat ("# Replica: {0}\n", replica); - VerbosePrintFormat ("# Use STable: {0}\n", useStable); - VerbosePrintFormat ("# Table prefix: {0}\n", tablePrefix); + VerbosePrint("###################################################################\n"); + VerbosePrintFormat("# Server IP: {0}\n", host); + VerbosePrintFormat("# User: {0}\n", user); + VerbosePrintFormat("# Password: {0}\n", password); + VerbosePrintFormat("# Number of Columns per record: {0}\n", colsPerRecord); + VerbosePrintFormat("# Number of Threads: {0}\n", numOfThreads); + VerbosePrintFormat("# Number of Tables: {0}\n", numOfTables); + VerbosePrintFormat("# Number of records per Table: {0}\n", recordsPerTable); + VerbosePrintFormat("# Records/Request: {0}\n", recordsPerRequest); + VerbosePrintFormat("# Database name: {0}\n", dbName); + VerbosePrintFormat("# Replica: {0}\n", replica); + VerbosePrintFormat("# Use STable: {0}\n", useStable); + VerbosePrintFormat("# Table prefix: {0}\n", tablePrefix); if (useStable == true) { VerbosePrintFormat("# STable prefix: {0}\n", stablePrefix); } - VerbosePrintFormat ("# Data order: {0}\n", order); - VerbosePrintFormat ("# Data out of order rate: {0}\n", rateOfOutorder); - VerbosePrintFormat ("# Delete method: {0}\n", methodOfDelete); - VerbosePrintFormat ("# Query command: {0}\n", query); - VerbosePrintFormat ("# Query Mode: {0}\n", queryMode); - VerbosePrintFormat ("# Insert Only: {0}\n", isInsertOnly); - VerbosePrintFormat ("# Verbose output {0}\n", verbose); - VerbosePrintFormat ("# Test time: {0}\n", DateTime.Now.ToString("h:mm:ss tt")); + VerbosePrintFormat("# Data order: {0}\n", order); + VerbosePrintFormat("# Data out of order rate: {0}\n", rateOfOutorder); + VerbosePrintFormat("# Delete method: {0}\n", methodOfDelete); + VerbosePrintFormat("# Query command: {0}\n", query); + VerbosePrintFormat("# Query Mode: {0}\n", queryMode); + VerbosePrintFormat("# Insert Only: {0}\n", isInsertOnly); + VerbosePrintFormat("# Verbose output {0}\n", verbose); + VerbosePrintFormat("# Test time: {0}\n", DateTime.Now.ToString("h:mm:ss tt")); - VerbosePrint ("###################################################################\n"); + VerbosePrint("###################################################################\n"); if (skipReadKey == false) { @@ -406,7 +406,7 @@ namespace TDengineDriver sql.Clear(); sql.Append("CREATE TABLE IF NOT EXISTS "). Append(this.dbName).Append(".").Append(this.stablePrefix). - Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10), v10 tinyint unsigned, v11 smallint unsigned, v12 int unsigned, v13 bigint unsigned) tags(t1 int)"); IntPtr res = TDengine.Query(this.conn, sql.ToString()); if (res != IntPtr.Zero) { @@ -523,7 +523,7 @@ namespace TDengineDriver int offset = IntPtr.Size * fields; IntPtr data = Marshal.ReadIntPtr(rowdata, offset); - builder.Append("---"); + builder.Append(" | "); if (data == IntPtr.Zero) { @@ -538,7 +538,7 @@ namespace TDengineDriver builder.Append(v1); break; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - byte v2 = Marshal.ReadByte(data); + sbyte v2 = (sbyte)Marshal.ReadByte(data); builder.Append(v2); break; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: @@ -573,9 +573,25 @@ namespace TDengineDriver string v10 = Marshal.PtrToStringAnsi(data); builder.Append(v10); break; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + byte v11 = Marshal.ReadByte(data); + builder.Append(v11); + break; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + ushort v12 = (ushort)Marshal.ReadInt16(data); + builder.Append(v12); + break; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + uint v13 = (uint)Marshal.ReadInt32(data); + builder.Append(v13); + break; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + ulong v14 = (ulong)Marshal.ReadInt64(data); + builder.Append(v14); + break; } } - builder.Append("---"); + builder.Append(" | "); VerbosePrint(builder.ToString() + "\n"); builder.Clear(); @@ -642,8 +658,8 @@ namespace TDengineDriver tester.ExecuteQuery(); watch.Stop(); elapsedMs = watch.Elapsed.TotalMilliseconds; - Console.WriteLine("C# taosdemo: Spent {0} seconds to query {1} records.\n", - elapsedMs/1000, + Console.WriteLine("C# taosdemo: Spent {0} seconds to query {1} records.\n", + elapsedMs / 1000, tester.recordsPerTable * tester.numOfTables ); } @@ -714,7 +730,7 @@ namespace TDengineDriver long baseTimestamp = 1609430400000; // 2021/01/01 0:0:0 VerbosePrintFormat("beginTime is {0} + {1}h:{2}m:{3}s\n", baseTimestamp, h, m, s); - long beginTimestamp = baseTimestamp + ((h*60 + m) * 60 + s) * 1000; + long beginTimestamp = baseTimestamp + ((h * 60 + m) * 60 + s) * 1000; Random random = new Random(); long rowsInserted = 0; @@ -755,11 +771,16 @@ namespace TDengineDriver sql.Append("(") .Append(writeTimeStamp) - .Append(", 1, 2, 3,") - .Append(i + batch) - .Append(", 5, 6, 7, 'abc', 'def')"); + .Append(", 1, -2, -3,") + .Append(i + batch - 127) + .Append(", -5, -6, -7, 'abc', 'def', 254, 65534,") + .Append(4294967294 - (uint)i - (uint)batch) + .Append(",") + .Append(18446744073709551614 - (ulong)i - (ulong)batch) + .Append(")"); } + VerbosePrint(sql.ToString() + "\n"); IntPtr res = TDengine.Query(this.conn, sql.ToString()); if (res == IntPtr.Zero) { @@ -837,7 +858,7 @@ namespace TDengineDriver } else { - sql = sql.Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10))"); + sql = sql.Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10), v10 tinyint unsigned, v11 smallint unsigned, v12 int unsigned, v13 bigint unsigned)"); } IntPtr res = TDengine.Query(this.conn, sql.ToString()); if (res != IntPtr.Zero) From 25953723e42d9c91587ce11578b65eda7a660649 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 14 Apr 2021 18:05:00 +0800 Subject: [PATCH 20/35] [TD-3162]: taos shell command line arugments. (#5810) Co-authored-by: Shuduo Sang --- src/kit/shell/src/shellLinux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 3f6b3da9bf..37050c416c 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -37,7 +37,7 @@ static struct argp_option options[] = { {"password", 'p', "PASSWORD", OPTION_ARG_OPTIONAL, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, - {"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."}, + {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, {"dump-config", 'C', 0, 0, "Dump configuration."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, From 94b21d4861f1a3d3f87129b4cee5b99049d99895 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Apr 2021 18:12:10 +0800 Subject: [PATCH 21/35] [TD-3028] : add service ports section in faq. --- .../cn/02.getting-started/01.docker/docs.md | 2 +- documentation20/cn/03.architecture/docs.md | 2 +- documentation20/cn/08.connector/docs.md | 2 +- documentation20/cn/11.administrator/docs.md | 5 ++--- documentation20/cn/13.faq/docs.md | 15 +++++++++++++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/documentation20/cn/02.getting-started/01.docker/docs.md b/documentation20/cn/02.getting-started/01.docker/docs.md index d41a7a9d89..30803d9777 100644 --- a/documentation20/cn/02.getting-started/01.docker/docs.md +++ b/documentation20/cn/02.getting-started/01.docker/docs.md @@ -201,7 +201,7 @@ $ curl -u root:taosdata -d 'show databases' 127.0.0.1:6041/rest/sql - 第一条命令，启动一个运行了 TDengine 的 docker 容器，并且将容器的 6041 端口映射到宿主机的 6041 端口上。 - 第二条命令，通过 RESTful 接口访问 TDengine，这时连接的是本机的 6041 端口，可见连接成功。 -注意：在这个示例中，出于方便性考虑，只映射了 RESTful 需要的 6041 端口。如果希望以非 RESTful 方式连接 TDengine 服务，则需要映射从 6030 开始的共 11 个端口。在例子中，挂载本地目录也只是处理了配置文件所在的 /etc/taos 目录，而没有挂载数据存储目录。 +注意：在这个示例中，出于方便性考虑，只映射了 RESTful 需要的 6041 端口。如果希望以非 RESTful 方式连接 TDengine 服务，则需要映射从 6030 开始的共 11 个端口（完整的端口情况请参见 [TDengine 2.0 端口说明](https://www.taosdata.com/cn/documentation/faq#port)）。在例子中，挂载本地目录也只是处理了配置文件所在的 /etc/taos 目录，而没有挂载数据存储目录。 2，直接通过 exec 命令，进入到 docker 容器中去做开发。也即，把程序代码放在 TDengine 服务端所在的同一个 Docker 容器中，连接容器本地的 TDengine 服务。 diff --git a/documentation20/cn/03.architecture/docs.md b/documentation20/cn/03.architecture/docs.md index 87553fa8ad..3d6e5e4f21 100644 --- a/documentation20/cn/03.architecture/docs.md +++ b/documentation20/cn/03.architecture/docs.md @@ -178,7 +178,7 @@ TDengine 分布式架构的逻辑结构图如下： **FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 -**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。（另外还可能有 RESTful、Arbitrator 所使用的端口，那样的话就一共是 13 个。）使用时，需要确保防火墙将这些端口打开，以备使用。每个数据节点可以配置不同的serverPort。 +**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。（另外还可能有 RESTful、Arbitrator 所使用的端口，那样的话就一共是 13 个。）使用时，需要确保防火墙将这些端口打开，以备使用。每个数据节点可以配置不同的serverPort。（详细的端口情况请参见 [TDengine 2.0 端口说明](https://www.taosdata.com/cn/documentation/faq#port)） **集群对外连接:** TDengine集群可以容纳单个、多个甚至几千个数据节点。应用只需要向集群中任何一个数据节点发起连接即可，连接需要提供的网络参数是一数据节点的End Point(FQDN加配置的端口号）。通过命令行CLI启动应用taos时，可以通过选项-h来指定数据节点的FQDN, -P来指定其配置的端口号，如果端口不配置，将采用TDengine的系统配置参数serverPort。 diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 6811315e7d..3dbf72a23e 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -743,7 +743,7 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 下面仅列出一些与RESTful接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 -- httpPort: 对外提供RESTful服务的端口号，默认绑定到6041 +- 对外提供RESTful服务的端口号，默认绑定到 6041（实际取值是 serverPort + 11，因此可以通过修改 serverPort 参数的设置来修改） - httpMaxThreads: 启动的线程数量，默认为2（2.0.17版本开始，默认值改为CPU核数的一半向下取整） - restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index b89016fafb..90c9b6d082 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -100,8 +100,7 @@ taosd -C - firstEp: taosd启动时，主动连接的集群中首个dnode的end point, 默认值为localhost:6030。 - fqdn：数据节点的FQDN，缺省为操作系统配置的第一个hostname。如果习惯IP地址访问，可设置为该节点的IP地址。 -- serverPort：taosd启动后，对外服务的端口号，默认值为6030。 -- httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求, 默认值为6041。 +- serverPort：taosd启动后，对外服务的端口号，默认值为6030。（RESTful服务使用的端口号是在此基础上+11，即默认值为6041。） - dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。 - logDir：日志文件目录，客户端和服务器的运行日志文件将写入该目录。默认值：/var/log/taos。 - arbitrator：系统中裁决器的end point, 缺省值为空。 @@ -115,7 +114,7 @@ taosd -C - queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为 MB（2.0.15 以前的版本中，此参数的单位是字节）。 - ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程；最大值2表示最大建立2倍CPU核数的查询线程。默认为1，表示最大和CPU核数相等的查询线程。该值可以为小数，即0.5表示最大建立CPU核数一半的查询线程。 -**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030到6042共13个端口，而且必须TCP和UDP都打开。 +**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030到6042共13个端口，而且必须TCP和UDP都打开。（详细的端口情况请参见 [TDengine 2.0 端口说明](https://www.taosdata.com/cn/documentation/faq#port)） 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index c01247d345..5a0f890cae 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -166,3 +166,18 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 2. 如果在 taos.cfg 中设置了 timezone 参数，则客户端会以这个配置文件中的设置为准。 3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。 + +## 19. TDengine 都会用到哪些网络端口？ + +在 TDengine 2.0 版本中，会用到以下这些网络端口（以默认端口 6030 为前提进行说明，如果修改了配置文件中的设置，那么这里列举的端口都会出现变化），管理员可以参考这里的信息调整防火墙设置： + +| 协议 | 默认端口 | 用途说明 | 修改方法 | +| --- | --------- | ------------------------------- | ------------------------------ | +| TCP | 6030 | 客户端与服务端之间通讯。 | 由配置文件设置 serverPort 决定。 | +| TCP | 6035 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | +| TCP | 6040 | 多节点集群的节点间数据同步。 | 随 serverPort 端口变化。 | +| TCP | 6041 | 客户端与服务端之间的 RESTful 通讯。 | 随 serverPort 端口变化。 | +| TCP | 6042 | Arbitrator 的服务端口。 | 因 Arbitrator 启动参数设置变化。 | +| TCP | 6060 | 企业版内 Monitor 服务的网络端口。 | | +| UDP | 6030-6034 | 客户端与服务端之间通讯。 | 随 serverPort 端口变化。 | +| UDP | 6035-6039 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | From bcebfbe844b4dfd238ebb880fbff896f51d50cad Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 15 Apr 2021 09:51:55 +0800 Subject: [PATCH 22/35] [TD-3762]: support us and ms percision for timestamp in JDBC (#5818) * [TD-3762]: support us and ms percision for timestamp in JDBC * change * change * change * change --- .../com/taosdata/jdbc/AbstractConnection.java | 18 +- .../com/taosdata/jdbc/TSDBConnection.java | 1 + .../java/com/taosdata/jdbc/TSDBDriver.java | 5 + .../java/com/taosdata/jdbc/TSDBResultSet.java | 7 +- .../taosdata/jdbc/TSDBResultSetRowData.java | 15 +- .../taosdata/jdbc/rs/RestfulConnection.java | 1 + .../taosdata/jdbc/rs/RestfulResultSet.java | 62 ++++++- .../taosdata/jdbc/rs/RestfulStatement.java | 30 ++-- .../taosdata/jdbc/utils/UtcTimestampUtil.java | 12 ++ .../TwoTypeTimestampPercisionInJniTest.java | 89 ++++++++++ ...woTypeTimestampPercisionInRestfulTest.java | 168 ++++++++++++++++++ 11 files changed, 383 insertions(+), 25 deletions(-) create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java index 976078da95..2970f6c2d3 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java @@ -4,13 +4,23 @@ import java.sql.*; import java.util.Enumeration; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.*; public abstract class AbstractConnection extends WrapperImpl implements Connection { protected volatile boolean isClosed; protected volatile String catalog; - protected volatile Properties clientInfoProps = new Properties(); + protected final Properties clientInfoProps = new Properties(); + + protected AbstractConnection(Properties properties) { + Set propNames = properties.stringPropertyNames(); + for (String propName : propNames) { + clientInfoProps.setProperty(propName, properties.getProperty(propName)); + } + String timestampFormat = properties.getProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, "STRING"); + clientInfoProps.setProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, timestampFormat); + } @Override public abstract Statement createStatement() throws SQLException; @@ -35,7 +45,6 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti } - @Override public void setAutoCommit(boolean autoCommit) throws SQLException { if (isClosed()) @@ -441,9 +450,8 @@ public abstract class AbstractConnection extends WrapperImpl implements Connecti if (isClosed) throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); - if (clientInfoProps == null) - clientInfoProps = new Properties(); - clientInfoProps.setProperty(name, value); + if (clientInfoProps != null) + clientInfoProps.setProperty(name, value); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 5b4615485d..c8ab9fb15a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -28,6 +28,7 @@ public class TSDBConnection extends AbstractConnection { } public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { + super(info); this.databaseMetaData = meta; connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 5f599df130..bbd8519a03 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -95,6 +95,11 @@ public class TSDBDriver extends AbstractDriver { */ public static final String PROPERTY_KEY_BATCH_LOAD = "batchfetch"; + /** + * timestamp format for JDBC-RESTful,should one of the options: string or timestamp or utc + */ + public static final String PROPERTY_KEY_TIMESTAMP_FORMAT = "timestampFormat"; + private TSDBDatabaseMetaData dbMetaData = null; static { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index a20ddaa836..2576a25f0d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -203,7 +203,11 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { this.lastWasNull = this.rowData.wasNull(columnIndex - 1); if (!lastWasNull) { - res = this.rowData.getLong(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + Object value = this.rowData.get(columnIndex - 1); + if (value instanceof Timestamp) + res = ((Timestamp) value).getTime(); + else + res = this.rowData.getLong(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); } return res; } @@ -273,7 +277,6 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, this.columnMetaDataList.size()); Timestamp res = null; - if (this.getBatchFetch()) return this.blockData.getTimestamp(columnIndex - 1); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index 7cf5f0d79a..34470fbc4e 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -17,6 +17,7 @@ package com.taosdata.jdbc; import java.math.BigDecimal; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -299,7 +300,19 @@ public class TSDBResultSetRowData { } public void setTimestamp(int col, long ts) { - data.set(col, new Timestamp(ts)); + //TODO: this implementation contains logical error + // when precision is us the (long ts) is 16 digital number + // when precision is ms, the (long ts) is 13 digital number + // we need a JNI function like this: + // public void setTimestamp(int col, long epochSecond, long nanoAdjustment) + if (ts < 1_0000_0000_0000_0L) { + data.set(col, new Timestamp(ts)); + } else { + long epochSec = ts / 1000_000l; + long nanoAdjustment = ts % 1000_000l * 1000l; + Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + data.set(col, timestamp); + } } public Timestamp getTimestamp(int col) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index 1f3ed2d144..b810f9aeb5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -22,6 +22,7 @@ public class RestfulConnection extends AbstractConnection { private final DatabaseMetaData metadata; public RestfulConnection(String host, String port, Properties props, String database, String url) { + super(props); this.host = host; this.port = Integer.parseInt(port); this.database = database; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 5c2d4c45b0..32e517f564 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -5,13 +5,14 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; -import com.taosdata.jdbc.AbstractResultSet; -import com.taosdata.jdbc.TSDBConstants; -import com.taosdata.jdbc.TSDBError; -import com.taosdata.jdbc.TSDBErrorNumbers; +import com.taosdata.jdbc.*; +import com.taosdata.jdbc.utils.UtcTimestampUtil; import java.math.BigDecimal; import java.sql.*; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Calendar; @@ -19,6 +20,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { private volatile boolean isClosed; private int pos = -1; + private final String database; private final Statement statement; // data @@ -65,7 +67,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { } } - private Object parseColumnData(JSONArray row, int colIndex, int taosType) { + private Object parseColumnData(JSONArray row, int colIndex, int taosType) throws SQLException { switch (taosType) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: return row.getBoolean(colIndex); @@ -81,8 +83,44 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return row.getFloat(colIndex); case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return row.getDouble(colIndex); - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - return new Timestamp(row.getDate(colIndex).getTime()); + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + if (row.get(colIndex) == null) + return null; + String timestampFormat = this.statement.getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT); + if ("TIMESTAMP".equalsIgnoreCase(timestampFormat)) { + Long value = row.getLong(colIndex); + //TODO: + if (value < 1_0000_0000_0000_0L) + return new Timestamp(value); + long epochSec = value / 1000_000l; + long nanoAdjustment = value % 1000_000l * 1000l; + return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + } + if ("UTC".equalsIgnoreCase(timestampFormat)) { + String value = row.getString(colIndex); + long epochSec = Timestamp.valueOf(value.substring(0, 19).replace("T", " ")).getTime() / 1000; + int fractionalSec = Integer.parseInt(value.substring(20, value.length() - 5)); + long nanoAdjustment = 0; + if (value.length() > 28) { + // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSSSSS+0x00 + nanoAdjustment = fractionalSec * 1000l; + } else { + // ms timestamp: yyyy-MM-ddTHH:mm:ss.SSS+0x00 + nanoAdjustment = fractionalSec * 1000_000l; + } + ZoneOffset zoneOffset = ZoneOffset.of(value.substring(value.length() - 5)); + Instant instant = Instant.ofEpochSecond(epochSec, nanoAdjustment).atOffset(zoneOffset).toInstant(); + return Timestamp.from(instant); + } + String value = row.getString(colIndex); + if (value.length() <= 23) // ms timestamp: yyyy-MM-dd HH:mm:ss.SSS + return row.getTimestamp(colIndex); + // us timestamp: yyyy-MM-dd HH:mm:ss.SSSSSS + long epochSec = Timestamp.valueOf(value.substring(0, 19)).getTime() / 1000; + long nanoAdjustment = Integer.parseInt(value.substring(20)) * 1000l; + Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + return timestamp; + } case TSDBConstants.TSDB_DATA_TYPE_BINARY: return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes(); case TSDBConstants.TSDB_DATA_TYPE_NCHAR: @@ -215,6 +253,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { Object value = resultSet.get(pos).get(columnIndex - 1); if (value == null) return 0; + if (value instanceof Timestamp) { + return ((Timestamp) value).getTime(); + } long valueAsLong = 0; try { @@ -307,6 +348,13 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return null; if (value instanceof Timestamp) return (Timestamp) value; +// if (value instanceof Long) { +// if (1_0000_0000_0000_0L > (long) value) +// return Timestamp.from(Instant.ofEpochMilli((long) value)); +// long epochSec = (long) value / 1000_000L; +// long nanoAdjustment = (long) ((long) value % 1000_000L * 1000); +// return Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); +// } return Timestamp.valueOf(value.toString()); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 9071c04672..e9cc3a009f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.taosdata.jdbc.AbstractStatement; +import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBError; import com.taosdata.jdbc.TSDBErrorNumbers; import com.taosdata.jdbc.utils.HttpClientPoolUtil; @@ -34,14 +35,11 @@ public class RestfulStatement extends AbstractStatement { if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); - final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { - return executeOneQuery(url, sql); + return executeOneQuery(sql); } -// if (this.database != null && !this.database.trim().replaceAll("\\s","").isEmpty()) -// HttpClientPoolUtil.execute(url, "use " + this.database); - return executeOneQuery(url, sql); + return executeOneQuery(sql); } @Override @@ -56,8 +54,6 @@ public class RestfulStatement extends AbstractStatement { return executeOneUpdate(url, sql); } -// if (this.database != null && !this.database.trim().replaceAll("\\s", "").isEmpty()) -// HttpClientPoolUtil.execute(url, "use " + this.database); return executeOneUpdate(url, sql); } @@ -78,14 +74,21 @@ public class RestfulStatement extends AbstractStatement { //如果执行了use操作应该将当前Statement的catalog设置为新的database boolean result = true; - final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("TIMESTAMP")) { + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt"; + } + if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("UTC")) { + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc"; + } + if (SqlSyntaxValidator.isUseSql(sql)) { HttpClientPoolUtil.execute(url, sql); this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); result = false; } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { - executeOneQuery(url, sql); + executeOneQuery(sql); } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { executeOneUpdate(url, sql); result = false; @@ -101,11 +104,18 @@ public class RestfulStatement extends AbstractStatement { return result; } - private ResultSet executeOneQuery(String url, String sql) throws SQLException { + private ResultSet executeOneQuery(String sql) throws SQLException { if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); // row data + String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + String timestampFormat = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT); + if ("TIMESTAMP".equalsIgnoreCase(timestampFormat)) + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt"; + if ("UTC".equalsIgnoreCase(timestampFormat)) + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc"; + String result = HttpClientPoolUtil.execute(url, sql); JSONObject resultJson = JSON.parseObject(result); if (resultJson.getString("status").equals("error")) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java new file mode 100644 index 0000000000..04a11a2beb --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/UtcTimestampUtil.java @@ -0,0 +1,12 @@ +package com.taosdata.jdbc.utils; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; + +public class UtcTimestampUtil { + public static final DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-ddTHH:mm:ss.SSS+") +// .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java new file mode 100644 index 0000000000..f9b111bb12 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInJniTest.java @@ -0,0 +1,89 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class TwoTypeTimestampPercisionInJniTest { + + private static final String host = "127.0.0.1"; + private static final String ms_timestamp_db = "ms_precision_test"; + private static final String us_timestamp_db = "us_precision_test"; + private static final long timestamp1 = System.currentTimeMillis(); + private static final long timestamp2 = timestamp1 * 1000 + 123; + + private static Connection conn; + + @Test + public void testCase1() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); + rs.next(); + long ts = rs.getTimestamp(1).getTime(); + Assert.assertEquals(timestamp1, ts); + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase2() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + us_timestamp_db + ".weather"); + rs.next(); + + Timestamp timestamp = rs.getTimestamp(1); + System.out.println(timestamp); + long ts = timestamp.getTime(); + Assert.assertEquals(timestamp1, ts); + int nanos = timestamp.getNanos(); + Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); + + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + ms_timestamp_db); + stmt.execute("create database if not exists " + ms_timestamp_db + " precision 'ms'"); + stmt.execute("create table " + ms_timestamp_db + ".weather(ts timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ms_timestamp_db + ".weather(ts,f1) values(" + timestamp1 + ", 127)"); + + stmt.execute("drop database if exists " + us_timestamp_db); + stmt.execute("create database if not exists " + us_timestamp_db + " precision 'us'"); + stmt.execute("create table " + us_timestamp_db + ".weather(ts timestamp, f1 int)"); + stmt.executeUpdate("insert into " + us_timestamp_db + ".weather(ts,f1) values(" + timestamp2 + ", 127)"); + stmt.close(); + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java new file mode 100644 index 0000000000..ed4f979ef3 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TwoTypeTimestampPercisionInRestfulTest.java @@ -0,0 +1,168 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class TwoTypeTimestampPercisionInRestfulTest { + + private static final String host = "127.0.0.1"; + private static final String ms_timestamp_db = "ms_precision_test"; + private static final String us_timestamp_db = "us_precision_test"; + private static final long timestamp1 = System.currentTimeMillis(); + private static final long timestamp2 = timestamp1 * 1000 + 123; + + private static Connection conn1; + private static Connection conn2; + private static Connection conn3; + + @Test + public void testCase1() { + try (Statement stmt = conn1.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); + rs.next(); + long ts = rs.getTimestamp(1).getTime(); + Assert.assertEquals(timestamp1, ts); + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase2() { + try (Statement stmt = conn1.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + us_timestamp_db + ".weather"); + rs.next(); + + Timestamp timestamp = rs.getTimestamp(1); + long ts = timestamp.getTime(); + Assert.assertEquals(timestamp1, ts); + int nanos = timestamp.getNanos(); + Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); + + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase3() { + try (Statement stmt = conn2.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); + rs.next(); + Timestamp rsTimestamp = rs.getTimestamp(1); + long ts = rsTimestamp.getTime(); + Assert.assertEquals(timestamp1, ts); + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase4() { + try (Statement stmt = conn2.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + us_timestamp_db + ".weather"); + rs.next(); + + Timestamp timestamp = rs.getTimestamp(1); + long ts = timestamp.getTime(); + Assert.assertEquals(timestamp1, ts); + int nanos = timestamp.getNanos(); + Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); + + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase5() { + try (Statement stmt = conn3.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + ms_timestamp_db + ".weather"); + rs.next(); + long ts = rs.getTimestamp(1).getTime(); + Assert.assertEquals(timestamp1, ts); + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase6() { + try (Statement stmt = conn3.createStatement()) { + ResultSet rs = stmt.executeQuery("select last_row(ts) from " + us_timestamp_db + ".weather"); + rs.next(); + + Timestamp timestamp = rs.getTimestamp(1); + long ts = timestamp.getTime(); + Assert.assertEquals(timestamp1, ts); + int nanos = timestamp.getNanos(); + Assert.assertEquals(timestamp2 % 1000_000l * 1000, nanos); + + ts = rs.getLong(1); + Assert.assertEquals(timestamp1, ts); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); +// properties.setProperty(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT, "TIMESTAMP"); + + String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn1 = DriverManager.getConnection(url, properties); + + url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata×tampFormat=timestamp"; + conn2 = DriverManager.getConnection(url, properties); + + url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata×tampFormat=utc"; + conn3 = DriverManager.getConnection(url, properties); + + Statement stmt = conn1.createStatement(); + stmt.execute("drop database if exists " + ms_timestamp_db); + stmt.execute("create database if not exists " + ms_timestamp_db + " precision 'ms'"); + stmt.execute("create table " + ms_timestamp_db + ".weather(ts timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ms_timestamp_db + ".weather(ts,f1) values(" + timestamp1 + ", 127)"); + + stmt.execute("drop database if exists " + us_timestamp_db); + stmt.execute("create database if not exists " + us_timestamp_db + " precision 'us'"); + stmt.execute("create table " + us_timestamp_db + ".weather(ts timestamp, f1 int)"); + stmt.executeUpdate("insert into " + us_timestamp_db + ".weather(ts,f1) values(" + timestamp2 + ", 127)"); + stmt.close(); + } + + @AfterClass + public static void afterClass() { + try { + if (conn1 != null) + conn1.close(); + if (conn2 != null) + conn2.close(); + if (conn3 != null) + conn3.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} From 80a87159ffa92e64c24c5d8a4a3c1088936d9175 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Thu, 15 Apr 2021 10:56:48 +0800 Subject: [PATCH 23/35] [TD-2976]: check 1970 timestamp support in C# taosdemo (#5813) --- tests/examples/C#/taosdemo/taosdemo.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs index 7c34f1ec04..4fc7232ad2 100644 --- a/tests/examples/C#/taosdemo/taosdemo.cs +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -385,7 +385,7 @@ namespace TDengineDriver public void CreateDb() { StringBuilder sql = new StringBuilder(); - sql.Append("CREATE DATABASE IF NOT EXISTS ").Append(this.dbName).Append(" replica ").Append(this.replica); + sql.Append("CREATE DATABASE IF NOT EXISTS ").Append(this.dbName).Append(" replica ").Append(this.replica).Append(" keep 36500"); IntPtr res = TDengine.Query(this.conn, sql.ToString()); if (res != IntPtr.Zero) { @@ -728,7 +728,7 @@ namespace TDengineDriver int m = now.Minute; int s = now.Second; - long baseTimestamp = 1609430400000; // 2021/01/01 0:0:0 + long baseTimestamp = -16094340000; // 1969-06-29 01:21:00 VerbosePrintFormat("beginTime is {0} + {1}h:{2}m:{3}s\n", baseTimestamp, h, m, s); long beginTimestamp = baseTimestamp + ((h * 60 + m) * 60 + s) * 1000; Random random = new Random(); From de01b7b98f60da3b0c71c1b81647a21fa7fe9186 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 15 Apr 2021 15:30:08 +0800 Subject: [PATCH 24/35] [TD-3755] : describe the columns num limitation in selecting. --- documentation20/cn/12.taos-sql/docs.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index b6d81720cf..ebb781bceb 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -1226,6 +1226,7 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P - 列名最大长度为 64，最多允许 1024 列，最少需要 2 列，第一列必须是时间戳 - 标签最多允许 128 个，可以 1 个，标签总长度不超过 16k 个字符 - SQL 语句最大长度 65480 个字符，但可通过系统配置参数 maxSQLLength 修改，最长可配置为 1M +- SELECT 语句的查询结果，最多允许返回 1024 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 ## TAOS SQL其他约定 From 93ad8531e7311c8c484346026b5989dfbab1773b Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 16 Apr 2021 15:27:24 +0800 Subject: [PATCH 25/35] [TD-2639] : fix version num of long term support release. --- documentation20/cn/11.administrator/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index 90c9b6d082..f8dc794d7d 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -149,7 +149,7 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 - maxVgroupsPerDb: 每个数据库中能够使用的最大vgroup个数。 - arbitrator: 系统中裁决器的end point，缺省为空。 -- timezone、locale、charset 的配置见客户端配置。（2.0.19.1 及以上的版本里，集群中加入新节点已不要求 locale 和 charset 参数取值一致） +- timezone、locale、charset 的配置见客户端配置。（2.0.20.0 及以上的版本里，集群中加入新节点已不要求 locale 和 charset 参数取值一致） 为方便调试，可通过SQL语句临时调整每个dnode的日志配置，系统重启后会失效： From a7f1d41e84b6184c0a9f037eb7850173b8146877 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 19 Apr 2021 13:59:48 +0800 Subject: [PATCH 26/35] [TD-3611] : fix description about function top & bottom. --- documentation20/cn/12.taos-sql/docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index ebb781bceb..2415b7cd01 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -960,7 +960,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。 + 功能说明： 统计表/超级表中某列的值最大 *k* 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 返回结果数据类型：同应用的字段。 @@ -994,7 +994,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。 + 功能说明：统计表/超级表中某列的值最小 *k* 个非 NULL 值。如果多条数据取值一样，全部取用又会超出 k 条限制时，系统会从相同值中随机选取符合要求的数量返回。 返回结果数据类型：同应用的字段。 From c3dc60e8fcabc62a73663e8c6e40abaa6fec5d1f Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 20 Apr 2021 09:32:53 +0800 Subject: [PATCH 27/35] Release/s120 (#5846) * change version * change jdbc version * [TD-3796]: test null value in JDBC-JNI Co-authored-by: plum-lihui --- cmake/install.inc | 2 +- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 +- src/connector/go | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../NullValueInResultSetForJdbcJniTest.java | 64 +++++++++++++++++++ 8 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java diff --git a/cmake/install.inc b/cmake/install.inc index cb571b1620..50dc4162f9 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.26-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.27-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/cmake/version.inc b/cmake/version.inc index fe4c017c71..8035b31cc7 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.19.0") + SET(TD_VER_NUMBER "2.0.20.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 65bf5447e2..31343ed293 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.19.0' +version: '2.0.20.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.19.0 + - usr/lib/libtaos.so.2.0.20.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/connector/go b/src/connector/go index 050667e5b4..d99751356e 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit d99751356e285696f57bc604304ffafd10287439 diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 83f7563dc3..5d5a6f5f12 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.26-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.27-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 611b1c608a..d4febe70d6 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.26 + 2.0.27 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 719cd40938..b8a88562ab 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.26 + 2.0.27 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java new file mode 100644 index 0000000000..782125144c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcJniTest.java @@ -0,0 +1,64 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class NullValueInResultSetForJdbcJniTest { + + private static final String host = "127.0.0.1"; + Connection conn; + + @Test + public void test() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + Object value = rs.getObject(i); + System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); + } + System.out.println(); + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() throws SQLException { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)"); + stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)"); + stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)"); + stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)"); + stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)"); + stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)"); + stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)"); + stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')"); + stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} From a5ecf2793ac1a55f6436f0c26a4c5c692a306a4f Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 20 Apr 2021 17:35:47 +0800 Subject: [PATCH 28/35] [TD-3869] : add notes about "TAOS_SUBSCRIBE_CALLBACK". --- documentation20/cn/07.advanced-features/docs.md | 2 +- documentation20/cn/08.connector/docs.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/07.advanced-features/docs.md b/documentation20/cn/07.advanced-features/docs.md index 650a2ca96b..1077f299ee 100644 --- a/documentation20/cn/07.advanced-features/docs.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -120,7 +120,7 @@ if (async) { } ``` -TDengine中的订阅既可以是同步的，也可以是异步的，上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据，而异步则由API在内部的另一个线程中调用`taos_consume`，然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 +TDengine中的订阅既可以是同步的，也可以是异步的，上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据，而异步则由API在内部的另一个线程中调用`taos_consume`，然后把拉取到的数据交给回调函数`subscribe_callback`去处理。（注意，`subscribe_callback` 中不宜做较为耗时的操作，否则有可能导致客户端阻塞等不可控的问题。） 参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误，因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。 diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 3dbf72a23e..ad3179c310 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -377,6 +377,7 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * res：查询结果集，注意结果集中可能没有记录 * param：调用 `taos_subscribe`时客户程序提供的附加参数 * code：错误码 + **注意**：在这个回调函数里不可以做耗时过长的处理，尤其是对于返回的结果集中数据较多的情况，否则有可能导致客户端阻塞等异常状态。如果必须进行复杂计算，则建议在另外的线程中进行处理。 * `TAOS_RES *taos_consume(TAOS_SUB *tsub)` From 41a58388f541b0efed6240b4a29aa274f391ca54 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 21 Apr 2021 15:37:39 +0800 Subject: [PATCH 29/35] [TD-3880]: C# driver test miss unsigned type. (#5871) for develop branch. Co-authored-by: Shuduo Sang --- src/connector/C#/TDengineDriver.cs | 22 ++- tests/examples/C#/TDengineDriver.cs | 263 +++++++++++++++------------- tests/examples/C#/TDengineTest.cs | 16 ++ 3 files changed, 171 insertions(+), 130 deletions(-) diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs index 205269501d..2c150341f6 100644 --- a/src/connector/C#/TDengineDriver.cs +++ b/src/connector/C#/TDengineDriver.cs @@ -31,7 +31,11 @@ namespace TDengineDriver TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes TSDB_DATA_TYPE_BINARY = 8, // string TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes - TSDB_DATA_TYPE_NCHAR = 10 // unicode string + TSDB_DATA_TYPE_NCHAR = 10, // unicode string + TSDB_DATA_TYPE_UTINYINT = 11,// 1 byte + TSDB_DATA_TYPE_USMALLINT= 12,// 2 bytes + TSDB_DATA_TYPE_UINT = 13, // 4 bytes + TSDB_DATA_TYPE_UBIGINT= 14 // 8 bytes } enum TDengineInitOption @@ -53,15 +57,23 @@ namespace TDengineDriver switch ((TDengineDataType)type) { case TDengineDataType.TSDB_DATA_TYPE_BOOL: - return "BOOLEAN"; + return "BOOL"; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - return "BYTE"; + return "TINYINT"; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - return "SHORT"; + return "SMALLINT"; case TDengineDataType.TSDB_DATA_TYPE_INT: return "INT"; case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - return "LONG"; + return "BIGINT"; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + return "TINYINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + return "SMALLINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + return "INT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + return "BIGINT UNSIGNED"; case TDengineDataType.TSDB_DATA_TYPE_FLOAT: return "FLOAT"; case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index b6f143e181..2c150341f6 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -19,136 +19,149 @@ using System.Runtime.InteropServices; namespace TDengineDriver { - enum TDengineDataType { - TSDB_DATA_TYPE_NULL = 0, // 1 bytes - TSDB_DATA_TYPE_BOOL = 1, // 1 bytes - TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes - TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes - TSDB_DATA_TYPE_INT = 4, // 4 bytes - TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes - TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes - TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes - TSDB_DATA_TYPE_BINARY = 8, // string - TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes - TSDB_DATA_TYPE_NCHAR = 10 // unicode string - } - - enum TDengineInitOption - { - TSDB_OPTION_LOCALE = 0, - TSDB_OPTION_CHARSET = 1, - TSDB_OPTION_TIMEZONE = 2, - TDDB_OPTION_CONFIGDIR = 3, - TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 - } - - class TDengineMeta - { - public string name; - public short size; - public byte type; - public string TypeName() + enum TDengineDataType { - switch ((TDengineDataType)type) - { - case TDengineDataType.TSDB_DATA_TYPE_BOOL: - return "BOOLEAN"; - case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - return "BYTE"; - case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - return "SHORT"; - case TDengineDataType.TSDB_DATA_TYPE_INT: - return "INT"; - case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - return "LONG"; - case TDengineDataType.TSDB_DATA_TYPE_FLOAT: - return "FLOAT"; - case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE"; - case TDengineDataType.TSDB_DATA_TYPE_BINARY: - return "STRING"; - case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP"; - case TDengineDataType.TSDB_DATA_TYPE_NCHAR: - return "NCHAR"; - default: - return "undefine"; - } - } - } - - class TDengine - { - public const int TSDB_CODE_SUCCESS = 0; - - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] - static extern public void Init(); - - [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] - static extern public void Cleanup(); - - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] - static extern public void Options(int option, string value); - - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_errstr(IntPtr res); - static public string Error(IntPtr res) - { - IntPtr errPtr = taos_errstr(res); - return Marshal.PtrToStringAnsi(errPtr); + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10, // unicode string + TSDB_DATA_TYPE_UTINYINT = 11,// 1 byte + TSDB_DATA_TYPE_USMALLINT= 12,// 2 bytes + TSDB_DATA_TYPE_UINT = 13, // 4 bytes + TSDB_DATA_TYPE_UBIGINT= 14 // 8 bytes } - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] - static extern public int ErrorNo(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Query(IntPtr conn, string sqlstr); - - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] - static extern public int AffectRows(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] - static extern public int FieldCount(IntPtr res); - - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_fetch_fields(IntPtr res); - static public List FetchFields(IntPtr res) + enum TDengineInitOption { - const int fieldSize = 68; - - List metas = new List(); - if (res == IntPtr.Zero) - { - return metas; - } - - int fieldCount = FieldCount(res); - IntPtr fieldsPtr = taos_fetch_fields(res); - - for (int i = 0; i < fieldCount; ++i) - { - int offset = i * fieldSize; - - TDengineMeta meta = new TDengineMeta(); - meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); - metas.Add(meta); - } - - return metas; + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 } - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FetchRows(IntPtr res); + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOL"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "TINYINT"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SMALLINT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "BIGINT"; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + return "TINYINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + return "SMALLINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + return "INT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + return "BIGINT UNSIGNED"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FreeResult(IntPtr res); + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] - static extern public int Close(IntPtr taos); - } -} \ No newline at end of file + [DllImport("taos", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); + + [DllImport("taos", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); + + [DllImport("taos", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + + [DllImport("taos", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } + + [DllImport("taos", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); + + [DllImport("taos", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; + + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + } +} diff --git a/tests/examples/C#/TDengineTest.cs b/tests/examples/C#/TDengineTest.cs index 6b3f1160ad..24c555e8d8 100644 --- a/tests/examples/C#/TDengineTest.cs +++ b/tests/examples/C#/TDengineTest.cs @@ -410,6 +410,22 @@ namespace TDengineDriver string v10 = Marshal.PtrToStringAnsi(data); builder.Append(v10); break; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + byte v11 = Marshal.ReadByte(data); + builder.Append(v11); + break; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + ushort v12 = (ushort)Marshal.ReadInt16(data); + builder.Append(v12); + break; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + uint v13 = (uint)Marshal.ReadInt32(data); + builder.Append(v13); + break; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + ulong v14 = (ulong)Marshal.ReadInt64(data); + builder.Append(v14); + break; } } builder.Append("---"); From 27b1adf87656c4e94dbf87317df75c12e7121646 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 22 Apr 2021 14:13:43 +0800 Subject: [PATCH 30/35] [TD-2639] : fix minor typo. --- documentation20/cn/13.faq/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index 5a0f890cae..e561f91c94 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -102,7 +102,7 @@ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支 批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 -## 12. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ +## 12. windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： ```JAVA From 9cfe0cd76bcc83a026dba3a9501a0a0d4c0be9e7 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 22 Apr 2021 14:35:29 +0800 Subject: [PATCH 31/35] [TD-3666] : add notes about Binary type. --- documentation20/cn/12.taos-sql/docs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 2415b7cd01..04e67eb4e2 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -48,15 +48,15 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM | 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | | 5 | DOUBLE | 8 | 双精度浮点型，有效位数 15-16，范围 [-1.7E308, 1.7E308] | -| 6 | BINARY | 自定义 | 用于记录 ASCII 型字符串。理论上，最长可以有 16374 字节，但由于每行数据最多 16K 字节，实际上限一般小于理论值。 binary 仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如 binary(20) 定义了最长为 20 个字符的字符串，每个字符占 1 byte 的存储空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 | +| 6 | BINARY | 自定义 | 记录二进制字节型字符串，建议只用于处理 ASCII 可见字符。理论上，最长可以有 16374 字节，但由于每行数据最多 16K 字节，实际上限一般小于理论值。 binary 仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如 binary(20) 定义了最长为 20 个字符的字符串，每个字符占 1 byte 的存储空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 | | 7 | SMALLINT | 2 | 短整型， 范围 [-32767, 32767], -32768 用于 NULL | | 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128 用于 NULL | | 9 | BOOL | 1 | 布尔型，{true, false} | -| 10 | NCHAR | 自定义 | 用于记录非 ASCII 型字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | +| 10 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | **Tips**: 1. TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 -2. 应避免使用 BINARY 类型来保存非 ASCII 型的字符串，会很容易导致数据乱码等错误。正确的做法是使用 NCHAR 类型来保存中文字符。 +2. **注意**，虽然 Binary 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 Binary 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 nchar 类型进行保存。如果强行使用 Binary 类型保存中文字符，虽然有时也能正常读写，但很容易出现数据乱码甚至数据损坏。 ## 数据库管理 From d60028bc5c4053921c699fc1bc71f108a26cf68a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 22 Apr 2021 14:38:04 +0800 Subject: [PATCH 32/35] [TM-205] : fix description about default value of "offlineThreshold". --- documentation20/cn/11.administrator/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index f8dc794d7d..72fcd05d52 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -144,7 +144,7 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - numOfMnodes：系统中管理节点个数。默认值：3。 - balance：是否启动负载均衡。0：否，1：是。默认值：1。 - mnodeEqualVnodeNum: 一个mnode等同于vnode消耗的个数。默认值：4。 -- offlineThreshold: dnode离线阈值，超过该时间将导致该dnode从集群中删除。单位为秒，默认值：86400*10（即10天）。 +- offlineThreshold: dnode离线阈值，超过该时间将导致该dnode从集群中删除。单位为秒，默认值：86400*100（即100天）。 - statusInterval: dnode向mnode报告状态时长。单位为秒，默认值：1。 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 - maxVgroupsPerDb: 每个数据库中能够使用的最大vgroup个数。 From 55af2031b9bdb04391bc261dd3c2194962c1f0a7 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 22 Apr 2021 17:25:07 +0800 Subject: [PATCH 33/35] [TD-3666] : should avoid storing multi-bytes chars in Binary. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 04e67eb4e2..17e494a7a9 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -56,7 +56,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM **Tips**: 1. TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 -2. **注意**，虽然 Binary 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 Binary 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 nchar 类型进行保存。如果强行使用 Binary 类型保存中文字符，虽然有时也能正常读写，但很容易出现数据乱码甚至数据损坏。 +2. **注意**，虽然 Binary 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 Binary 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 nchar 类型进行保存。如果强行使用 Binary 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏。 ## 数据库管理 From 490b34c5d90ee13cc7a917ffbb017983bf7b2f58 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 23 Apr 2021 09:56:40 +0800 Subject: [PATCH 34/35] Hotfix/td 3841 (#5892) * [TD-3841]: getFloat methed get 0 when data value is null * change jdbc driver version number * change * [TD-3841]: test Mybatis query null value * change --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/AbstractResultSet.java | 3 +- .../taosdata/jdbc/rs/RestfulResultSet.java | 52 +++++++---- .../com/taosdata/jdbc/cases/TD3841Test.java | 91 +++++++++++++++++++ tests/examples/JDBC/springbootdemo/pom.xml | 4 +- .../SpringbootdemoApplication.java | 1 + .../controller/WeatherController.java | 2 +- .../springbootdemo/domain/Weather.java | 12 +-- .../service/WeatherService.java | 2 +- 12 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java diff --git a/cmake/install.inc b/cmake/install.inc index 50dc4162f9..9e325531d5 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.27-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.28-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 5d5a6f5f12..de4b8f6bfb 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.27-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.28-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index d4febe70d6..a31796ffde 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.27 + 2.0.28 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index b8a88562ab..3400a82e73 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.27 + 2.0.28 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java index abd348e68c..4b5b88d93b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java @@ -10,6 +10,7 @@ import java.util.Map; public abstract class AbstractResultSet extends WrapperImpl implements ResultSet { private int fetchSize; + protected boolean wasNull; protected void checkAvailability(int columnIndex, int bounds) throws SQLException { if (isClosed()) @@ -28,7 +29,7 @@ public abstract class AbstractResultSet extends WrapperImpl implements ResultSet @Override public boolean wasNull() throws SQLException { - return false; + return wasNull; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 32e517f564..db635f5f79 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -6,12 +6,10 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import com.taosdata.jdbc.*; -import com.taosdata.jdbc.utils.UtcTimestampUtil; import java.math.BigDecimal; import java.sql.*; import java.time.Instant; -import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Calendar; @@ -89,7 +87,7 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { String timestampFormat = this.statement.getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT); if ("TIMESTAMP".equalsIgnoreCase(timestampFormat)) { Long value = row.getLong(colIndex); - //TODO: + //TODO: this implementation has bug if the timestamp bigger than 9999_9999_9999_9 if (value < 1_0000_0000_0000_0L) return new Timestamp(value); long epochSec = value / 1000_000l; @@ -164,12 +162,12 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { } } - @Override - public boolean wasNull() throws SQLException { - if (isClosed()) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - return resultSet.isEmpty(); - } +// @Override +// public boolean wasNull() throws SQLException { +// if (isClosed()) +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); +// return resultSet.isEmpty(); +// } @Override public String getString(int columnIndex) throws SQLException { @@ -188,8 +186,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return false; + } + wasNull = false; if (value instanceof Boolean) return (boolean) value; return Boolean.valueOf(value.toString()); @@ -200,8 +201,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; long valueAsLong = Long.parseLong(value.toString()); if (valueAsLong == Byte.MIN_VALUE) return 0; @@ -221,8 +225,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; long valueAsLong = Long.parseLong(value.toString()); if (valueAsLong == Short.MIN_VALUE) return 0; @@ -236,8 +243,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; long valueAsLong = Long.parseLong(value.toString()); if (valueAsLong == Integer.MIN_VALUE) return 0; @@ -251,12 +261,14 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; if (value instanceof Timestamp) { return ((Timestamp) value).getTime(); } - long valueAsLong = 0; try { valueAsLong = Long.parseLong(value.toString()); @@ -273,8 +285,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; if (value instanceof Float || value instanceof Double) return (float) value; return Float.parseFloat(value.toString()); @@ -285,8 +300,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return 0; + } + wasNull = false; if (value instanceof Double || value instanceof Float) return (double) value; return Double.parseDouble(value.toString()); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java new file mode 100644 index 0000000000..c6fba81eb2 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TD3841Test.java @@ -0,0 +1,91 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.utils.TimestampUtil; +import org.junit.*; + +import java.sql.*; +import java.util.Properties; + +public class TD3841Test { + private static final String host = "127.0.0.1"; + private static Properties properties; + private static Connection conn_restful; + private static Connection conn_jni; + + @Test + public void testRestful() throws SQLException { + String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn_restful = DriverManager.getConnection(url, properties); + + try (Statement stmt = conn_restful.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 smallint, f7 tinyint, f8 bool, f9 binary(64), f10 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, " + TimestampUtil.datetimeToLong("2021-04-21 12:00:00.000") + ")"); + ResultSet rs = stmt.executeQuery("select * from weather"); + rs.next(); + + Assert.assertEquals("2021-04-21 12:00:00.000", TimestampUtil.longToDatetime(rs.getTimestamp(2).getTime())); + Assert.assertEquals(true, rs.getInt(3) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getLong(4) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getFloat(5) == 0.0f && rs.wasNull()); + Assert.assertEquals(true, rs.getDouble(6) == 0.0f && rs.wasNull()); + Assert.assertEquals(true, rs.getByte(7) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getShort(8) == 0 && rs.wasNull()); + Assert.assertEquals(null, rs.getBytes(9)); + Assert.assertEquals(null, rs.getString(10)); + + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testJNI() throws SQLException { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn_jni = DriverManager.getConnection(url, properties); + + try (Statement stmt = conn_jni.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 smallint, f7 tinyint, f8 bool, f9 binary(64), f10 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, " + TimestampUtil.datetimeToLong("2021-04-21 12:00:00.000") + ")"); + ResultSet rs = stmt.executeQuery("select * from weather"); + rs.next(); + + Assert.assertEquals("2021-04-21 12:00:00.000", TimestampUtil.longToDatetime(rs.getTimestamp(2).getTime())); + Assert.assertEquals(true, rs.getInt(3) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getLong(4) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getFloat(5) == 0.0f && rs.wasNull()); + Assert.assertEquals(true, rs.getDouble(6) == 0.0f && rs.wasNull()); + Assert.assertEquals(true, rs.getByte(7) == 0 && rs.wasNull()); + Assert.assertEquals(true, rs.getShort(8) == 0 && rs.wasNull()); + Assert.assertEquals(null, rs.getBytes(9)); + Assert.assertEquals(null, rs.getString(10)); + + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + } + + @AfterClass + public static void afterClass() throws SQLException { + if (conn_restful != null) + conn_restful.close(); + if (conn_jni != null) + conn_jni.close(); + } +} diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index bd5f7efbc0..6c83718896 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -63,9 +63,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.20 + 2.0.28 - + diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index 8f30c29946..fa10f3b092 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -7,6 +7,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; @MapperScan(basePackages = {"com.taosdata.example.springbootdemo.dao"}) @SpringBootApplication public class SpringbootdemoApplication { + public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index c153e27701..cf14f5d84a 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -45,7 +45,7 @@ public class WeatherController { * @return */ @PostMapping("/{temperature}/{humidity}") - public int saveWeather(@PathVariable float temperature, @PathVariable int humidity) { + public int saveWeather(@PathVariable float temperature, @PathVariable float humidity) { return weatherService.save(temperature, humidity); } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java index 255b2cdca9..c11b9a6f50 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -8,8 +8,8 @@ public class Weather { @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS", timezone = "GMT+8") private Timestamp ts; - private float temperature; - private float humidity; + private Float temperature; + private Float humidity; private String location; private int groupId; @@ -30,19 +30,19 @@ public class Weather { this.ts = ts; } - public float getTemperature() { + public Float getTemperature() { return temperature; } - public void setTemperature(float temperature) { + public void setTemperature(Float temperature) { this.temperature = temperature; } - public float getHumidity() { + public Float getHumidity() { return humidity; } - public void setHumidity(float humidity) { + public void setHumidity(Float humidity) { this.humidity = humidity; } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index 0aef828e1c..26d09c7d12 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -39,7 +39,7 @@ public class WeatherService { return weatherMapper.select(limit, offset); } - public int save(float temperature, int humidity) { + public int save(float temperature, float humidity) { Weather weather = new Weather(); weather.setTemperature(temperature); weather.setHumidity(humidity); From a72bb339c79700c920b0f8bb87d2801acd589644 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 23 Apr 2021 09:57:06 +0800 Subject: [PATCH 35/35] Hotfix/sangshuduo/td 3580 taosdump human timeformat (#5887) * [TD-3580] : taosdump support human readable time format. * [TD-3580] : taosdump support human readable time format. fix no-value-given segfault. * [TD-3580]: taosdump supports human readable time format. fix condition * [TD-3580]: taosdump support human readable time format. provide more info. Co-authored-by: Shuduo Sang --- src/kit/taosdump/taosdump.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 092374cef1..96a1cd16f8 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -214,8 +214,8 @@ static struct argp_option options[] = { // dump format options {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Epoch or ISO8601/RFC3339 format is acceptable. For example: 2017-10-01T18:00:00+0800", 3}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, @@ -482,7 +482,8 @@ static int queryDbImpl(TAOS *taos, char *command) { static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-E") == 0) { + if ((strcmp(argv[i], "-S") == 0) + || (strcmp(argv[i], "-E") == 0)) { if (argv[i+1]) { char *tmp = strdup(argv[++i]); @@ -509,7 +510,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(-1); } } else { - errorPrint("%s() LN%d, -E need a valid value following!\n", __func__, __LINE__); + errorPrint("%s need a valid value following!\n", argv[i]); exit(-1); } } else if (strcmp(argv[i], "-g") == 0) { @@ -522,7 +523,8 @@ int main(int argc, char *argv[]) { /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ - parse_args(argc, argv, &g_args); + if (argc > 1) + parse_args(argc, argv, &g_args); argp_parse(&argp, argc, argv, 0, 0, &g_args);