From a214d562abf578423524b98ddc66a55b0a5dc9f6 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 23 Jun 2020 09:57:06 +0800 Subject: [PATCH 01/63] [modify for coverity scan] --- src/util/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 02d72dd1f4..5feda312b1 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -56,7 +56,7 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, year -= 1; } - int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + + int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + (int64_t)(367*mon)/12 + day) + year*365 - 719499)*24 + hour)*60 + min)*60 + sec); return (res + timezone); From 8808272bedbc4ade32a03d96e4510d082bd2d3b7 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 23 Jun 2020 11:21:26 +0800 Subject: [PATCH 02/63] [modify for coverity scan] --- src/kit/taosdump/taosdump.c | 29 +++++++++++++++++++---------- src/util/src/tlog.c | 9 +++++---- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 678de7daa7..63cdf259d6 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -644,14 +644,15 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { (void)lseek(fd, 0, SEEK_SET); + STableRecord tableInfo; while (1) { - memset(&tableRecord, 0, sizeof(STableRecord)); - ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord)); + memset(&tableInfo, 0, sizeof(STableRecord)); + ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord)); if (ret <= 0) break; - tableRecord.name[sizeof(tableRecord.name) - 1] = 0; - tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0; - taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); + tableInfo.name[sizeof(tableInfo.name) - 1] = 0; + tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0; + taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp); } close(fd); @@ -910,14 +911,22 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) { (void)lseek(fd, 0, SEEK_SET); + STableRecord tableInfo; + char tableName[TSDB_TABLE_NAME_LEN] ; + char metricName[TSDB_TABLE_NAME_LEN]; while (1) { - memset(&tableRecord, 0, sizeof(STableRecord)); - ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord)); + memset(&tableInfo, 0, sizeof(STableRecord)); + memset(tableName, 0, TSDB_TABLE_NAME_LEN); + memset(metricName, 0, TSDB_TABLE_NAME_LEN); + ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord)); if (ret <= 0) break; - tableRecord.name[sizeof(tableRecord.name) - 1] = 0; - tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0; - taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); + //tableInfo.name[sizeof(tableInfo.name) - 1] = 0; + //tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0; + //taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp); + tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1); + tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1); + taosDumpTable(tableName, metricName, arguments, fp); } close(fd); diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 50dae7b177..581c4e2e9f 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -276,14 +276,15 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { } } - sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); + char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; + sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); pthread_mutex_init(&tsLogObj.logMutex, NULL); umask(0); - tsLogObj.logHandle->fd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (tsLogObj.logHandle->fd < 0) { - printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); + printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } taosLockFile(tsLogObj.logHandle->fd); @@ -291,7 +292,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { // only an estimate for number of lines struct stat filestat; if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) { - printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); + printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filestat.st_size; From f74a895febd21bc580d6cd01fb9c25e7aa404882 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 24 Jun 2020 09:32:47 +0800 Subject: [PATCH 03/63] [modify for coverity scan] --- src/os/linux/src/linuxSysPara.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index a1d013fa72..fce22d6160 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -160,7 +160,7 @@ static void taosGetSystemTimezone() { /* load time zone string from /etc/timezone */ FILE *f = fopen("/etc/timezone", "r"); - char buf[65] = {0}; + char buf[68] = {0}; if (f != NULL) { int len = fread(buf, 64, 1, f); if(len < 64 && ferror(f)) { From 850b1fe3fac7a5c78eb203b6ea72ed2b2c1d8745 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 24 Jun 2020 10:36:32 +0800 Subject: [PATCH 04/63] [modify for coverity scan] --- src/os/linux/src/linuxSysPara.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index fce22d6160..96a8d1cb81 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -170,18 +170,17 @@ static void taosGetSystemTimezone() { } fclose(f); - } - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } } - // get and set default timezone tzset(); From 19db51164d32e9b16684abf04ac3852931e55236 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 23 Jul 2020 13:51:33 +0800 Subject: [PATCH 05/63] test coverity --- tests/pytest/util/dnodes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 582bd0abae..ec3865f4f2 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -351,7 +351,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -360,7 +360,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -467,7 +467,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -476,7 +476,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( From a9c951bd1243b03600429d8daa07e7029f9272df Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 18 Aug 2020 14:28:35 +0800 Subject: [PATCH 06/63] just a test --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac3a6dac07..2ba4a08c84 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2430,7 +2430,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co } CATCH( code ) { CLEANUP_EXECUTE(); terrno = code; - tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases + //tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases goto _error; // TODO: more error handling From d123d8001f7808d510852b2796a433fab6ddb403 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Sep 2020 11:19:45 +0800 Subject: [PATCH 07/63] TD-1415 --- src/vnode/src/vnodeWrite.c | 11 ++++++++-- tests/test/c/CMakeLists.txt | 12 +++++------ tests/test/c/createNormalTable.c | 35 ++++++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 1a9b05ed34..0c310439bb 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -130,8 +130,15 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe int code = TSDB_CODE_SUCCESS; STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont); - if (pCfg == NULL) return terrno; - if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno; + if (pCfg == NULL) { + ASSERT(terrno != 0); + return terrno; + } + + if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) { + code = terrno; + ASSERT(code != 0); + } tsdbClearTableCfg(pCfg); return code; diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index ffab39d41c..c75316de81 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,16 +31,16 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - #add_executable(createNormalTable createNormalTable.c) - #target_link_libraries(createNormalTable taos_static tutil common pthread) + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) - add_executable(httpTest httpTest.c) - target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + #add_executable(httpTest httpTest.c) + #target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) - add_executable(cacheTest cacheTest.c) - target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + #add_executable(cacheTest cacheTest.c) + #target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() diff --git a/tests/test/c/createNormalTable.c b/tests/test/c/createNormalTable.c index 18a648b9e1..60253e2add 100644 --- a/tests/test/c/createNormalTable.c +++ b/tests/test/c/createNormalTable.c @@ -50,7 +50,9 @@ void createDbAndSTable(); int main(int argc, char *argv[]) { shellParseArgument(argc, argv); taos_init(); - createDbAndSTable(); + if (replica != 0) { + createDbAndSTable(); + } pPrint("%d threads are spawned to create table", numOfThreads); @@ -134,14 +136,31 @@ void *threadFunc(void *param) { int64_t startMs = taosGetTimestampMs(); - for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { - sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); - if (code != 0) { - pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + if (replica != 0) { + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + taos_free_result(pSql); + } + } else { + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "insert into %s%d values(now, 1)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + if (code != TSDB_CODE_MND_INVALID_TABLE_NAME) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + if (code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + exit(0); + } + } + taos_free_result(pSql); } - taos_free_result(pSql); } float createTableSpeed = 0; From 36d126c7cec90907b7d4dc0fe13e613aeab7c924 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 22 Sep 2020 12:51:37 +0800 Subject: [PATCH 08/63] update version.inc to 2.0.4.0 --- cmake/version.inc | 2 +- src/connector/go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 52d62fca65..aa8a4b6463 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.3.0") + SET(TD_VER_NUMBER "2.0.4.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/src/connector/go b/src/connector/go index 8c58c512b6..567b7b12f3 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 +Subproject commit 567b7b12f3fd2775c718d284beffc8c38dd6c219 From 40b6ce676142816ac74be97ada419e7179abdd7e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Sep 2020 15:33:08 +0800 Subject: [PATCH 09/63] [td-225] fix coverity problem --- src/client/src/tscUtil.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 20c3bc2cb6..7387c23e1f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -402,6 +402,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { return; } + void *p = pSql; + tscDebug("%p start to free sqlObj", pSql); STscObj* pTscObj = pSql->pTscObj; @@ -421,7 +423,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { tsem_destroy(&pSql->rspSem); free(pSql); - tscDebug("%p free sqlObj completed", pSql); + + tscDebug("%p free sqlObj completed", p); int32_t ref = T_REF_DEC(pTscObj); assert(ref >= 0); From fb064f5337835778bf15efa36f684eb7a894b1fb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Sep 2020 10:52:43 +0000 Subject: [PATCH 10/63] minor changes --- cmake/version.inc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/version.inc b/cmake/version.inc index aa8a4b6463..52fbe3ca58 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -42,6 +42,12 @@ IF (DEFINED CPUTYPE) ELSE () IF (TD_WINDOWS_32) SET(TD_VER_CPUTYPE "x86") + ELSEIF (TD_LINUX_32) + SET(TD_VER_CPUTYPE "x86") + ELSEIF (TD_ARM_32) + SET(TD_VER_CPUTYPE "x86") + ELSEIF (TD_MIPS_32) + SET(TD_VER_CPUTYPE "x86") ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () From 8ba206115a18239e43453abd11186648b04d16ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Sep 2020 14:39:14 +0800 Subject: [PATCH 11/63] [td-1575] --- src/util/inc/tskiplist.h | 1 + src/util/src/tskiplist.c | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 4ba620dce0..a14a856561 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -136,6 +136,7 @@ typedef struct SSkipListIterator { SSkipListNode *cur; int32_t step; // the number of nodes that have been checked already int32_t order; // order of the iterator + SSkipListNode *next; // next points to the true qualified node in skip list } SSkipListIterator; /** diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 303c2440bf..bacdaef6c8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -79,9 +79,12 @@ static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order) { +static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order, SSkipListNode** pCur) { __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; + if (pCur != NULL) { + *pCur = NULL; + } if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; @@ -93,6 +96,9 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ pNode = p; p = SL_GET_FORWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } @@ -107,6 +113,9 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ pNode = p; p = SL_GET_BACKWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } @@ -295,7 +304,7 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { pthread_rwlock_wrlock(pSkipList->lock); } - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -452,7 +461,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { pthread_rwlock_wrlock(pSkipList->lock); } - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -545,7 +554,7 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* pthread_rwlock_rdlock(pSkipList->lock); } - iter->cur = getPriorNode(pSkipList, val, order); + iter->cur = getPriorNode(pSkipList, val, order, &iter->next); if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); @@ -567,8 +576,22 @@ bool tSkipListIterNext(SSkipListIterator *iter) { if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0); } else { // descending order iterate iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0); } if (pSkipList->lock) { @@ -715,9 +738,11 @@ SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) iter->order = order; if(order == TSDB_ORDER_ASC) { iter->cur = pSkipList->pHead; + iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0); } else { iter->cur = pSkipList->pTail; + iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0); } - + return iter; } \ No newline at end of file From 5337236c04f7d40fba79785dc269f8f3ea007726 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Sep 2020 13:19:57 +0000 Subject: [PATCH 12/63] [TD-1595] --- src/client/src/tscSQLParser.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b9d4cc13d1..f690d13164 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -820,20 +820,20 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, then set it into path SStrToken t = {0}; getCurrentDBName(pSql, &t); if (t.n == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); - if (code != 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } else { + code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); + if (code != TSDB_CODE_SUCCESS) { + code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + } } if (code != TSDB_CODE_SUCCESS) { From d627ab2a88e37c9e8f38e4163decb79fb7cc855d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Sep 2020 15:01:18 +0000 Subject: [PATCH 13/63] TD-1574 --- src/common/src/tglobal.c | 2 +- src/os/inc/osDir.h | 1 + src/os/src/detail/CMakeLists.txt | 1 + src/os/src/detail/osDir.c | 53 +++++++++++++++++++++++++++++--- src/util/CMakeLists.txt | 2 +- src/util/src/tlog.c | 12 ++++++-- 6 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index a75e712f38..2630e3c468 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -1014,7 +1014,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsLogKeepDays; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; + cfg.minValue = -365000; cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index 17683743e3..4a522dadb5 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -25,6 +25,7 @@ void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); void taosRename(char* oldName, char *newName); void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); +int32_t taosCompressFile(char *srcFileName, char *destFileName); #ifdef __cplusplus } diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index 9f710e3ddf..afb8935453 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(.) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) AUX_SOURCE_DIRECTORY(. SRC) SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w) diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 93651c78ef..bdb840ce01 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -17,6 +17,9 @@ #include "os.h" #include "tglobal.h" #include "tulog.h" +#include "zlib.h" + +#define COMPRESS_STEP_SIZE 163840 void taosRemoveDir(char *rootDir) { DIR *dir = opendir(rootDir); @@ -73,11 +76,11 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { if (de->d_type & DT_DIR) { continue; } else { - // struct stat fState; - // if (stat(fname, &fState) < 0) { - // continue; - // } int32_t len = (int32_t)strlen(filename); + if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) { + len -= 3; + } + int64_t fileSec = 0; for (int i = len - 1; i >= 0; i--) { if (filename[i] == '.') { @@ -100,3 +103,45 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { closedir(dir); rmdir(rootDir); } + +int32_t taosCompressFile(char *srcFileName, char *destFileName) { + int32_t ret = 0; + int32_t len = 0; + char * data = malloc(COMPRESS_STEP_SIZE); + FILE * srcFp = NULL; + gzFile dstFp = NULL; + + srcFp = fopen(srcFileName, "r"); + if (srcFp == NULL) { + ret = -1; + goto cmp_end; + } + + int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) { + ret = -2; + goto cmp_end; + } + + dstFp = gzdopen(fd, "wb6f"); + if (dstFp == NULL) { + ret = -3; + goto cmp_end; + } + + while (!feof(srcFp)) { + len = (uLong)fread(data, 1, COMPRESS_STEP_SIZE, srcFp); + gzwrite(dstFp, data, len); + } + +cmp_end: + if (srcFp) { + fclose(srcFp); + } + if (dstFp) { + gzclose(dstFp); + } + free(data); + + return ret; +} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index e63a085cc8..89c8e3dc39 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -3,7 +3,7 @@ PROJECT(TDengine) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) -TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4) +TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index e5afe1b68e..09b0933fd6 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -139,14 +139,22 @@ static void taosUnLockFile(int32_t fd) { } static void taosKeepOldLog(char *oldName) { - if (tsLogKeepDays <= 0) return; + if (tsLogKeepDays == 0) return; int64_t fileSec = taosGetTimestampSec(); char fileName[LOG_FILE_NAME_LEN + 20]; snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); taosRename(oldName, fileName); - taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays); + if (tsLogKeepDays < 0) { + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)remove(fileName); + } + } + + taosRemoveOldLogFiles(tsLogDir, ABS(tsLogKeepDays)); } static void *taosThreadToOpenNewFile(void *param) { From 02d1583725496bce0937ff305a5d21b431830155 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Sep 2020 11:09:16 +0800 Subject: [PATCH 14/63] [td-1523] fix memory leaks and refactor some functions. --- src/client/inc/tscUtil.h | 2 ++ src/client/src/tscAsync.c | 5 +---- src/client/src/tscPrepare.c | 6 ++---- src/client/src/tscServer.c | 23 ++++++----------------- src/client/src/tscSql.c | 7 ++----- src/client/src/tscStream.c | 4 +--- src/client/src/tscSub.c | 4 +--- src/client/src/tscSystem.c | 12 +++--------- src/client/src/tscUtil.c | 24 ++++++++++++++---------- src/util/src/tcache.c | 2 +- 10 files changed, 33 insertions(+), 56 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 79a792ab65..0323434a99 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -258,6 +258,8 @@ void tscDoQuery(SSqlObj* pSql); */ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd); +void registerSqlObj(SSqlObj* pSql); + SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql); void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 09610575f6..c5d622e245 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -51,10 +51,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->fp = fp; pSql->fetchFp = fp; - uint64_t handle = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); - - T_REF_INC(pSql->pTscObj); + registerSqlObj(pSql); pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index c4ca6793ff..cdbd8685df 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -545,10 +545,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->cmd.numOfParams = 0; pSql->cmd.batchSize = 0; - - uint64_t handle = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); - T_REF_INC(pSql->pTscObj); + + registerSqlObj(pSql); int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8eaa406bce..b26abf2145 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1987,15 +1987,11 @@ static void createHBObj(STscObj* pObj) { pSql->param = pObj; pSql->pTscObj = pObj; pSql->signature = pSql; + + registerSqlObj(pSql); + tscDebug("%p HB is allocated, pObj:%p", pSql, pObj); + pObj->pHb = pSql; - - tscAddSubqueryInfo(&pObj->pHb->cmd); - - int64_t ad = (int64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &ad, sizeof(int64_t), &pSql, sizeof(int64_t), 2 * 60 * 1000); - T_REF_INC(pObj); - - tscDebug("%p HB is allocated, pObj:%p", pObj->pHb, pObj); } int tscProcessConnectRsp(SSqlObj *pSql) { @@ -2170,11 +2166,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; - T_REF_INC(pNew->pTscObj); - - // TODO add test case on x86 platform - uint64_t adr = (uint64_t) pNew; - pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000); + registerSqlObj(pNew); tscAddSubqueryInfo(&pNew->cmd); @@ -2301,10 +2293,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; - T_REF_INC(pNew->pTscObj); - - uint64_t p = (uint64_t) pNew; - pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); + registerSqlObj(pNew); tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); pNew->fp = tscTableMetaCallBack; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e1a07fdcfe..347e3cb508 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -156,10 +156,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con *taos = pObj; } - T_REF_INC(pSql->pTscObj); - - uint64_t key = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); + registerSqlObj(pSql); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); return pSql; @@ -270,7 +267,7 @@ void taos_close(TAOS *taos) { pHb->pRpcCtx = NULL; } - tscDebug("%p, HB is freed", pHb); + tscDebug("%p HB is freed", pHb); taos_free_result(pHb); } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 35cd09a033..81b8cf7359 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -510,9 +510,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { return; } - uint64_t handle = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); - T_REF_INC(pSql->pTscObj); + registerSqlObj(pSql); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 760c5f5a51..7913e0fa03 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -152,9 +152,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* goto fail; } - uint64_t handle = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); - T_REF_INC(pSql->pTscObj); + registerSqlObj(pSql); code = tsParseSql(pSql, false); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 82ce1d3679..620fe13a9f 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -122,11 +122,8 @@ void taos_init_imp(void) { tscInitMsgsFp(); int queueSize = tsMaxConnections*2; - if (tscEmbedded == 0) { - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / 2.0); - } else { - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / 4.0); - } + double factor = (tscEmbedded == 0)? 2.0:4.0; + tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); if (tscNumOfThreads < 2) tscNumOfThreads = 2; @@ -140,11 +137,8 @@ void taos_init_imp(void) { if(0 == tscEmbedded){ taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); } - - int64_t refreshTime = tsTableMetaKeepTimer; - refreshTime = refreshTime > 10 ? 10 : refreshTime; - refreshTime = refreshTime < 10 ? 10 : refreshTime; + int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime/2, false, tscFreeSqlObjInCache, "sqlObj"); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7f445344b1..5d439927e2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -424,12 +424,12 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); - tscDebug("%p free sqlObj completed", p); - int32_t ref = T_REF_DEC(pTscObj); assert(ref >= 0); + tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); if (ref == 0) { + tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); tscCloseTscObj(pTscObj); } } @@ -1783,6 +1783,16 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { pRes->numOfRows = 0; } +void registerSqlObj(SSqlObj* pSql) { + int64_t DEFAULT_LIFE_TIME = 2 * 600 * 1000; // 1200 sec + + int32_t ref = T_REF_INC(pSql->pTscObj); + tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); + + uint64_t p = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &p, sizeof(uint64_t), DEFAULT_LIFE_TIME); +} + SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { @@ -1822,10 +1832,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); - - T_REF_INC(pNew->pTscObj); - uint64_t p = (uint64_t) pNew; - pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); + registerSqlObj(pNew); return pNew; } @@ -2063,10 +2070,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex); } - T_REF_INC(pNew->pTscObj); - - uint64_t p = (uint64_t) pNew; - pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10); + registerSqlObj(pNew); return pNew; _error: diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 49b9996cf4..2637699adb 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -97,7 +97,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); assert(size > 0); - uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes", pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize); if (pCacheObj->freeFp) { From 3a26dd3bfcd019f5ba45164a220c997c91c69139 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Sep 2020 11:14:35 +0800 Subject: [PATCH 15/63] [td-225]fix compiler error. --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5d439927e2..21524f8fd2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1784,7 +1784,7 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } void registerSqlObj(SSqlObj* pSql) { - int64_t DEFAULT_LIFE_TIME = 2 * 600 * 1000; // 1200 sec + int32_t DEFAULT_LIFE_TIME = 2 * 600 * 1000; // 1200 sec int32_t ref = T_REF_INC(pSql->pTscObj); tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); From 4149545bc0ef7f077f4abfe760d188fd4a922026 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 24 Sep 2020 11:30:54 +0800 Subject: [PATCH 16/63] [TD-1572]: A JdbcTaosdemo is supported now. --- tests/examples/JDBC/JDBCDemo/.gitignore | 2 + tests/examples/JDBC/JDBCDemo/pom.xml | 7 +- .../com/taosdata/example/JdbcTaosdemo.java | 285 ++++++++++++++++++ .../example/domain/JdbcTaosdemoConfig.java | 183 +++++++++++ .../example/task/CreateTableTask.java | 44 +++ .../example/task/InsertTableDatetimeTask.java | 47 +++ .../example/task/InsertTableTask.java | 45 +++ .../taosdata/example/utils/TimeStampUtil.java | 35 +++ .../src/main/resources/log4j.properties | 21 ++ 9 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 tests/examples/JDBC/JDBCDemo/.gitignore create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties diff --git a/tests/examples/JDBC/JDBCDemo/.gitignore b/tests/examples/JDBC/JDBCDemo/.gitignore new file mode 100644 index 0000000000..d587e5fe38 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/.gitignore @@ -0,0 +1,2 @@ +out/ +logs/ diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 92d757edfd..f3a31d08a0 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -63,7 +63,12 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.6 + + + log4j + log4j + 1.2.17 diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java new file mode 100644 index 0000000000..461a639682 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java @@ -0,0 +1,285 @@ +package com.taosdata.example; + +import com.taosdata.example.domain.JdbcTaosdemoConfig; +import com.taosdata.example.task.CreateTableTask; +import com.taosdata.example.task.InsertTableDatetimeTask; +import com.taosdata.example.task.InsertTableTask; +import com.taosdata.example.utils.TimeStampUtil; +import com.taosdata.jdbc.TSDBDriver; +import org.apache.log4j.Logger; + +import java.sql.*; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +public class JdbcTaosdemo { + + private static Logger logger = Logger.getLogger(JdbcTaosdemo.class); + private static AtomicLong beginTimestamp = new AtomicLong(TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000")); + private final JdbcTaosdemoConfig config; + private Connection connection; + private static final String[] locations = {"Beijing", "Shanghai", "Guangzhou", "Shenzhen", "HangZhou", "Tianjin", "Wuhan", "Changsha", "Nanjing", "Xian"}; + private static Random random = new Random(System.currentTimeMillis()); + + public JdbcTaosdemo(JdbcTaosdemoConfig config) { + this.config = config; + } + + private static void printHelp() { + System.out.println("Usage: java -jar JDBCConnectorChecker.jar -h host [OPTION...]"); + System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); + System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); + System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); + System.out.println("-d database Destination database. Default is 'test'"); + System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); + System.out.println("-T num_of_threads The number of threads. Default is 10"); + System.out.println("-t num_of_tables The number of tables. Default is 10000"); + System.out.println("-n num_of_records_per_table The number of records per table. Default is 100000"); + System.out.println("-D delete table Delete data methods. Default is false"); + System.out.println("--help Give this help list"); + } + + public static void main(String[] args) { + JdbcTaosdemoConfig config = JdbcTaosdemoConfig.build(args); + + boolean isHelp = Arrays.asList(args).contains("--help"); + if (isHelp) { + printHelp(); + return; + } + if (config.getHost() == null) { + printHelp(); + return; + } + + boolean infinite = Arrays.asList().contains("--infinite"); + JdbcTaosdemo taosdemo = new JdbcTaosdemo(config); + taosdemo.init(); + taosdemo.dropDatabase(); + taosdemo.createDatabase(); + taosdemo.useDatabase(); + taosdemo.createSuperTable(); + taosdemo.createTableMultiThreads(); + if (infinite) { + taosdemo.insertInfinite(); + } else { + taosdemo.insertMultiThreads(); + taosdemo.countFromSuperTable(); + if (config.isDeleteTable()) + taosdemo.dropSuperTable(); + taosdemo.close(); + } + } + + + /** + * establish the connection + */ + private void init() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + connection = getConnection(config); + if (connection != null) + logger.info("[ OK ] Connection established."); + } catch (ClassNotFoundException | SQLException e) { + logger.error(e.getMessage()); + throw new RuntimeException("connection failed: " + config.getHost()); + } + } + + public static Connection getConnection(JdbcTaosdemoConfig config) throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, config.getHost()); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, config.getUser()); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, config.getPassword()); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + return DriverManager.getConnection("jdbc:TAOS://" + config.getHost() + ":" + config.getPort() + "/" + config.getDbName() + "", properties); + } + + /** + * create database + */ + private void createDatabase() { + String sql = "create database if not exists " + config.getDbName() + " keep " + config.getKeep() + " days " + config.getDays(); + execute(sql); + } + + private void dropDatabase() { + String sql = "drop database if exists " + config.getDbName(); + execute(sql); + } + + /** + * use database + */ + private void useDatabase() { + String sql = "use " + config.getDbName(); + execute(sql); + } + + private void createSuperTable() { + String sql = "create table if not exists " + config.getStbName() + "(ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int)"; + execute(sql); + } + + /** + * create table use super table with multi threads + */ + private void createTableMultiThreads() { + try { + final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); + List threads = new ArrayList<>(); + for (int i = 0; i < config.getNumberOfThreads(); i++) { + Thread thread = new Thread(new CreateTableTask(config, i * tableSize, tableSize), "Thread-" + i); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(); + } + logger.info(">>> Multi Threads create table finished."); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + private void insertInfinite() { + try { + final long startDatetime = TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000"); + final long finishDatetime = TimeStampUtil.datetimeToLong("2030-01-01 00:00:00.000"); + + final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); + List threads = new ArrayList<>(); + for (int i = 0; i < config.getNumberOfThreads(); i++) { + Thread thread = new Thread(new InsertTableDatetimeTask(config, i * tableSize, tableSize, startDatetime, finishDatetime), "Thread-" + i); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(); + } + logger.info(">>> Multi Threads insert table finished."); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + private void insertMultiThreads() { + try { + final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); + final int numberOfRecordsPerTable = config.getNumberOfRecordsPerTable(); + List threads = new ArrayList<>(); + for (int i = 0; i < config.getNumberOfThreads(); i++) { + Thread thread = new Thread(new InsertTableTask(config, i * tableSize, tableSize, numberOfRecordsPerTable), "Thread-" + i); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(); + } + logger.info(">>> Multi Threads insert table finished."); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + public static String insertSql(int tableIndex, JdbcTaosdemoConfig config) { + float current = 10 + random.nextFloat(); + int voltage = 200 + random.nextInt(20); + float phase = random.nextFloat(); + String sql = "insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + + "values(" + beginTimestamp.getAndIncrement() + ", " + current + ", " + voltage + ", " + phase + ") "; + return sql; + } + + public static String insertSql(int tableIndex, long ts, JdbcTaosdemoConfig config) { + float current = 10 + random.nextFloat(); + int voltage = 200 + random.nextInt(20); + float phase = random.nextFloat(); + String sql = "insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + + "values(" + ts + ", " + current + ", " + voltage + ", " + phase + ") "; + return sql; + } + + + public static String createTableSql(int tableIndex, JdbcTaosdemoConfig config) { + String location = locations[random.nextInt(locations.length)]; + return "create table d" + tableIndex + " using " + config.getDbName() + "." + config.getStbName() + " tags('" + location + "'," + tableIndex + ")"; + } + + private void countFromSuperTable() { + String sql = "select count(*) from " + config.getDbName() + "." + config.getStbName(); + executeQuery(sql); + } + + private void close() { + try { + if (connection != null) { + this.connection.close(); + logger.info("connection closed."); + } + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + /** + * drop super table + */ + private void dropSuperTable() { + String sql = "drop table if exists " + config.getDbName() + "." + config.getStbName(); + execute(sql); + } + + /** + * execute sql, use this method when sql is create, alter, drop.. + */ + private void execute(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, execute, (end - start)); + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + private void printSql(String sql, boolean succeed, long cost) { + logger.info("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private void executeQuery(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, true, (end - start)); + printResult(resultSet); + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } + + private void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + sb.append(columnLabel + ": " + value + "\t"); + } + logger.info(sb.toString()); + } + } + +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java new file mode 100644 index 0000000000..efaddbfa0d --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java @@ -0,0 +1,183 @@ +package com.taosdata.example.domain; + +public class JdbcTaosdemoConfig { + + //The host to connect to TDengine. Must insert one + private String host; + //The TCP/IP port number to use for the connection. Default is 6030. + private int port = 6030; + //The TDengine user name to use when connecting to the server. Default is 'root' + private String user = "root"; + //The password to use when connecting to the server. Default is 'taosdata' + private String password = "taosdata"; + //Destination database. Default is 'test' + private String dbName = "test"; + + // + private int keep = 365 * 20; + + private int days = 30; + + //Super table Name. Default is 'meters' + private String stbName = "meters"; + + + //Table name prefix. Default is 'd' + private String tbPrefix = "d"; + //The number of threads. Default is 10. + private int numberOfThreads = 10; + //The number of tables. Default is 10000. + private int numberOfTable = 10000; + //The number of records per table. Default is 100000 + private int numberOfRecordsPerTable = 100000; + //Delete data. Default is false + private boolean deleteTable = true; + + /** + * parse args from command line + * + * @param args command line args + * @return JdbcTaosdemoConfig + */ + public static JdbcTaosdemoConfig build(String[] args) { + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(); + for (int i = 0; i < args.length; i++) { + if ("-h".equals(args[i]) && i < args.length - 1) { + config.setHost(args[++i]); + } + if ("-p".equals(args[i]) && i < args.length - 1) { + config.setPort(Integer.parseInt(args[++i])); + } + if ("-u".equals(args[i]) && i < args.length - 1) { + config.setUser(args[++i]); + } + if ("-P".equals(args[i]) && i < args.length - 1) { + config.setPassword(args[++i]); + } + if ("-d".equals(args[i]) && i < args.length - 1) { + config.setDbName(args[++i]); + } + if ("-m".equals(args[i]) && i < args.length - 1) { + config.setTbPrefix(args[++i]); + } + if ("-T".equals(args[i]) && i < args.length - 1) { + config.setNumberOfThreads(Integer.parseInt(args[++i])); + } + if ("-t".equals(args[i]) && i < args.length - 1) { + config.setNumberOfTable(Integer.parseInt(args[++i])); + } + if ("-n".equals(args[i]) && i < args.length - 1) { + config.setNumberOfRecordsPerTable(Integer.parseInt(args[++i])); + } + if ("-D".equals(args[i]) && i < args.length - 1) { + config.setDeleteTable(Boolean.parseBoolean(args[++i])); + } + + } + return config; + } + + public void setHost(String host) { + this.host = host; + } + + public String getHost() { + return host; + } + + public String getDbName() { + return dbName; + } + + public void setDbName(String dbName) { + this.dbName = dbName; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getStbName() { + return stbName; + } + + public void setStbName(String stbName) { + this.stbName = stbName; + } + + public String getTbPrefix() { + return tbPrefix; + } + + public void setTbPrefix(String tbPrefix) { + this.tbPrefix = tbPrefix; + } + + public int getNumberOfThreads() { + return numberOfThreads; + } + + public void setNumberOfThreads(int numberOfThreads) { + this.numberOfThreads = numberOfThreads; + } + + public int getNumberOfTable() { + return numberOfTable; + } + + public void setNumberOfTable(int numberOfTable) { + this.numberOfTable = numberOfTable; + } + + public int getNumberOfRecordsPerTable() { + return numberOfRecordsPerTable; + } + + public void setNumberOfRecordsPerTable(int numberOfRecordsPerTable) { + this.numberOfRecordsPerTable = numberOfRecordsPerTable; + } + + public boolean isDeleteTable() { + return deleteTable; + } + + public void setDeleteTable(boolean deleteTable) { + this.deleteTable = deleteTable; + } + + public int getKeep() { + return keep; + } + + public void setKeep(int keep) { + this.keep = keep; + } + + public int getDays() { + return days; + } + + public void setDays(int days) { + this.days = days; + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java new file mode 100644 index 0000000000..167dbdb648 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java @@ -0,0 +1,44 @@ +package com.taosdata.example.task; + +import com.taosdata.example.JdbcTaosdemo; +import com.taosdata.example.domain.JdbcTaosdemoConfig; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +public class CreateTableTask implements Runnable { + + private static Logger logger = Logger.getLogger(CreateTableTask.class); + private final JdbcTaosdemoConfig config; + private final int startIndex; + private final int tableNumber; + + public CreateTableTask(JdbcTaosdemoConfig config, int startIndex, int tableNumber) { + this.config = config; + this.startIndex = startIndex; + this.tableNumber = tableNumber; + } + + @Override + public void run() { + try { + Connection connection = JdbcTaosdemo.getConnection(config); + for (int i = startIndex; i < startIndex + tableNumber; i++) { + Statement statement = connection.createStatement(); + String sql = JdbcTaosdemo.createTableSql(i + 1, config); +// long start = System.currentTimeMillis(); + boolean execute = statement.execute(sql); +// long end = System.currentTimeMillis(); +// printSql(sql, execute, (end - start)); + statement.close(); + logger.info(">>> " + sql); + } + connection.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java new file mode 100644 index 0000000000..a54f73a258 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java @@ -0,0 +1,47 @@ +package com.taosdata.example.task; + +import com.taosdata.example.JdbcTaosdemo; +import com.taosdata.example.domain.JdbcTaosdemoConfig; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +public class InsertTableDatetimeTask implements Runnable { + private static Logger logger = Logger.getLogger(InsertTableDatetimeTask.class); + + private final JdbcTaosdemoConfig config; + private final int startTableIndex; + private final int tableNumber; + private final long startDatetime; + private final long finishedDatetime; + + public InsertTableDatetimeTask(JdbcTaosdemoConfig config, int startTableIndex, int tableNumber, long startDatetime, long finishedDatetime) { + this.config = config; + this.startTableIndex = startTableIndex; + this.tableNumber = tableNumber; + this.startDatetime = startDatetime; + this.finishedDatetime = finishedDatetime; + } + + @Override + public void run() { + try { + Connection connection = JdbcTaosdemo.getConnection(config); + for (long ts = startDatetime; ts < finishedDatetime; ts++) { + for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) { + String sql = JdbcTaosdemo.insertSql(i + 1, ts, config); + Statement statement = connection.createStatement(); + statement.execute(sql); + statement.close(); + logger.info(Thread.currentThread().getName() + ">>> " + sql); + } + } + connection.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java new file mode 100644 index 0000000000..88f65f739d --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java @@ -0,0 +1,45 @@ +package com.taosdata.example.task; + +import com.taosdata.example.JdbcTaosdemo; +import com.taosdata.example.domain.JdbcTaosdemoConfig; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +public class InsertTableTask implements Runnable { + private static Logger logger = Logger.getLogger(InsertTableTask.class); + + private final JdbcTaosdemoConfig config; + private final int startIndex; + private final int tableNumber; + private final int recordsNumber; + + public InsertTableTask(JdbcTaosdemoConfig config, int startIndex, int tableNumber, int recordsNumber) { + this.config = config; + this.startIndex = startIndex; + this.tableNumber = tableNumber; + this.recordsNumber = recordsNumber; + } + + @Override + public void run() { + try { + Connection connection = JdbcTaosdemo.getConnection(config); + for (int i = startIndex; i < startIndex + tableNumber; i++) { + for (int j = 0; j < recordsNumber; j++) { + String sql = JdbcTaosdemo.insertSql(i + 1, config); + Statement statement = connection.createStatement(); + statement.execute(sql); + statement.close(); + logger.info(Thread.currentThread().getName() + ">>> " + sql); + } + } + connection.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java new file mode 100644 index 0000000000..2ec91ef1cb --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java @@ -0,0 +1,35 @@ +package com.taosdata.example.utils; + +import java.sql.Date; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +public class TimeStampUtil { + private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + + public static long datetimeToLong(String dateTime) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + try { + return sdf.parse(dateTime).getTime(); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } + + public static String longToDatetime(long time) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + return sdf.format(new Date(time)); + } + + public static void main(String[] args) { + final String startTime = "2005-01-01 00:00:00.000"; + + long start = TimeStampUtil.datetimeToLong(startTime); + System.out.println(start); + + String datetime = TimeStampUtil.longToDatetime(1519833600000L); + System.out.println(datetime); + } + + +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties new file mode 100644 index 0000000000..b445e5f52e --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties @@ -0,0 +1,21 @@ +### 设置### +log4j.rootLogger=debug,stdout,DebugLog,ErrorLog +### 输出信息到控制抬 ### +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n +### 输出DEBUG 级别以上的日志到=logs/error.log ### +log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DebugLog.File=logs/debug.log +log4j.appender.DebugLog.Append=true +log4j.appender.DebugLog.Threshold=DEBUG +log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout +log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n +### 输出ERROR 级别以上的日志到=logs/error.log ### +log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender +log4j.appender.ErrorLog.File=logs/error.log +log4j.appender.ErrorLog.Append=true +log4j.appender.ErrorLog.Threshold=ERROR +log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout +log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n \ No newline at end of file From 4aaf2bd99c7252f3a3b520a7de70e35888e3dd68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Sep 2020 13:52:30 +0800 Subject: [PATCH 17/63] [td-225] refactor codes. --- src/client/inc/tsclient.h | 4 ++-- src/client/src/tscServer.c | 3 ++- src/client/src/tscSql.c | 34 ++++++++++++++++++---------------- src/client/src/tscSub.c | 6 ++++++ src/client/src/tscUtil.c | 27 +++++++++++++-------------- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 55ca02dfb5..ea42b0f6a3 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -89,7 +89,7 @@ typedef struct STableComInfo { typedef struct SCMCorVgroupInfo { int32_t version; - int8_t inUse; + int8_t inUse; int8_t numOfEps; SEpAddr epAddr[TSDB_MAX_REPLICA]; } SCMCorVgroupInfo; @@ -107,7 +107,7 @@ typedef struct STableMeta { } STableMeta; typedef struct STableMetaInfo { - STableMeta * pTableMeta; // table meta, cached in client side and acquired by name + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name SVgroupsInfo *vgroupList; SArray *pVgroupTables; // SArray diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b26abf2145..3cbb0d936e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -128,6 +128,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); taosCorEndWrite(&pVgroupInfo->version); } + void tscPrintMgmtEp() { SRpcEpSet dump; tscDumpMgmtEpSet(&dump); @@ -745,7 +746,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; } @@ -2294,6 +2294,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; registerSqlObj(pNew); + tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); pNew->fp = tscTableMetaCallBack; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 347e3cb508..430a762321 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -786,14 +786,17 @@ int taos_validate_sql(TAOS *taos, const char *sql) { } SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; pSql->signature = pSql; + SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; pRes->numOfTotal = 0; pRes->numOfClauseTotal = 0; + tscDebug("%p Valid SQL: %s pObj:%p", pSql, sql, pObj); int32_t sqlLen = (int32_t)strlen(sql); @@ -829,11 +832,12 @@ int taos_validate_sql(TAOS *taos, const char *sql) { tsem_wait(&pSql->rspSem); code = pSql->res.code; } + if (code != TSDB_CODE_SUCCESS) { tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj); } - taos_free_result(pSql); + taos_free_result(pSql); return code; } @@ -932,34 +936,32 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); pSql->pTscObj = taos; pSql->signature = pSql; + SSqlRes *pRes = &pSql->res; + pRes->code = 0; pRes->numOfTotal = 0; // the number of getting table meta from server pRes->numOfClauseTotal = 0; - pRes->code = 0; - assert(pSql->fp == NULL); tscDebug("%p tableNameList: %s pObj:%p", pSql, tableNameList, pObj); int32_t tblListLen = (int32_t)strlen(tableNameList); if (tblListLen > MAX_TABLE_NAME_LENGTH) { tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH); - pRes->code = TSDB_CODE_TSC_INVALID_SQL; - taosTFree(pSql); - return pRes->code; + tscFreeSqlObj(pSql); + return TSDB_CODE_TSC_INVALID_SQL; } char *str = calloc(1, tblListLen + 1); if (str == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); - taosTFree(pSql); - return pRes->code; + tscFreeSqlObj(pSql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } strtolower(str, tableNameList); - pRes->code = (uint8_t)tscParseTblNameList(pSql, str, tblListLen); + int32_t code = (uint8_t) tscParseTblNameList(pSql, str, tblListLen); /* * set the qhandle to 0 before return in order to erase the qhandle value assigned in the previous successful query. @@ -969,17 +971,17 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { pRes->qhandle = 0; free(str); - if (pRes->code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pSql); - return pRes->code; + return code; } tscDoQuery(pSql); - tscDebug("%p load multi metermeta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); - if (pRes->code != TSDB_CODE_SUCCESS) { - tscPartiallyFreeSqlObj(pSql); + tscDebug("%p load multi table meta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + if ((code = pRes->code) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); } - return pRes->code; + return code; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 7913e0fa03..2c81bd7c7c 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -105,6 +105,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* code = TAOS_SYSTEM_ERROR(errno); goto fail; } + tstrncpy(pSub->topic, topic, sizeof(pSub->topic)); pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); if (pSub->progress == NULL) { @@ -119,6 +120,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto fail; } + pSql->signature = pSql; pSql->pTscObj = pObj; pSql->pSubscription = pSub; @@ -142,6 +144,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto fail; } + strtolower(pSql->sqlstr, pSql->sqlstr); pRes->qhandle = 0; pRes->numOfRows = 1; @@ -159,6 +162,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* tsem_wait(&pSub->sem); code = pSql->res.code; } + if (code != TSDB_CODE_SUCCESS) { line = __LINE__; goto fail; @@ -180,8 +184,10 @@ fail: } else { tscFreeSqlObj(pSql); } + pSql = NULL; } + if (pSub != NULL) { taosArrayDestroy(pSub->progress); tsem_destroy(&pSub->sem); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 21524f8fd2..33362409cf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -391,10 +391,21 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { */ void tscFreeSqlObjInCache(void *pSql) { assert(pSql != NULL); + SSqlObj** p = (SSqlObj**)pSql; + STscObj* pTscObj = (*p)->pTscObj; assert((*p)->self != 0 && (*p)->self == (p)); tscFreeSqlObj(*p); + + int32_t ref = T_REF_DEC(pTscObj); + assert(ref >= 0); + + tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); + if (ref == 0) { + tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); + tscCloseTscObj(pTscObj); + } } void tscFreeSqlObj(SSqlObj* pSql) { @@ -402,10 +413,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { return; } - void *p = pSql; - tscDebug("%p start to free sqlObj", pSql); - STscObj* pTscObj = pSql->pTscObj; tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); @@ -423,15 +431,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { tsem_destroy(&pSql->rspSem); free(pSql); - - int32_t ref = T_REF_DEC(pTscObj); - assert(ref >= 0); - - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); - if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); - tscCloseTscObj(pTscObj); - } } void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { @@ -1821,8 +1820,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed", pSql); - - free(pNew); + tscFreeSqlObj(pNew); return NULL; } @@ -1832,6 +1830,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + registerSqlObj(pNew); return pNew; } From 4806e7bf3dd3b4e7d6ab92bdc111f8ce238c93c1 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 24 Sep 2020 16:30:14 +0800 Subject: [PATCH 18/63] fix td-1488 --- src/client/inc/tsclient.h | 5 +++-- src/client/src/tscParseInsert.c | 3 +-- src/client/src/tscServer.c | 35 ++++++++++----------------------- src/client/src/tscUtil.c | 1 + 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 55ca02dfb5..500f784ec5 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -261,7 +261,7 @@ typedef struct { }; int32_t insertType; - int32_t clauseIndex; // index of multiple subclause query + int32_t clauseIndex; // index of multiple subclause query char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; @@ -276,7 +276,8 @@ typedef struct { int32_t numOfParams; int8_t dataSourceType; // load data from file or not - int8_t submitSchema; // submit block is built with table schema + int8_t submitSchema; // submit block is built with table schema + STagData tagData; SHashObj *pTableList; // referred table involved in sql SArray *pDataBlocks; // SArray submit data blocks after parsing sql } SSqlCmd; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6fd97c09e9..0480370dfa 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -791,7 +791,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sql += index; tscAllocPayload(pCmd, sizeof(STagData)); - STagData *pTag = (STagData *) pCmd->payload; + STagData *pTag = &pCmd->tagData; memset(pTag, 0, sizeof(STagData)); @@ -946,7 +946,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); } - pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen) + pTag->dataLen; pTag->dataLen = htonl(pTag->dataLen); if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8eaa406bce..72fe64d4a4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1495,43 +1495,29 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SCMTableInfoMsg *pInfoMsg; - char * pMsg; - int msgLen = 0; - - char *tmpData = NULL; - uint32_t len = pSql->cmd.payloadLen; - if (len > 0) { - if ((tmpData = calloc(1, len)) == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // STagData is in binary format, strncpy is not available - memcpy(tmpData, pSql->cmd.payload, len); - } - SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pInfoMsg = (SCMTableInfoMsg *)pCmd->payload; + SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload; strcpy(pInfoMsg->tableId, pTableMetaInfo->name); pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); - pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); + char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); - if (pSql->cmd.autoCreated && len > 0) { - memcpy(pInfoMsg->tags, tmpData, len); - pMsg += len; + size_t len = htonl(pCmd->tagData.dataLen); + if (pSql->cmd.autoCreated) { + if (len > 0) { + len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen); + memcpy(pInfoMsg->tags, &pCmd->tagData, len); + pMsg += len; + } } pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; - taosTFree(tmpData); - - assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); return TSDB_CODE_SUCCESS; } @@ -2192,8 +2178,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); - memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists. - pNew->cmd.payloadLen = pSql->cmd.payloadLen; + memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData)); tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); pNew->fp = tscTableMetaCallBack; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 77088f5b7c..875411d9a3 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1794,6 +1794,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->command = cmd; pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; + memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData)); if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pNew); From afd549828ccb6a2fdc88276f9a01838c02187b8d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 24 Sep 2020 17:51:44 +0800 Subject: [PATCH 19/63] fix some err --- tests/pytest/insert/retentionpolicy.py | 48 ++++++++++++++++---------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index c50de31cc4..c69060b5ae 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -43,7 +43,8 @@ class TDTestRetetion: else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows) - os.system("timedatectl set-ntp true") + os.system("sudo timedatectl set-ntp true") + time.sleep(40) tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) def run(self): @@ -53,7 +54,7 @@ class TDTestRetetion: tdSql.execute('use test;') tdSql.execute('create table test(ts timestamp,i int);') - cmd = 'insert into test values(now-2d,11)(now-1d,11)(now,11)(now+1d,11);' + cmd = 'insert into test values(now-2d,1)(now-1d,2)(now,3)(now+1d,4);' tdLog.info(cmd) tdSql.execute(cmd) tdSql.query('select * from test') @@ -61,46 +62,55 @@ class TDTestRetetion: tdLog.info("=============== step2") tdDnodes.stop(1) - os.system("timedatectl set-ntp false") - os.system("date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("sudo timedatectl set-ntp false") + os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") tdDnodes.start(1) - cmd = 'insert into test values(now,11);' + cmd = 'insert into test values(now,5);' + tdDnodes.stop(1) + tdDnodes.start(1) + tdLog.info(cmd) tdSql.execute(cmd) - queryRows=tdSql.query('select * from test') - if queryRows==4: - tdSql.checkRows(4) + self.queryRows=tdSql.query('select * from test') + if self.queryRows==4: + self.checkRows(4,cmd) return 0 else: - tdSql.checkRows(5) - + self.checkRows(5,cmd) tdLog.info("=============== step3") tdDnodes.stop(1) - os.system("date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") tdDnodes.start(1) - cmd = 'insert into test values(now-1d,11);' tdLog.info(cmd) tdSql.execute(cmd) - queryRows=tdSql.query('select * from test') - tdSql.checkRows(6) + self.queryRows=tdSql.query('select * from test') + if self.queryRows==4: + self.checkRows(4,cmd) + return 0 + cmd = 'insert into test values(now-1d,6);' + tdLog.info(cmd) + tdSql.execute(cmd) + self.queryRows=tdSql.query('select * from test') + self.checkRows(6,cmd) tdLog.info("=============== step4") tdDnodes.stop(1) tdDnodes.start(1) - cmd = 'insert into test values(now,11);' + cmd = 'insert into test values(now,7);' tdLog.info(cmd) tdSql.execute(cmd) - tdSql.query('select * from test') - tdSql.checkRows(7) + self.queryRows=tdSql.query('select * from test') + self.checkRows(7,cmd) tdLog.info("=============== step5") tdDnodes.stop(1) tdDnodes.start(1) cmd='select * from test where ts > now-1d' - queryRows=tdSql.query('select * from test where ts > now-1d') + self.queryRows=tdSql.query('select * from test where ts > now-1d') self.checkRows(1,cmd) def stop(self): - os.system("timedatectl set-ntp true") + os.system("sudo timedatectl set-ntp true") + time.sleep(40) tdSql.close() tdLog.success("%s successfully executed" % __file__) From f39cd1bf1e4036714811205b47be29d5a1e3e7c0 Mon Sep 17 00:00:00 2001 From: elvizlai Date: Fri, 25 Sep 2020 10:09:50 +0800 Subject: [PATCH 20/63] typo: fix chinese semicolon --- .../webdocs/markdowndocs/advanced features-ch.md | 10 +++++----- .../webdocs/markdowndocs/architecture-ch.md | 4 ++-- documentation20/webdocs/markdowndocs/cluster-ch.md | 6 +++--- documentation20/webdocs/markdowndocs/cluster.md | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index b1d050c8cc..ed02af25f2 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -39,7 +39,7 @@ create table D1002 using meters tags ("Beijing.Haidian", 2); 我们已经知道，可以通过下面这条SQL语句以一分钟为时间窗口、30秒为前向增量统计这些电表的平均电压。 ```sql -select avg(voltage) from meters interval(1m) sliding(30s)； +select avg(voltage) from meters interval(1m) sliding(30s); ``` 每次执行这条语句，都会重新计算所有数据。 @@ -47,14 +47,14 @@ select avg(voltage) from meters interval(1m) sliding(30s)； 可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行： ```sql -select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s)； +select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s); ``` 这样做没有问题，但TDengine提供了更简单的方法， 只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如： ```sql -create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s)； +create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s); ``` 会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句， @@ -80,7 +80,7 @@ taos> select * from avg_vol; 比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 ```sql -create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s)； +create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` 需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。 @@ -396,7 +396,7 @@ ts: 1597465200000 current: 11.2 voltage: 220 phase: 1 location: Beijing.Haidian ```sql # taos -taos> use power； +taos> use power; taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1); ``` diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index 460e655f5f..7dfff1f21f 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -276,14 +276,14 @@ SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成 在TDengine中引入关键词interval来进行时间轴上固定长度时间窗口的切分，并按照时间窗口对数据进行聚合，对窗口范围内的数据按需进行聚合。例如： ```mysql -select count(*) from d1001 interval(1h)； +select count(*) from d1001 interval(1h); ``` 针对d1001设备采集的数据，按照1小时的时间窗口返回每小时存储的记录数量。 在需要连续获得查询结果的应用场景下，如果给定的时间区间存在数据缺失，会导致该区间数据结果也丢失。TDengine提供策略针对时间轴聚合计算的结果进行插值，通过使用关键词Fill就能够对时间轴聚合结果进行插值。例如： ```mysql -select count(*) from d1001 interval(1h) fill(prev)； +select count(*) from d1001 interval(1h) fill(prev); ``` 针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index c36819e5c7..13b773b3e3 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -89,7 +89,7 @@ taos> 2. 在第一个数据节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: ``` - CREATE DNODE "h2.taos.com:6030"； + CREATE DNODE "h2.taos.com:6030"; ``` 将新数据节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新数据节点的End Point。 @@ -97,7 +97,7 @@ taos> 3. 然后执行命令 ``` - SHOW DNODES； + SHOW DNODES; ``` 查看新节点是否被成功加入。如果该被加入的数据节点处于离线状态，请做两个检查 @@ -122,7 +122,7 @@ taos> 执行CLI程序taos, 使用root账号登录进系统, 执行: ``` -CREATE DNODE "fqdn:port"； +CREATE DNODE "fqdn:port"; ``` 将新数据节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个数据节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置FQDN，可能导致生成的数据节点的End Point不是所期望的】 diff --git a/documentation20/webdocs/markdowndocs/cluster.md b/documentation20/webdocs/markdowndocs/cluster.md index 43b35dbc66..f5fa6af48c 100644 --- a/documentation20/webdocs/markdowndocs/cluster.md +++ b/documentation20/webdocs/markdowndocs/cluster.md @@ -46,7 +46,7 @@ taos> 5. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 使用命令: ``` - CREATE DNODE "h2.taos.com:6030"； + CREATE DNODE "h2.taos.com:6030"; ``` 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point @@ -54,7 +54,7 @@ taos> 6. 使用命令 ``` - SHOW DNODES； + SHOW DNODES; ``` 查看新节点是否被成功加入。 @@ -71,7 +71,7 @@ taos> ###添加节点 执行CLI程序taos, 使用root账号登录进系统, 执行: ``` -CREATE DNODE "fqdn:port"； +CREATE DNODE "fqdn:port"; ``` 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。 From a5a4f7026f8f3bd81e76ac691a6a8ccb3e84eada Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 25 Sep 2020 10:26:58 +0800 Subject: [PATCH 21/63] update jdbcTaosdemo --- tests/examples/JDBC/JDBCDemo/.gitignore | 21 +++++++- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- .../example/JDBCConnectorChecker.java | 1 - .../{ => jdbcTaosdemo}/JdbcTaosdemo.java | 48 +++++++++---------- .../domain/JdbcTaosdemoConfig.java | 27 +++++++---- .../task/CreateTableTask.java | 6 +-- .../task/InsertTableDatetimeTask.java | 13 +++-- .../task/InsertTableTask.java | 8 ++-- .../utils/TimeStampUtil.java | 2 +- .../src/main/resources/log4j.properties | 11 +++-- 10 files changed, 85 insertions(+), 54 deletions(-) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/JdbcTaosdemo.java (84%) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/domain/JdbcTaosdemoConfig.java (78%) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/task/CreateTableTask.java (89%) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/task/InsertTableDatetimeTask.java (79%) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/task/InsertTableTask.java (84%) rename tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/{ => jdbcTaosdemo}/utils/TimeStampUtil.java (95%) diff --git a/tests/examples/JDBC/JDBCDemo/.gitignore b/tests/examples/JDBC/JDBCDemo/.gitignore index d587e5fe38..b79f223d17 100644 --- a/tests/examples/JDBC/JDBCDemo/.gitignore +++ b/tests/examples/JDBC/JDBCDemo/.gitignore @@ -1,2 +1,19 @@ -out/ -logs/ +# custom +/out/ +/logs/ +*.jar + +# Created by .ignore support plugin (hsz.mobi) +.gitignore + +# Build Artifacts +.gradle/* +build/* +target/* +bin/* +dependency-reduced-pom.xml + +# Eclipse Project Files +.classpath +.project +.settings/* diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index f3a31d08a0..2113074674 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -63,7 +63,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.6 + 2.0.4 log4j diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCConnectorChecker.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCConnectorChecker.java index 1e801bc658..24a4b3a77e 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCConnectorChecker.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCConnectorChecker.java @@ -11,7 +11,6 @@ public class JDBCConnectorChecker { private static String tbName = "weather"; private Connection connection; - /** * get connection **/ diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java similarity index 84% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java index 461a639682..874f950132 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcTaosdemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java @@ -1,10 +1,10 @@ -package com.taosdata.example; +package com.taosdata.example.jdbcTaosdemo; -import com.taosdata.example.domain.JdbcTaosdemoConfig; -import com.taosdata.example.task.CreateTableTask; -import com.taosdata.example.task.InsertTableDatetimeTask; -import com.taosdata.example.task.InsertTableTask; -import com.taosdata.example.utils.TimeStampUtil; +import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.task.CreateTableTask; +import com.taosdata.example.jdbcTaosdemo.task.InsertTableDatetimeTask; +import com.taosdata.example.jdbcTaosdemo.task.InsertTableTask; +import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil; import com.taosdata.jdbc.TSDBDriver; import org.apache.log4j.Logger; @@ -25,34 +25,20 @@ public class JdbcTaosdemo { this.config = config; } - private static void printHelp() { - System.out.println("Usage: java -jar JDBCConnectorChecker.jar -h host [OPTION...]"); - System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); - System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); - System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); - System.out.println("-d database Destination database. Default is 'test'"); - System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); - System.out.println("-T num_of_threads The number of threads. Default is 10"); - System.out.println("-t num_of_tables The number of tables. Default is 10000"); - System.out.println("-n num_of_records_per_table The number of records per table. Default is 100000"); - System.out.println("-D delete table Delete data methods. Default is false"); - System.out.println("--help Give this help list"); - } public static void main(String[] args) { JdbcTaosdemoConfig config = JdbcTaosdemoConfig.build(args); boolean isHelp = Arrays.asList(args).contains("--help"); if (isHelp) { - printHelp(); + JdbcTaosdemoConfig.printHelp(); return; } if (config.getHost() == null) { - printHelp(); + JdbcTaosdemoConfig.printHelp(); return; } - boolean infinite = Arrays.asList().contains("--infinite"); JdbcTaosdemo taosdemo = new JdbcTaosdemo(config); taosdemo.init(); taosdemo.dropDatabase(); @@ -60,7 +46,10 @@ public class JdbcTaosdemo { taosdemo.useDatabase(); taosdemo.createSuperTable(); taosdemo.createTableMultiThreads(); + + boolean infinite = Arrays.asList(args).contains("--infinite"); if (infinite) { + logger.info("!!! Infinite Insert Mode Started. !!!!"); taosdemo.insertInfinite(); } else { taosdemo.insertMultiThreads(); @@ -206,6 +195,17 @@ public class JdbcTaosdemo { return sql; } + public static String batchInsertSql(int tableIndex, long ts, int valueCnt, JdbcTaosdemoConfig config) { + float current = 10 + random.nextFloat(); + int voltage = 200 + random.nextInt(20); + float phase = random.nextFloat(); + StringBuilder sb = new StringBuilder(); + sb.append("insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + "values"); + for (int i = 0; i < valueCnt; i++) { + sb.append("(" + (ts + i) + ", " + current + ", " + voltage + ", " + phase + ") "); + } + return sb.toString(); + } public static String createTableSql(int tableIndex, JdbcTaosdemoConfig config) { String location = locations[random.nextInt(locations.length)]; @@ -252,7 +252,7 @@ public class JdbcTaosdemo { } } - private void printSql(String sql, boolean succeed, long cost) { + private static void printSql(String sql, boolean succeed, long cost) { logger.info("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); } @@ -269,7 +269,7 @@ public class JdbcTaosdemo { } } - private void printResult(ResultSet resultSet) throws SQLException { + private static void printResult(ResultSet resultSet) throws SQLException { ResultSetMetaData metaData = resultSet.getMetaData(); while (resultSet.next()) { StringBuilder sb = new StringBuilder(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java similarity index 78% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java index efaddbfa0d..ef0f7bef12 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/domain/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java @@ -1,4 +1,4 @@ -package com.taosdata.example.domain; +package com.taosdata.example.jdbcTaosdemo.domain; public class JdbcTaosdemoConfig { @@ -12,16 +12,12 @@ public class JdbcTaosdemoConfig { private String password = "taosdata"; //Destination database. Default is 'test' private String dbName = "test"; - - // + //keep private int keep = 365 * 20; - + // private int days = 30; - //Super table Name. Default is 'meters' private String stbName = "meters"; - - //Table name prefix. Default is 'd' private String tbPrefix = "d"; //The number of threads. Default is 10. @@ -33,6 +29,20 @@ public class JdbcTaosdemoConfig { //Delete data. Default is false private boolean deleteTable = true; + public static void printHelp() { + System.out.println("Usage: java -jar JDBCConnectorChecker.jar -h host [OPTION...]"); + System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); + System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); + System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); + System.out.println("-d database Destination database. Default is 'test'"); + System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); + System.out.println("-T num_of_threads The number of threads. Default is 10"); + System.out.println("-t num_of_tables The number of tables. Default is 10000"); + System.out.println("-n num_of_records_per_table The number of records per table. Default is 100000"); + System.out.println("-D delete table Delete data methods. Default is false"); + System.out.println("--help Give this help list"); + } + /** * parse args from command line * @@ -40,6 +50,7 @@ public class JdbcTaosdemoConfig { * @return JdbcTaosdemoConfig */ public static JdbcTaosdemoConfig build(String[] args) { + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(); for (int i = 0; i < args.length; i++) { if ("-h".equals(args[i]) && i < args.length - 1) { @@ -72,8 +83,8 @@ public class JdbcTaosdemoConfig { if ("-D".equals(args[i]) && i < args.length - 1) { config.setDeleteTable(Boolean.parseBoolean(args[++i])); } - } + return config; } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java similarity index 89% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java index 167dbdb648..2b1a08ab40 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/CreateTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java @@ -1,7 +1,7 @@ -package com.taosdata.example.task; +package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.JdbcTaosdemo; -import com.taosdata.example.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; +import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; import org.apache.log4j.Logger; import java.sql.Connection; diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java similarity index 79% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java index a54f73a258..d477ceeaed 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableDatetimeTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java @@ -1,7 +1,7 @@ -package com.taosdata.example.task; +package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.JdbcTaosdemo; -import com.taosdata.example.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; +import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; import org.apache.log4j.Logger; import java.sql.Connection; @@ -29,9 +29,12 @@ public class InsertTableDatetimeTask implements Runnable { public void run() { try { Connection connection = JdbcTaosdemo.getConnection(config); - for (long ts = startDatetime; ts < finishedDatetime; ts++) { + int valueCnt = 100; + for (long ts = startDatetime; ts < finishedDatetime; ts+= valueCnt) { for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) { - String sql = JdbcTaosdemo.insertSql(i + 1, ts, config); +// String sql = JdbcTaosdemo.insertSql(i + 1, ts, config); + + String sql = JdbcTaosdemo.batchInsertSql(i + 1, ts, valueCnt, config); Statement statement = connection.createStatement(); statement.execute(sql); statement.close(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java similarity index 84% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java index 88f65f739d..9ae50d830e 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/task/InsertTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java @@ -1,7 +1,7 @@ -package com.taosdata.example.task; +package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.JdbcTaosdemo; -import com.taosdata.example.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; +import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; import org.apache.log4j.Logger; import java.sql.Connection; @@ -9,7 +9,7 @@ import java.sql.SQLException; import java.sql.Statement; public class InsertTableTask implements Runnable { - private static Logger logger = Logger.getLogger(InsertTableTask.class); + private static final Logger logger = Logger.getLogger(InsertTableTask.class); private final JdbcTaosdemoConfig config; private final int startIndex; diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java similarity index 95% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java index 2ec91ef1cb..d00471f581 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/utils/TimeStampUtil.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java @@ -1,4 +1,4 @@ -package com.taosdata.example.utils; +package com.taosdata.example.jdbcTaosdemo.utils; import java.sql.Date; import java.text.ParseException; diff --git a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties index b445e5f52e..0ed95d04d3 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties +++ b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties @@ -1,10 +1,11 @@ ### 设置### -log4j.rootLogger=debug,stdout,DebugLog,ErrorLog +#log4j.rootLogger=debug,stdout,DebugLog,ErrorLog +log4j.rootLogger=debug,DebugLog,ErrorLog ### 输出信息到控制抬 ### -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n +#log4j.appender.stdout=org.apache.log4j.ConsoleAppender +#log4j.appender.stdout.Target=System.out +#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +#log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n ### 输出DEBUG 级别以上的日志到=logs/error.log ### log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log From e74edd3d04462db4acc1ce907876b05d2699b8e6 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 25 Sep 2020 10:34:49 +0800 Subject: [PATCH 22/63] [TD-1504] --- src/connector/go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/go b/src/connector/go index 06ec30a0f1..8d7bf74385 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 06ec30a0f1762e8169bf6b9045c82bcaa52bcdf0 +Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b From c35887815bc01d3f59033067929363e444d3ee80 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 25 Sep 2020 11:47:26 +0800 Subject: [PATCH 23/63] Support custom IP --- tests/examples/python/read_example.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index dd1475ec14..73052ab2df 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -22,8 +22,12 @@ if __name__ == '__main__': # @password : Password # @database : Database to use when connecting to TDengine server # @config : Configuration directory - conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") - + if len(sys.argv)>1: + hostname=sys.argv[1] + conn = taos.connect(host=hostname, user="root", password="taosdata", config="/etc/taos") + else: + conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") + # Generate a cursor object to run SQL commands c1 = conn.cursor() # Create a database named db From a5f68b0ff8722575c1af901fd8e49319cd0a8b2d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Sep 2020 14:22:49 +0800 Subject: [PATCH 24/63] [td-1604]. --- src/client/src/tscParseInsert.c | 44 ++++++++++++++++----------------- src/client/src/tscPrepare.c | 6 ----- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 327aac22d1..c7f168fa59 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -406,7 +406,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start return TSDB_CODE_SUCCESS; } -int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, +int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, SSqlCmd* pCmd, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; SStrToken sToken = {0}; @@ -426,12 +426,17 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ *str += index; if (sToken.type == TK_QUESTION) { + if (pCmd->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { + *code = tscSQLSyntaxErrMsg(pCmd->payload, "? only allowed in binding insertion", *str); + return -1; + } + uint32_t offset = (uint32_t)(start - pDataBlocks->pData); if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { continue; } - strcpy(error, "client out of memory"); + strcpy(pCmd->payload, "client out of memory"); *code = TSDB_CODE_TSC_OUT_OF_MEMORY; return -1; } @@ -439,8 +444,7 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ int16_t type = sToken.type; if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) { - tscSQLSyntaxErrMsg(error, "invalid data or symbol", sToken.z); - *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + *code = tscSQLSyntaxErrMsg(pCmd->payload, "invalid data or symbol", sToken.z); return -1; } @@ -470,14 +474,14 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, error, str, isPrimaryKey, timePrec); + int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, pCmd->payload, str, isPrimaryKey, timePrec); if (ret != TSDB_CODE_SUCCESS) { *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; // NOTE: here 0 mean error! } if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) { - tscInvalidSQLErrMsg(error, "client time/server time can not be mixed up", sToken.z); + tscInvalidSQLErrMsg(pCmd->payload, "client time/server time can not be mixed up", sToken.z); *code = TSDB_CODE_TSC_INVALID_TIME_STAMP; return -1; } @@ -522,7 +526,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, - SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { + SParsedDataColInfo *spd, SSqlCmd* pCmd, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; SStrToken sToken; @@ -534,8 +538,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe int32_t precision = tinfo.precision; if (spd->hasVal[0] == false) { - strcpy(error, "primary timestamp column can not be null"); - *code = TSDB_CODE_TSC_INVALID_SQL; + *code = tscInvalidSQLErrMsg(pCmd->payload, "primary timestamp column can not be null", *str); return -1; } @@ -547,17 +550,17 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe *str += index; if (numOfRows >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { int32_t tSize; - int32_t retcode = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); - if (retcode != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client - strcpy(error, "client out of memory"); - *code = retcode; + *code = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); + if (*code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client + strcpy(pCmd->payload, "client out of memory"); return -1; } + ASSERT(tSize > maxRows); maxRows = tSize; } - int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, error, precision, code, tmpTokenBuf); + int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, pCmd, precision, code, tmpTokenBuf); if (len <= 0) { // error message has been set in tsParseOneRowData return -1; } @@ -568,7 +571,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe sToken = tStrGetToken(*str, &index, false, 0, NULL); *str += index; if (sToken.n == 0 || sToken.type != TK_RP) { - tscSQLSyntaxErrMsg(error, ") expected", *str); + tscSQLSyntaxErrMsg(pCmd->payload, ") expected", *str); *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } @@ -577,7 +580,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe } if (numOfRows <= 0) { - strcpy(error, "no any data points"); + strcpy(pCmd->payload, "no any data points"); *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } else { @@ -704,7 +707,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int32_t numOfRows = tsParseValues(str, dataBuf, pTableMeta, maxNumOfRows, spd, pCmd->payload, &code, tmpTokenBuf); + int32_t numOfRows = tsParseValues(str, dataBuf, pTableMeta, maxNumOfRows, spd, pCmd, &code, tmpTokenBuf); free(tmpTokenBuf); if (numOfRows <= 0) { return code; @@ -724,10 +727,6 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st dataBuf->vgId = pTableMeta->vgroupInfo.vgId; dataBuf->numOfTables = 1; - /* - * the value of pRes->numOfRows does not affect the true result of AFFECTED ROWS, - * which is actually returned from server. - */ *totalNum += numOfRows; return TSDB_CODE_SUCCESS; } @@ -1460,8 +1459,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { char *lineptr = line; strtolower(line, line); - int32_t len = - tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tokenBuf); + int32_t len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd, tinfo.precision, &code, tokenBuf); if (len <= 0 || pTableDataBlock->numOfParams > 0) { pSql->res.code = code; break; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index c4ca6793ff..73425a81b3 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -43,10 +43,6 @@ typedef struct SNormalStmt { tVariant* params; } SNormalStmt; -//typedef struct SInsertStmt { -// -//} SInsertStmt; - typedef struct STscStmt { bool isInsert; STscObj* taos; @@ -54,7 +50,6 @@ typedef struct STscStmt { SNormalStmt normal; } STscStmt; - static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_t len) { uint16_t size = stmt->numParts + 1; if (size > stmt->sizeParts) { @@ -514,7 +509,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { SSqlObj* pSql = pStmt->pSql; size_t sqlLen = strlen(sql); - //doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; pSql->param = (void*) pSql; From d3d4691c946d0823cf36a11a7d95e86a7b650354 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 14:36:20 +0800 Subject: [PATCH 25/63] TD-1550 --- src/tsdb/src/tsdbMemTable.c | 2 + tests/test/c/CMakeLists.txt | 12 +++--- tests/test/c/importPerTable.c | 73 +++++++++++++++++++++++------------ 3 files changed, 57 insertions(+), 30 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f6a7f1b35c..4cf8ddd4bd 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -262,7 +262,9 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pIMem != NULL) { ASSERT(pRepo->commit); + tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo)); code = pthread_join(pRepo->commitThread, NULL); + tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo)); if (code != 0) { tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index ffab39d41c..a4eb0b13a6 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -22,8 +22,8 @@ IF (TD_LINUX) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) - #add_executable(importPerTable importPerTable.c) - #target_link_libraries(importPerTable taos_static pthread) + add_executable(importPerTable importPerTable.c) + target_link_libraries(importPerTable taos_static pthread) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) @@ -37,10 +37,10 @@ IF (TD_LINUX) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) - add_executable(httpTest httpTest.c) - target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + #add_executable(httpTest httpTest.c) + #target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) - add_executable(cacheTest cacheTest.c) - target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + #add_executable(cacheTest cacheTest.c) + #target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() diff --git a/tests/test/c/importPerTable.c b/tests/test/c/importPerTable.c index b4e8b68703..68f7898050 100644 --- a/tests/test/c/importPerTable.c +++ b/tests/test/c/importPerTable.c @@ -20,6 +20,7 @@ #include "ttimer.h" #include "tutil.h" #include "tglobal.h" +#include "osTime.h" #define MAX_RANDOM_POINTS 20000 #define GREEN "\033[1;32m" @@ -43,14 +44,16 @@ void createDbAndTable(); void insertData(); int32_t randomData[MAX_RANDOM_POINTS]; -int64_t rowsPerTable = 10000; +int64_t rowsPerTable = 1000000; int64_t pointsPerTable = 1; -int64_t numOfThreads = 1; -int64_t numOfTablesPerThread = 1; +int64_t numOfThreads = 10; +int64_t numOfTablesPerThread = 100; char dbName[32] = "db"; char stableName[64] = "st"; -int32_t cache = 16384; -int32_t tables = 1000; +int64_t totalUs = 0; +int64_t reqNum = 0; +int64_t maxUs = 0; +int64_t minUs = 100000000; int main(int argc, char *argv[]) { shellParseArgument(argc, argv); @@ -58,6 +61,38 @@ int main(int argc, char *argv[]) { taos_init(); createDbAndTable(); insertData(); + int64_t avgUs = totalUs / reqNum; + pError("%s totalUs:%ld, avgUs:%ld maxUs:%ld minUs:%ld reqNum:%ld %s\n", GREEN, totalUs, avgUs, maxUs, minUs, reqNum, NC); +} + +int32_t query(void *con, char *qstr) { + int64_t begin = taosGetTimestampUs(); + + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to exec sql:%s, code:%d reason:%s", qstr, taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + int64_t us = taosGetTimestampUs() - begin; + maxUs = MAX(us, maxUs); + minUs = MIN(us, minUs); + atomic_add_fetch_64(&totalUs, us); + atomic_add_fetch_64(&reqNum, 1); + if (reqNum > 100000) { + int64_t avgUs = totalUs / reqNum; + if (us > avgUs * 100) { + pError("sql:%s", qstr); + pError("%s totalUs:%ld, avgUs:%ld maxUs:%ld minUs:%ld reqNum:%ld %s\n", GREEN, totalUs, avgUs, maxUs, minUs, + reqNum, NC); + taosMsleep(1000); + exit(0); + } + } + + return code; } void createDbAndTable() { @@ -79,14 +114,14 @@ void createDbAndTable() { exit(1); } - sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); - if (taos_query(con, qstr)) { + sprintf(qstr, "create database if not exists %s", dbName); + if (query(con, qstr)) { pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); exit(0); } sprintf(qstr, "use %s", dbName); - if (taos_query(con, qstr)) { + if (query(con, qstr)) { pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } @@ -102,14 +137,14 @@ void createDbAndTable() { } sprintf(qstr + len, ") tags(t int)"); - if (taos_query(con, qstr)) { + if (query(con, qstr)) { pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } for (int64_t t = 0; t < totalTables; ++t) { sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); - if (taos_query(con, qstr)) { + if (query(con, qstr)) { pError("failed to create table %s%" PRId64 ", reason:%s", stableName, t, taos_errstr(con)); exit(0); } @@ -122,7 +157,7 @@ void createDbAndTable() { } sprintf(qstr + len, ")"); - if (taos_query(con, qstr)) { + if (query(con, qstr)) { pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); exit(0); } @@ -207,7 +242,7 @@ void *syncTest(void *param) { } sprintf(qstr, "use %s", pInfo->dbName); - taos_query(con, qstr); + query(con, qstr); gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; @@ -229,7 +264,7 @@ void *syncTest(void *param) { } len += sprintf(sql + len, ")"); if (len > maxBytes) { - if (taos_query(con, qstr)) { + if (query(con, qstr)) { pError("thread:%d, failed to import table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, table, row, taos_errstr(con)); } @@ -246,7 +281,7 @@ void *syncTest(void *param) { } if (len != strlen(inserStr)) { - taos_query(con, qstr); + query(con, qstr); } gettimeofday(&systemTime, NULL); @@ -284,10 +319,6 @@ void printHelp() { printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); printf("%s%s\n", indent, "-n"); printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread); - printf("%s%s\n", indent, "-tables"); - printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables); - printf("%s%s\n", indent, "-cache"); - printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache); exit(EXIT_SUCCESS); } @@ -311,10 +342,6 @@ void shellParseArgument(int argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTablesPerThread = atoi(argv[++i]); - } else if (strcmp(argv[i], "-tables") == 0) { - tables = atoi(argv[++i]); - } else if (strcmp(argv[i], "-cache") == 0) { - cache = atoi(argv[++i]); } else { } } @@ -323,8 +350,6 @@ void shellParseArgument(int argc, char *argv[]) { pPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC); pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC); - pPrint("%scache:%d%s", GREEN, cache, NC); - pPrint("%stables:%d%s", GREEN, tables, NC); pPrint("%sdbName:%s%s", GREEN, dbName, NC); pPrint("%stableName:%s%s", GREEN, stableName, NC); pPrint("%sstart to run%s", GREEN, NC); From 9431641ff1c9343aa543019d7bd6c8c486404210 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 07:05:49 +0000 Subject: [PATCH 26/63] minor changes --- src/rpc/src/rpcMain.c | 6 +++--- src/rpc/src/rpcTcp.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index cb318d5c24..50b1507a56 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -323,14 +323,14 @@ void *rpcMallocCont(int contLen) { tError("failed to malloc msg, size:%d", size); return NULL; } else { - tDebug("malloc mem: %p", start); + tDebug("malloc msg: %p", start); } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } void rpcFreeCont(void *cont) { - if ( cont ) { + if (cont) { char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); free(temp); tDebug("free mem: %p", temp); @@ -553,7 +553,7 @@ static void rpcFreeMsg(void *msg) { if ( msg ) { char *temp = (char *)msg - sizeof(SRpcReqContext); free(temp); - tDebug("free mem: %p", temp); + tDebug("free msg: %p", temp); } } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 56b3fa8616..2a3facdb36 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -421,7 +421,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); buffer = malloc(msgLen + tsRpcOverhead); - if ( NULL == buffer) { + if (NULL == buffer) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { From 5ec266098f6bb25720bd94a1bd1782c3ec97cae3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 07:10:07 +0000 Subject: [PATCH 27/63] minor changes --- tests/test/c/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 4ff1947aee..befaec60a6 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -22,8 +22,8 @@ IF (TD_LINUX) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) - add_executable(importPerTable importPerTable.c) - target_link_libraries(importPerTable taos_static pthread) + #add_executable(importPerTable importPerTable.c) + #target_link_libraries(importPerTable taos_static pthread) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - add_executable(createNormalTable createNormalTable.c) - target_link_libraries(createNormalTable taos_static tutil common pthread) + #add_executable(createNormalTable createNormalTable.c) + #target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) From 56c8889ee4d6f1f4de2acac791cdce1f73a0d61a Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 25 Sep 2020 15:39:21 +0800 Subject: [PATCH 28/63] update JdbcTaosdemo --- .../example/jdbcTaosdemo/JdbcTaosdemo.java | 88 +++------ .../domain/JdbcTaosdemoConfig.java | 171 +++++++----------- .../jdbcTaosdemo/task/CreateTableTask.java | 12 +- .../task/InsertTableDatetimeTask.java | 13 +- .../jdbcTaosdemo/task/InsertTableTask.java | 17 +- .../jdbcTaosdemo/utils/ConnectionFactory.java | 32 ++++ .../jdbcTaosdemo/utils/SqlSpeller.java | 58 ++++++ .../src/main/resources/log4j.properties | 11 +- 8 files changed, 208 insertions(+), 194 deletions(-) create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java create mode 100644 tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java index 874f950132..5a6acb0291 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java @@ -4,30 +4,29 @@ import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; import com.taosdata.example.jdbcTaosdemo.task.CreateTableTask; import com.taosdata.example.jdbcTaosdemo.task.InsertTableDatetimeTask; import com.taosdata.example.jdbcTaosdemo.task.InsertTableTask; +import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; +import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil; -import com.taosdata.jdbc.TSDBDriver; import org.apache.log4j.Logger; import java.sql.*; -import java.util.*; -import java.util.concurrent.atomic.AtomicLong; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class JdbcTaosdemo { private static Logger logger = Logger.getLogger(JdbcTaosdemo.class); - private static AtomicLong beginTimestamp = new AtomicLong(TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000")); private final JdbcTaosdemoConfig config; private Connection connection; - private static final String[] locations = {"Beijing", "Shanghai", "Guangzhou", "Shenzhen", "HangZhou", "Tianjin", "Wuhan", "Changsha", "Nanjing", "Xian"}; - private static Random random = new Random(System.currentTimeMillis()); public JdbcTaosdemo(JdbcTaosdemoConfig config) { this.config = config; } - public static void main(String[] args) { - JdbcTaosdemoConfig config = JdbcTaosdemoConfig.build(args); + + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); if (isHelp) { @@ -67,7 +66,7 @@ public class JdbcTaosdemo { private void init() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - connection = getConnection(config); + connection = ConnectionFactory.build(config); if (connection != null) logger.info("[ OK ] Connection established."); } catch (ClassNotFoundException | SQLException e) { @@ -76,27 +75,19 @@ public class JdbcTaosdemo { } } - public static Connection getConnection(JdbcTaosdemoConfig config) throws SQLException { - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, config.getHost()); - properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, config.getUser()); - properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, config.getPassword()); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - return DriverManager.getConnection("jdbc:TAOS://" + config.getHost() + ":" + config.getPort() + "/" + config.getDbName() + "", properties); - } - /** * create database */ private void createDatabase() { - String sql = "create database if not exists " + config.getDbName() + " keep " + config.getKeep() + " days " + config.getDays(); + String sql = SqlSpeller.createDatabaseSQL(config.getDbName(), config.getKeep(), config.getDays()); execute(sql); } + /** + * drop database + */ private void dropDatabase() { - String sql = "drop database if exists " + config.getDbName(); + String sql = SqlSpeller.dropDatabaseSQL(config.getDbName()); execute(sql); } @@ -104,12 +95,15 @@ public class JdbcTaosdemo { * use database */ private void useDatabase() { - String sql = "use " + config.getDbName(); + String sql = SqlSpeller.useDatabaseSQL(config.getDbName()); execute(sql); } + /** + * create super table + */ private void createSuperTable() { - String sql = "create table if not exists " + config.getStbName() + "(ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int)"; + String sql = SqlSpeller.createSuperTableSQL(config.getStbName()); execute(sql); } @@ -128,13 +122,16 @@ public class JdbcTaosdemo { for (Thread thread : threads) { thread.join(); } - logger.info(">>> Multi Threads create table finished."); + logger.info("<<< Multi Threads create table finished."); } catch (InterruptedException e) { logger.error(e.getMessage()); e.printStackTrace(); } } + /** + * insert data infinitely + */ private void insertInfinite() { try { final long startDatetime = TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000"); @@ -150,7 +147,7 @@ public class JdbcTaosdemo { for (Thread thread : threads) { thread.join(); } - logger.info(">>> Multi Threads insert table finished."); + logger.info("<<< Multi Threads insert table finished."); } catch (InterruptedException e) { logger.error(e.getMessage()); e.printStackTrace(); @@ -170,48 +167,13 @@ public class JdbcTaosdemo { for (Thread thread : threads) { thread.join(); } - logger.info(">>> Multi Threads insert table finished."); + logger.info("<<< Multi Threads insert table finished."); } catch (InterruptedException e) { logger.error(e.getMessage()); e.printStackTrace(); } } - public static String insertSql(int tableIndex, JdbcTaosdemoConfig config) { - float current = 10 + random.nextFloat(); - int voltage = 200 + random.nextInt(20); - float phase = random.nextFloat(); - String sql = "insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + - "values(" + beginTimestamp.getAndIncrement() + ", " + current + ", " + voltage + ", " + phase + ") "; - return sql; - } - - public static String insertSql(int tableIndex, long ts, JdbcTaosdemoConfig config) { - float current = 10 + random.nextFloat(); - int voltage = 200 + random.nextInt(20); - float phase = random.nextFloat(); - String sql = "insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + - "values(" + ts + ", " + current + ", " + voltage + ", " + phase + ") "; - return sql; - } - - public static String batchInsertSql(int tableIndex, long ts, int valueCnt, JdbcTaosdemoConfig config) { - float current = 10 + random.nextFloat(); - int voltage = 200 + random.nextInt(20); - float phase = random.nextFloat(); - StringBuilder sb = new StringBuilder(); - sb.append("insert into " + config.getDbName() + "." + config.getTbPrefix() + "" + tableIndex + " " + "values"); - for (int i = 0; i < valueCnt; i++) { - sb.append("(" + (ts + i) + ", " + current + ", " + voltage + ", " + phase + ") "); - } - return sb.toString(); - } - - public static String createTableSql(int tableIndex, JdbcTaosdemoConfig config) { - String location = locations[random.nextInt(locations.length)]; - return "create table d" + tableIndex + " using " + config.getDbName() + "." + config.getStbName() + " tags('" + location + "'," + tableIndex + ")"; - } - private void countFromSuperTable() { String sql = "select count(*) from " + config.getDbName() + "." + config.getStbName(); executeQuery(sql); @@ -233,7 +195,7 @@ public class JdbcTaosdemo { * drop super table */ private void dropSuperTable() { - String sql = "drop table if exists " + config.getDbName() + "." + config.getStbName(); + String sql = SqlSpeller.dropSuperTableSQL(config.getDbName(), config.getStbName()); execute(sql); } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java index ef0f7bef12..3cca9a3d7a 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java @@ -1,6 +1,6 @@ package com.taosdata.example.jdbcTaosdemo.domain; -public class JdbcTaosdemoConfig { +public final class JdbcTaosdemoConfig { //The host to connect to TDengine. Must insert one private String host; @@ -10,37 +10,45 @@ public class JdbcTaosdemoConfig { private String user = "root"; //The password to use when connecting to the server. Default is 'taosdata' private String password = "taosdata"; + //Destination database. Default is 'test' private String dbName = "test"; //keep private int keep = 365 * 20; - // + //days private int days = 30; + //Super table Name. Default is 'meters' private String stbName = "meters"; //Table name prefix. Default is 'd' private String tbPrefix = "d"; - //The number of threads. Default is 10. - private int numberOfThreads = 10; - //The number of tables. Default is 10000. - private int numberOfTable = 10000; - //The number of records per table. Default is 100000 - private int numberOfRecordsPerTable = 100000; + //The number of tables. Default is 10. + private int numberOfTable = 10; + //The number of records per table. Default is 2 + private int numberOfRecordsPerTable = 2; + //The number of records per request. Default is 100 + private int numberOfRecordsPerRequest = 100; + + //The number of threads. Default is 1. + private int numberOfThreads = 1; //Delete data. Default is false - private boolean deleteTable = true; + private boolean deleteTable = false; public static void printHelp() { - System.out.println("Usage: java -jar JDBCConnectorChecker.jar -h host [OPTION...]"); + System.out.println("Usage: java -jar JDBCConnectorChecker.jar [OPTION...]"); + System.out.println("-h host The host to connect to TDengine. you must input one"); System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); System.out.println("-d database Destination database. Default is 'test'"); System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); - System.out.println("-T num_of_threads The number of threads. Default is 10"); - System.out.println("-t num_of_tables The number of tables. Default is 10000"); - System.out.println("-n num_of_records_per_table The number of records per table. Default is 100000"); + System.out.println("-t num_of_tables The number of tables. Default is 10"); + System.out.println("-n num_of_records_per_table The number of records per table. Default is 2"); + System.out.println("-r num_of_records_per_req The number of records per request. Default is 100"); + System.out.println("-T num_of_threads The number of threads. Default is 1"); System.out.println("-D delete table Delete data methods. Default is false"); System.out.println("--help Give this help list"); +// System.out.println("--infinite infinite insert mode"); } /** @@ -49,146 +57,97 @@ public class JdbcTaosdemoConfig { * @param args command line args * @return JdbcTaosdemoConfig */ - public static JdbcTaosdemoConfig build(String[] args) { - - JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(); + public JdbcTaosdemoConfig(String[] args) { for (int i = 0; i < args.length; i++) { if ("-h".equals(args[i]) && i < args.length - 1) { - config.setHost(args[++i]); + host = args[++i]; } if ("-p".equals(args[i]) && i < args.length - 1) { - config.setPort(Integer.parseInt(args[++i])); + port = Integer.parseInt(args[++i]); } if ("-u".equals(args[i]) && i < args.length - 1) { - config.setUser(args[++i]); + user = args[++i]; } if ("-P".equals(args[i]) && i < args.length - 1) { - config.setPassword(args[++i]); + password = args[++i]; } if ("-d".equals(args[i]) && i < args.length - 1) { - config.setDbName(args[++i]); + dbName = args[++i]; } if ("-m".equals(args[i]) && i < args.length - 1) { - config.setTbPrefix(args[++i]); - } - if ("-T".equals(args[i]) && i < args.length - 1) { - config.setNumberOfThreads(Integer.parseInt(args[++i])); + tbPrefix = args[++i]; } if ("-t".equals(args[i]) && i < args.length - 1) { - config.setNumberOfTable(Integer.parseInt(args[++i])); + numberOfTable = Integer.parseInt(args[++i]); } if ("-n".equals(args[i]) && i < args.length - 1) { - config.setNumberOfRecordsPerTable(Integer.parseInt(args[++i])); + numberOfRecordsPerTable = Integer.parseInt(args[++i]); + } + if ("-r".equals(args[i]) && i < args.length - 1) { + numberOfRecordsPerRequest = Integer.parseInt(args[++i]); + } + if ("-T".equals(args[i]) && i < args.length - 1) { + numberOfThreads = Integer.parseInt(args[++i]); } if ("-D".equals(args[i]) && i < args.length - 1) { - config.setDeleteTable(Boolean.parseBoolean(args[++i])); + deleteTable = Boolean.parseBoolean(args[++i]); } } - - return config; - } - - public void setHost(String host) { - this.host = host; } public String getHost() { return host; } - public String getDbName() { - return dbName; - } - - public void setDbName(String dbName) { - this.dbName = dbName; - } - public int getPort() { return port; } - public void setPort(int port) { - this.port = port; - } - public String getUser() { return user; } - public void setUser(String user) { - this.user = user; - } - public String getPassword() { return password; } - public void setPassword(String password) { - this.password = password; - } - - public String getStbName() { - return stbName; - } - - public void setStbName(String stbName) { - this.stbName = stbName; - } - - public String getTbPrefix() { - return tbPrefix; - } - - public void setTbPrefix(String tbPrefix) { - this.tbPrefix = tbPrefix; - } - - public int getNumberOfThreads() { - return numberOfThreads; - } - - public void setNumberOfThreads(int numberOfThreads) { - this.numberOfThreads = numberOfThreads; - } - - public int getNumberOfTable() { - return numberOfTable; - } - - public void setNumberOfTable(int numberOfTable) { - this.numberOfTable = numberOfTable; - } - - public int getNumberOfRecordsPerTable() { - return numberOfRecordsPerTable; - } - - public void setNumberOfRecordsPerTable(int numberOfRecordsPerTable) { - this.numberOfRecordsPerTable = numberOfRecordsPerTable; - } - - public boolean isDeleteTable() { - return deleteTable; - } - - public void setDeleteTable(boolean deleteTable) { - this.deleteTable = deleteTable; + public String getDbName() { + return dbName; } public int getKeep() { return keep; } - public void setKeep(int keep) { - this.keep = keep; - } - public int getDays() { return days; } - public void setDays(int days) { - this.days = days; + public String getStbName() { + return stbName; + } + + public String getTbPrefix() { + return tbPrefix; + } + + public int getNumberOfTable() { + return numberOfTable; + } + + public int getNumberOfRecordsPerTable() { + return numberOfRecordsPerTable; + } + + public int getNumberOfThreads() { + return numberOfThreads; + } + + public boolean isDeleteTable() { + return deleteTable; + } + + public int getNumberOfRecordsPerRequest() { + return numberOfRecordsPerRequest; } } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java index 2b1a08ab40..1da2c8647e 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java @@ -1,7 +1,8 @@ package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; +import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; import org.apache.log4j.Logger; import java.sql.Connection; @@ -24,14 +25,11 @@ public class CreateTableTask implements Runnable { @Override public void run() { try { - Connection connection = JdbcTaosdemo.getConnection(config); + Connection connection = ConnectionFactory.build(config); for (int i = startIndex; i < startIndex + tableNumber; i++) { Statement statement = connection.createStatement(); - String sql = JdbcTaosdemo.createTableSql(i + 1, config); -// long start = System.currentTimeMillis(); - boolean execute = statement.execute(sql); -// long end = System.currentTimeMillis(); -// printSql(sql, execute, (end - start)); + String sql = SqlSpeller.createTableSQL(i + 1, config.getDbName(), config.getStbName()); + statement.execute(sql); statement.close(); logger.info(">>> " + sql); } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java index d477ceeaed..4f60c25646 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java @@ -1,7 +1,8 @@ package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; +import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; import org.apache.log4j.Logger; import java.sql.Connection; @@ -28,13 +29,11 @@ public class InsertTableDatetimeTask implements Runnable { @Override public void run() { try { - Connection connection = JdbcTaosdemo.getConnection(config); - int valueCnt = 100; - for (long ts = startDatetime; ts < finishedDatetime; ts+= valueCnt) { + Connection connection = ConnectionFactory.build(config); + int valuesCount = config.getNumberOfRecordsPerRequest(); + for (long ts = startDatetime; ts < finishedDatetime; ts += valuesCount) { for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) { -// String sql = JdbcTaosdemo.insertSql(i + 1, ts, config); - - String sql = JdbcTaosdemo.batchInsertSql(i + 1, ts, valueCnt, config); + String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, valuesCount); Statement statement = connection.createStatement(); statement.execute(sql); statement.close(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java index 9ae50d830e..d6d6ebbff1 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java @@ -1,15 +1,19 @@ package com.taosdata.example.jdbcTaosdemo.task; -import com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo; import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; +import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; +import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; +import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil; import org.apache.log4j.Logger; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.concurrent.atomic.AtomicLong; public class InsertTableTask implements Runnable { private static final Logger logger = Logger.getLogger(InsertTableTask.class); + private static AtomicLong beginTimestamp = new AtomicLong(TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000")); private final JdbcTaosdemoConfig config; private final int startIndex; @@ -26,10 +30,13 @@ public class InsertTableTask implements Runnable { @Override public void run() { try { - Connection connection = JdbcTaosdemo.getConnection(config); - for (int i = startIndex; i < startIndex + tableNumber; i++) { - for (int j = 0; j < recordsNumber; j++) { - String sql = JdbcTaosdemo.insertSql(i + 1, config); + Connection connection = ConnectionFactory.build(config); + // iterate insert + for (int j = 0; j < recordsNumber; j++) { + long ts = beginTimestamp.getAndIncrement(); + // insert data into echo table + for (int i = startIndex; i < startIndex + tableNumber; i++) { + String sql = SqlSpeller.insertOneRowSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts); Statement statement = connection.createStatement(); statement.execute(sql); statement.close(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java new file mode 100644 index 0000000000..52691f4de7 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java @@ -0,0 +1,32 @@ +package com.taosdata.example.jdbcTaosdemo.utils; + +import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; +import com.taosdata.jdbc.TSDBDriver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class ConnectionFactory { + + public static Connection build(JdbcTaosdemoConfig config) throws SQLException { + return build(config.getHost(), config.getPort(), config.getDbName(), config.getUser(), config.getPassword()); + } + + public static Connection build(String host, int port, String dbName) throws SQLException { + return build(host, port, dbName, "root", "taosdata"); + } + + private static Connection build(String host, int port, String dbName, String user, String password) throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, user); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, password); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + return DriverManager.getConnection("jdbc:TAOS://" + host + ":" + port + "/" + dbName + "", properties); + } + + +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java new file mode 100644 index 0000000000..ecab1a0359 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java @@ -0,0 +1,58 @@ +package com.taosdata.example.jdbcTaosdemo.utils; + +import java.util.Random; + +public class SqlSpeller { + private static final Random random = new Random(System.currentTimeMillis()); + private static final String[] locations = { + "Beijing", "Shanghai", "Guangzhou", "Shenzhen", + "HangZhou", "Tianjin", "Wuhan", "Changsha", "Nanjing", "Xian" + }; + + public static String createDatabaseSQL(String dbName, int keep, int days) { + return "create database if not exists " + dbName + " keep " + keep + " days " + days; + } + + public static String dropDatabaseSQL(String dbName) { + return "drop database if exists " + dbName; + } + + public static String useDatabaseSQL(String dbName) { + return "use " + dbName; + } + + public static String createSuperTableSQL(String superTableName) { + return "create table if not exists " + superTableName + "(ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int)"; + } + + public static String dropSuperTableSQL(String dbName, String superTableName) { + return "drop table if exists " + dbName + "." + superTableName; + } + + public static String createTableSQL(int tableIndex, String dbName, String superTableName) { + String location = locations[random.nextInt(locations.length)]; + return "create table d" + tableIndex + " using " + dbName + "." + superTableName + " tags('" + location + "'," + tableIndex + ")"; + } + + public static String insertOneRowSQL(String dbName, String tbPrefix, int tableIndex, long ts) { + float current = 10 + random.nextFloat(); + int voltage = 200 + random.nextInt(20); + float phase = random.nextFloat(); + String sql = "insert into " + dbName + "." + tbPrefix + "" + tableIndex + " " + "values(" + ts + ", " + current + ", " + voltage + ", " + phase + ")"; + return sql; + } + + public static String insertBatchSizeRowsSQL(String dbName, String tbPrefix, int tbIndex, long ts, int valuesCount) { + float current = 10 + random.nextFloat(); + int voltage = 200 + random.nextInt(20); + float phase = random.nextFloat(); + StringBuilder sb = new StringBuilder(); + sb.append("insert into " + dbName + "." + tbPrefix + "" + tbIndex + " " + "values"); + for (int i = 0; i < valuesCount; i++) { + sb.append("(" + (ts + i) + ", " + current + ", " + voltage + ", " + phase + ") "); + } + return sb.toString(); + } + + +} \ No newline at end of file diff --git a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties index 0ed95d04d3..b445e5f52e 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties +++ b/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties @@ -1,11 +1,10 @@ ### 设置### -#log4j.rootLogger=debug,stdout,DebugLog,ErrorLog -log4j.rootLogger=debug,DebugLog,ErrorLog +log4j.rootLogger=debug,stdout,DebugLog,ErrorLog ### 输出信息到控制抬 ### -#log4j.appender.stdout=org.apache.log4j.ConsoleAppender -#log4j.appender.stdout.Target=System.out -#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -#log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n ### 输出DEBUG 级别以上的日志到=logs/error.log ### log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log From 2cd68ffe6946a757d56d52a709ae7689b1f93752 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 15:40:17 +0800 Subject: [PATCH 29/63] TD-1574 compile in windows --- deps/zlib-1.2.11/CMakeLists.txt | 13 ++++++++----- src/os/src/detail/osDir.c | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/deps/zlib-1.2.11/CMakeLists.txt b/deps/zlib-1.2.11/CMakeLists.txt index 5502070819..f83aa70085 100644 --- a/deps/zlib-1.2.11/CMakeLists.txt +++ b/deps/zlib-1.2.11/CMakeLists.txt @@ -1,8 +1,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(z ${SRC}) -ENDIF () \ No newline at end of file +IF (TD_WINDOWS) + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /WX-") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /WX-") +ENDIF() + +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) +ADD_LIBRARY(z ${SRC}) diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index bdb840ce01..30fd05ffc7 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -130,7 +130,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { } while (!feof(srcFp)) { - len = (uLong)fread(data, 1, COMPRESS_STEP_SIZE, srcFp); + len = (int32_t)fread(data, 1, COMPRESS_STEP_SIZE, srcFp); gzwrite(dstFp, data, len); } From 1a4746d4255e650a678fd4e2078c18f2571fcc60 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 25 Sep 2020 17:12:15 +0800 Subject: [PATCH 30/63] update jdbc taosdemo --- .../example/jdbcTaosdemo/JdbcTaosdemo.java | 44 ++++++++++++++++--- .../jdbcTaosdemo/utils/SqlSpeller.java | 24 ++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java index 5a6acb0291..c30d85a084 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java @@ -25,7 +25,6 @@ public class JdbcTaosdemo { } public static void main(String[] args) { - JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); @@ -52,7 +51,15 @@ public class JdbcTaosdemo { taosdemo.insertInfinite(); } else { taosdemo.insertMultiThreads(); - taosdemo.countFromSuperTable(); + // single table select + taosdemo.selectFromTableLimit(); + taosdemo.selectCountFromTable(); + taosdemo.selectAvgMinMaxFromTable(); + // super table select + taosdemo.selectFromSuperTableLimit(); + taosdemo.selectCountFromSuperTable(); + taosdemo.selectAvgMinMaxFromSuperTable(); + // drop super table if (config.isDeleteTable()) taosdemo.dropSuperTable(); taosdemo.close(); @@ -174,8 +181,33 @@ public class JdbcTaosdemo { } } - private void countFromSuperTable() { - String sql = "select count(*) from " + config.getDbName() + "." + config.getStbName(); + private void selectFromTableLimit() { + String sql = SqlSpeller.selectFromTableLimitSQL(config.getDbName(), config.getTbPrefix(), 1, 10, 0); + executeQuery(sql); + } + + private void selectCountFromTable() { + String sql = SqlSpeller.selectCountFromTableSQL(config.getDbName(), config.getTbPrefix(), 1); + executeQuery(sql); + } + + private void selectAvgMinMaxFromTable() { + String sql = SqlSpeller.selectAvgMinMaxFromTableSQL("current", config.getDbName(), config.getTbPrefix(), 1); + executeQuery(sql); + } + + private void selectFromSuperTableLimit() { + String sql = SqlSpeller.selectFromSuperTableLimitSQL(config.getDbName(), config.getStbName(), 10, 0); + executeQuery(sql); + } + + private void selectCountFromSuperTable() { + String sql = SqlSpeller.selectCountFromSuperTableSQL(config.getDbName(), config.getStbName()); + executeQuery(sql); + } + + private void selectAvgMinMaxFromSuperTable() { + String sql = SqlSpeller.selectAvgMinMaxFromSuperTableSQL("current", config.getDbName(), config.getStbName()); executeQuery(sql); } @@ -215,7 +247,7 @@ public class JdbcTaosdemo { } private static void printSql(String sql, boolean succeed, long cost) { - logger.info("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); } private void executeQuery(String sql) { @@ -240,7 +272,7 @@ public class JdbcTaosdemo { String value = resultSet.getString(i); sb.append(columnLabel + ": " + value + "\t"); } - logger.info(sb.toString()); + System.out.println(sb.toString()); } } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java index ecab1a0359..7af97f3b1b 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java @@ -54,5 +54,29 @@ public class SqlSpeller { return sb.toString(); } + public static String selectFromTableLimitSQL(String dbName, String tbPrefix, int tbIndex, int limit, int offset) { + return "select * from " + dbName + "." + tbPrefix + "" + tbIndex + " limit " + limit + " offset " + offset; + } + + public static String selectCountFromTableSQL(String dbName, String tbPrefix, int tbIndex) { + return "select count(*) from " + dbName + "." + tbPrefix + "" + tbIndex; + } + + public static String selectAvgMinMaxFromTableSQL(String field, String dbName, String tbPrefix, int tbIndex) { + return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + tbPrefix + "" + tbIndex; + } + + public static String selectFromSuperTableLimitSQL(String dbName, String stbName, int limit, int offset) { + return "select * from " + dbName + "." + stbName + " limit " + limit + " offset " + offset; + } + + public static String selectCountFromSuperTableSQL(String dbName, String stableName) { + return "select count(*) from " + dbName + "." + stableName; + } + + public static String selectAvgMinMaxFromSuperTableSQL(String field, String dbName, String stbName) { + return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + stbName + ""; + } + } \ No newline at end of file From 7a840d7f2ac73a253f49ae55057c0c1b61034118 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 10:14:20 +0000 Subject: [PATCH 31/63] change maxVgroupsPerDb to 16 --- src/inc/taosdef.h | 2 ++ src/mnode/src/mnodeVgroup.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index f9d4b3d7bf..228aba2a5e 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -294,6 +294,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 64 #define TSDB_MAX_VNODES 2048 +#define TSDB_MIN_VNODES_PER_DB 2 +#define TSDB_MAX_VNODES_PER_DB 16 #define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_MGMT 1 diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9345bdae2c..7dbf605405 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -434,7 +434,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi int maxVgroupsPerDb = tsMaxVgroupsPerDb; if (maxVgroupsPerDb <= 0) { maxVgroupsPerDb = mnodeGetOnlinDnodesCpuCoreNum(); - maxVgroupsPerDb = MAX(maxVgroupsPerDb, 2); + maxVgroupsPerDb = MAX(maxVgroupsPerDb, TSDB_MIN_VNODES_PER_DB); + maxVgroupsPerDb = MIN(maxVgroupsPerDb, TSDB_MAX_VNODES_PER_DB); } int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES; From f2dafd9c7232132848a1cddb6e260e63c35a6907 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 14:42:56 +0000 Subject: [PATCH 32/63] TD-1473 --- src/balance/src/balance.c | 2 + src/mnode/inc/mnodeDef.h | 3 +- src/mnode/inc/mnodeDnode.h | 22 +++++ src/mnode/src/mnodeDnode.c | 134 +++++++++++++++++++-------- tests/script/jenkins/basic.txt | 1 + tests/script/unique/dnode/reason.sim | 132 ++++++++++++++++++++++++++ 6 files changed, 253 insertions(+), 41 deletions(-) create mode 100644 tests/script/unique/dnode/reason.sim diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 2b1888042c..3b9af741c3 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -389,6 +389,7 @@ void balanceReset() { pDnode->lastAccess = 0; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; + pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } mnodeDecDnodeRef(pDnode); @@ -551,6 +552,7 @@ static void balanceCheckDnodeAccess() { if (tsAccessSquence - pDnode->lastAccess > 3) { if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { pDnode->status = TAOS_DN_STATUS_OFFLINE; + pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; mInfo("dnode:%d, set to offline state", pDnode->dnodeId); balanceSetVgroupOffline(pDnode); } diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 682986b29f..8a2947dd18 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -69,7 +69,8 @@ typedef struct SDnodeObj { int16_t cpuAvgUsage; // calc from sys.cpu int16_t memoryAvgUsage; // calc from sys.mem int16_t bandwidthUsage; // calc from sys.band - int8_t reserved2[2]; + int8_t offlineReason; + int8_t reserved2[1]; } SDnodeObj; typedef struct SMnodeObj { diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index 13b6ec4411..b6ddb7a9bf 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -33,6 +33,28 @@ typedef enum { TAOS_DN_ALTERNATIVE_ROLE_VNODE } EDnodeAlternativeRole; +typedef enum EDnodeOfflineReason { + TAOS_DN_OFF_ONLINE = 0, + TAOS_DN_OFF_STATUS_MSG_TIMEOUT, + TAOS_DN_OFF_STATUS_NOT_RECEIVED, + TAOS_DN_OFF_RESET_BY_MNODE, + TAOS_DN_OFF_VERSION_NOT_MATCH, + TAOS_DN_OFF_DNODE_ID_NOT_MATCH, + TAOS_DN_OFF_CLUSTER_ID_NOT_MATCH, + TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH, + TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH, + TAOS_DN_OFF_MN_EQUAL_VN_NOT_MATCH, + TAOS_DN_OFF_OFFLINE_THRESHOLD_NOT_MATCH, + TAOS_DN_OFF_STATUS_INTERVAL_NOT_MATCH, + TAOS_DN_OFF_MAX_TAB_PER_VN_NOT_MATCH, + TAOS_DN_OFF_MAX_VG_PER_DB_NOT_MATCH, + TAOS_DN_OFF_ARBITRATOR_NOT_MATCH, + TAOS_DN_OFF_TIME_ZONE_NOT_MATCH, + TAOS_DN_OFF_LOCALE_NOT_MATCH, + TAOS_DN_OFF_CHARSET_NOT_MATCH, + TAOS_DN_OFF_OTHERS +} EDnodeOfflineReason; + int32_t mnodeInitDnodes(); void mnodeCleanupDnodes(); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index ac8730b0cc..61c1d4113f 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -60,6 +60,28 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); +static char* offlineReason[] = { + "", + "status msg timeout", + "status not received", + "status reset by mnode", + "version not match", + "dnodeId not match", + "clusterId not match", + "numOfMnodes not match", + "balance not match", + "mnEqualVn not match", + "offThreshold not match", + "interval not match", + "maxTabPerVn not match", + "maxVgPerDb not match", + "arbitrator not match", + "timezone not match", + "locale not match", + "charset not match", + "unknown", +}; + static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) { taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; @@ -70,6 +92,7 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->lastAccess = tsAccessSquence; + pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); @@ -334,74 +357,85 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { mInfo("cfg dnode rsp is received"); } -static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { +static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { if (clusterCfg->numOfMnodes != htonl(tsNumOfMnodes)) { mError("\"numOfMnodes\"[%d - %d] cfg parameters inconsistent", clusterCfg->numOfMnodes, htonl(tsNumOfMnodes)); - return false; - } - if (clusterCfg->enableBalance != htonl(tsEnableBalance)) { + return TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH; + } + if (clusterCfg->enableBalance != htonl(tsEnableBalance)) { mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, htonl(tsEnableBalance)); - return false; + return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; } if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) { - mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, htonl(tsMnodeEqualVnodeNum)); - return false; + mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, + htonl(tsMnodeEqualVnodeNum)); + return TAOS_DN_OFF_MN_EQUAL_VN_NOT_MATCH; } - if (clusterCfg->offlineThreshold != htonl(tsOfflineThreshold)) { - mError("\"offlineThreshold\"[%d - %d] cfg parameters inconsistent", clusterCfg->offlineThreshold, htonl(tsOfflineThreshold)); - return false; + if (clusterCfg->offlineThreshold != htonl(tsOfflineThreshold)) { + mError("\"offlineThreshold\"[%d - %d] cfg parameters inconsistent", clusterCfg->offlineThreshold, + htonl(tsOfflineThreshold)); + return TAOS_DN_OFF_OFFLINE_THRESHOLD_NOT_MATCH; } - if (clusterCfg->statusInterval != htonl(tsStatusInterval)) { - mError("\"statusInterval\"[%d - %d] cfg parameters inconsistent", clusterCfg->statusInterval, htonl(tsStatusInterval)); - return false; + if (clusterCfg->statusInterval != htonl(tsStatusInterval)) { + mError("\"statusInterval\"[%d - %d] cfg parameters inconsistent", clusterCfg->statusInterval, + htonl(tsStatusInterval)); + return TAOS_DN_OFF_STATUS_INTERVAL_NOT_MATCH; } - if (clusterCfg->maxtablesPerVnode != htonl(tsMaxTablePerVnode)) { - mError("\"maxTablesPerVnode\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxtablesPerVnode, htonl(tsMaxTablePerVnode)); - return false; + if (clusterCfg->maxtablesPerVnode != htonl(tsMaxTablePerVnode)) { + mError("\"maxTablesPerVnode\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxtablesPerVnode, + htonl(tsMaxTablePerVnode)); + return TAOS_DN_OFF_MAX_TAB_PER_VN_NOT_MATCH; } - if (clusterCfg->maxVgroupsPerDb != htonl(tsMaxVgroupsPerDb)) { - mError("\"maxVgroupsPerDb\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxVgroupsPerDb, htonl(tsMaxVgroupsPerDb)); - return false; + if (clusterCfg->maxVgroupsPerDb != htonl(tsMaxVgroupsPerDb)) { + mError("\"maxVgroupsPerDb\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxVgroupsPerDb, + htonl(tsMaxVgroupsPerDb)); + return TAOS_DN_OFF_MAX_VG_PER_DB_NOT_MATCH; } if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) { mError("\"arbitrator\"[%s - %s] cfg parameters inconsistent", clusterCfg->arbitrator, tsArbitrator); - return false; + return TAOS_DN_OFF_ARBITRATOR_NOT_MATCH; } int64_t checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; + char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { - mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64"] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone, clusterCfg->checkTime, checkTime); - return false; + if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && + (checkTime != clusterCfg->checkTime)) { + mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone, + tsTimezone, clusterCfg->checkTime, checkTime); + return TAOS_DN_OFF_TIME_ZONE_NOT_MATCH; } if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) { mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale); - return false; + return TAOS_DN_OFF_LOCALE_NOT_MATCH; } if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) { mError("\"charset\"[%s - %s] cfg parameters inconsistent.", clusterCfg->charset, tsCharset); - return false; + return TAOS_DN_OFF_CHARSET_NOT_MATCH; } - - return true; + + return 0; } static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { + SDnodeObj *pDnode = NULL; SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->lastReboot = htonl(pStatus->lastReboot); pStatus->numOfCores = htons(pStatus->numOfCores); - + uint32_t version = htonl(pStatus->version); if (version != tsVersion) { - mError("status msg version:%d not equal with mnode:%d", version, tsVersion); + pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp); + if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) { + pDnode->offlineReason = TAOS_DN_OFF_VERSION_NOT_MATCH; + } + mError("dnode:%d, status msg version:%d not equal with cluster:%d", pStatus->dnodeId, version, tsVersion); return TSDB_CODE_MND_INVALID_MSG_VERSION; } - SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp); if (pDnode == NULL) { @@ -411,7 +445,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } else { pDnode = mnodeGetDnode(pStatus->dnodeId); if (pDnode == NULL) { - mError("dnode id:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); + pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp); + if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) { + pDnode->offlineReason = TAOS_DN_OFF_DNODE_ID_NOT_MATCH; + } + mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); return TSDB_CODE_MND_DNODE_NOT_EXIST; } } @@ -426,6 +464,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { mDebug("dnode:%d %s, first access, set clusterId %s", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId()); } else { if (strncmp(pStatus->clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN - 1) != 0) { + if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) { + pDnode->offlineReason = TAOS_DN_OFF_CLUSTER_ID_NOT_MATCH; + } mError("dnode:%d, input clusterId %s not match with exist %s", pDnode->dnodeId, pStatus->clusterId, mnodeGetClusterId()); return TSDB_CODE_MND_INVALID_CLUSTER_ID; @@ -469,16 +510,19 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { // Verify whether the cluster parameters are consistent when status change from offline to ready - bool ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg)); - if (false == ret) { + int32_t ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg)); + if (0 != ret) { + pDnode->offlineReason = ret; mnodeDecDnodeRef(pDnode); rpcFreeCont(pRsp); - mError("dnode:%d, %s cluster cfg parameters inconsistent", pDnode->dnodeId, pStatus->dnodeEp); + mError("dnode:%d, %s cluster cfg parameters inconsistent, reason:%s", pDnode->dnodeId, pStatus->dnodeEp, + offlineReason[ret]); return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; } - + mDebug("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TAOS_DN_STATUS_READY; + pDnode->offlineReason = TAOS_DN_OFF_ONLINE; balanceSyncNotify(); balanceAsyncNotify(); } @@ -529,6 +573,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); pDnode->createdTime = taosGetTimestampMs(); pDnode->status = TAOS_DN_STATUS_OFFLINE; + pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); @@ -654,13 +699,13 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 6 + VARSTR_HEADER_SIZE; + pShow->bytes[cols] = 5 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -672,6 +717,12 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "offline reason"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -731,8 +782,11 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo *(int64_t *)pWrite = pDnode->createdTime; cols++; - - numOfRows++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + cols++; + + numOfRows++; mnodeDecDnodeRef(pDnode); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9e42adfea9..977ef452ab 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -278,6 +278,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim diff --git a/tests/script/unique/dnode/reason.sim b/tests/script/unique/dnode/reason.sim new file mode 100644 index 0000000000..ad61a81b97 --- /dev/null +++ b/tests/script/unique/dnode/reason.sim @@ -0,0 +1,132 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect +sql create dnode $hostname2 + +sql show dnodes +print dnode1 off: $data7_1 +print dnode2 off: $data7_2 +if $data7_2 != @status not received@ then + return -1 +endi + +print ========== step2 +system sh/exec.sh -n dnode2 -s start +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode2 off: $data7_2 + +print ========== step3 +system sh/exec.sh -n dnode2 -s stop +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode2 off: $data7_2 +if $data7_2 != @status msg timeout@ then + return -1 +endi + +print ========== step4 +sql drop dnode $hostname2 +sleep 5000 +sql show dnodes +if $rows != 1 then + return -1 +endi + +print ========== step5 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname2 +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode2 off: $data7_3 +if $data7_3 != @dnodeId not match@ then + return -1 +endi + +print ========== step6 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3 +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname4 + +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode4 off: $data7_4 +if $data7_4 != @mnEqualVn not match@ then + return -1 +endi + +print ========== step7 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode5 -c statusInterval -v 3 +system sh/exec.sh -n dnode5 -s start +sql create dnode $hostname5 + +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode5 off: $data7_5 +if $data7_5 != @interval not match@ then + return -1 +endi + +print ========== step8 +system sh/deploy.sh -n dnode6 -i 6 +system sh/cfg.sh -n dnode6 -c balance -v 0 +system sh/exec.sh -n dnode6 -s start +sql create dnode $hostname6 + +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode6 off: $data7_6 +if $data7_6 != @balance not match@ then + return -1 +endi + +print ========== step9 +system sh/deploy.sh -n dnode7 -i 7 +system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000 +system sh/exec.sh -n dnode7 -s start +sql create dnode $hostname7 + +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode7 off: $data7_7 +if $data7_7 != @maxTabPerVn not match@ then + return -1 +endi + +print ========== step10 +system sh/deploy.sh -n dnode8 -i 8 +system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3 +system sh/exec.sh -n dnode8 -s start +sql create dnode $hostname8 + +sleep 3000 +sql show dnodes +print dnode1 off: $data7_1 +print dnode8 off: $data7_8 +if $data7_8 != @maxVgPerDb not match@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file From 786f3256f21819b8d40afc9ea2b715b02fe7d566 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 14:45:21 +0000 Subject: [PATCH 33/63] minor changes --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 977ef452ab..000d36c178 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -303,8 +303,8 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim +#./test.sh -f unique/mnode/mgmt25.sim +#./test.sh -f unique/mnode/mgmt26.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim From 4736c6621dc4ed4f844949736e598239ccc20c41 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Sep 2020 16:26:16 +0000 Subject: [PATCH 34/63] TD-1418 --- src/sync/src/syncMain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 3f57a8b5cd..9b73558ba8 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -167,6 +167,8 @@ void *syncStart(const SSyncInfo *pInfo) { } } + syncAddNodeRef(pNode); + if (pNode->selfIndex < 0) { sInfo("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; @@ -194,7 +196,6 @@ void *syncStart(const SSyncInfo *pInfo) { } syncAddArbitrator(pNode); - syncAddNodeRef(pNode); taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); if (pNode->notifyRole) { From 9dbc9b6fd8d313e8216bb14cb51aa850942b5cec Mon Sep 17 00:00:00 2001 From: zyyang Date: Sun, 27 Sep 2020 09:14:43 +0800 Subject: [PATCH 35/63] [TD-1608]: fix bugs in jdbc parse url --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 13be9d538d..97d93fb0a1 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -179,7 +179,7 @@ public class TSDBDriver implements java.sql.Driver { } //load taos.cfg start - if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null){ + if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) { File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0]; List endpoints = loadConfigEndpoints(cfgFile); @@ -244,7 +244,7 @@ public class TSDBDriver implements java.sql.Driver { } public boolean acceptsURL(String url) throws SQLException { - return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX); + return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -291,7 +291,6 @@ public class TSDBDriver implements java.sql.Driver { return null; } -// String lowerUrl = url.toLowerCase(); if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) { return null; } From a1912ef83fe0fd62fc8e08e37966370e24faed70 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 05:25:15 +0000 Subject: [PATCH 36/63] TD-1415 --- tests/test/c/CMakeLists.txt | 3 + tests/test/c/invalidTableId.c | 220 ++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 tests/test/c/invalidTableId.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index befaec60a6..26aa20e647 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -42,5 +42,8 @@ IF (TD_LINUX) #add_executable(cacheTest cacheTest.c) #target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + + #add_executable(invalidTableId invalidTableId.c) + #target_link_libraries(invalidTableId taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/invalidTableId.c b/tests/test/c/invalidTableId.c new file mode 100644 index 0000000000..581387a4ee --- /dev/null +++ b/tests/test/c/invalidTableId.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "tglobal.h" +#include "hash.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +#define MAX_DB_NUM 100 +void * con; +char dbNames[MAX_DB_NUM][48]; +int32_t dbNum = 0; +void parseArgument(int argc, char *argv[]); +void connDb(); +void getDbNames(); +void printDbNames(); +void queryTables(char *dbName); +void checkTables(char *dbName); + +int main(int argc, char *argv[]) { + parseArgument(argc, argv); + taos_init(); + connDb(); + getDbNames(); + printDbNames(); + for (int dbIndex = 0; dbIndex < dbNum; ++dbIndex) { + queryTables((char*)(dbNames[dbIndex])); + checkTables((char*)(dbNames[dbIndex])); + } + + pPrint("all %d database is checked", dbNum); +} + +void connDb() { + con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(0); + } +} + +void getDbNames() { + if (dbNum != 0) return; + + char * qstr = "show databases"; + TAOS_RES *result = taos_query(con, qstr); + int32_t code = taos_errno(result); + if (result == NULL || code != 0) { + pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code)); + exit(0); + } + + TAOS_ROW row; + int num_fields = taos_num_fields(result); + if (num_fields <= 0) return; + while ((row = taos_fetch_row(result))) { + char * dbName = (char*)dbNames[dbNum]; + int32_t *length = taos_fetch_lengths(result); + int len = length[0]; + memcpy(dbName, (char *)row[0], len); + dbName[len] = 0; + dbNum++; + } + + taos_free_result(result); +} + +void printDbNames() { + for (int dbIndex = 0; dbIndex < dbNum; ++dbIndex) { + pPrint("db:%d %s", dbIndex, dbNames[dbIndex]); + } +} + +void queryTables(char *dbName) { + char qstr[1024]; + char fileName[1024]; + char ts[35] = {0}; + int32_t precision = 1000; + + sprintf(qstr, "show %s.tables", dbName); + sprintf(fileName, "%s_tables.txt", dbName); + + TAOS_RES *result = taos_query(con, qstr); + int32_t code = taos_errno(result); + if (result == NULL || code != 0) { + pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code)); + exit(0); + } + + FILE *fp = fopen(fileName, "w"); + if (!fp) return; + + TAOS_ROW row; + int32_t rows = 0; + while ((row = taos_fetch_row(result))) { + char tableName[256] = {0}; + int32_t *length = taos_fetch_lengths(result); + int len = length[0]; + memcpy(tableName, (char *)row[0], len); + tableName[len] = 0; + + int64_t t = *((int64_t *)row[1]); + time_t tt = t / 1000; + struct tm *ptm = localtime(&tt); + int32_t tl = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + snprintf(ts + tl, 5, ".%03ld", t % precision); + + // fprintf(fp, "%s %s\n", tableName, ts); + fprintf(fp, "%s.%s\n", dbName, tableName); + rows++; + } + + taos_free_result(result); + fclose(fp); + pPrint("db:%s has %d tables, write to %s", dbName, rows, fileName); +} + +void checkTables(char *dbName) { + char qstr[1024]; + char fileName1[1024]; + char fileName2[1024]; + + sprintf(qstr, "show %s.tables", dbName); + sprintf(fileName1, "%s_tables.txt", dbName); + sprintf(fileName2, "%s_count.txt", dbName); + + FILE *fp1 = fopen(fileName1, "r"); + if (!fp1) return; + + FILE *fp2 = fopen(fileName2, "w"); + if (!fp2) return; + + int32_t successRows = 0; + int32_t failedRows = 0; + char tbName[256]; + while (!feof(fp1)) { + int size = fscanf(fp1, "%s", tbName); + if (size <= 0) { + break; + } + + sprintf(qstr, "select count(*) from %s", tbName); + TAOS_RES *result = taos_query(con, qstr); + int32_t code = taos_errno(result); + if (result == NULL || code != 0) { + pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code)); + fprintf(fp2, "%s failed to exec sql:%s, code:0x%x reason:%s", tbName, qstr, code & 0XFFFF, tstrerror(code)); + taos_free_result(result); + failedRows++; + continue; + } + + TAOS_ROW row; + int64_t count = 0; + while ((row = taos_fetch_row(result))) { + count = *((int64_t *)row[0]); + } + fprintf(fp2, "%s %" PRId64 "\n", tbName, count); + + successRows++; + if (successRows % 1000 == 0) { + pPrint("query %d tables", successRows); + } + taos_free_result(result); + } + + fclose(fp1); + fclose(fp2); + pPrint("db:%s query tables, success:%d failed:%d write to %s", dbName, successRows, failedRows, fileName2); +} + +void printHelp() { + char indent[10] = " "; + printf("Used to checkTables\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be checked, default is ", "all"); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbNames[0], argv[++i]); + dbNum++; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else { + } + } + + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s start to checkTables %s", GREEN, NC); +} From 1f4d0b0c130c49511155636f32b0d70d23e6e55f Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sun, 27 Sep 2020 13:28:37 +0800 Subject: [PATCH 37/63] [TD-1465] --- cmake/install.inc | 42 ++++++++++++++++------------------- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 1 + packaging/tools/makeclient.sh | 7 ++++-- packaging/tools/makepkg.sh | 9 +++++--- 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 0531d40048..f5e01e2f1d 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -10,31 +10,27 @@ ELSEIF (TD_WINDOWS) SET(CMAKE_INSTALL_PREFIX C:/TDengine) ENDIF () - IF (NOT TD_GODLL) - #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) - #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) - INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) - IF (TD_POWER) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) - ELSE () - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) - ENDIF () + IF (TD_POWER) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) + ELSE () + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosdemo.exe DESTINATION .) + ENDIF () - #INSTALL(TARGETS taos RUNTIME DESTINATION driver) - #INSTALL(TARGETS shell RUNTIME DESTINATION .) - IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.0-dist.jar DESTINATION connector/jdbc) - ENDIF () - ELSE () - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver) + #INSTALL(TARGETS taos RUNTIME DESTINATION driver) + #INSTALL(TARGETS shell RUNTIME DESTINATION .) + IF (TD_MVN_INSTALLED) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.0-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index f05d7f3414..450c6a8f55 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -57,6 +57,7 @@ cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_pat cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector +cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index a8303c594a..4e40263dc4 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -64,6 +64,7 @@ cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/incl cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector +cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 665fb2845c..d69a8e6007 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -97,6 +97,8 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/python ${install_dir}/examples cp -r ${examples_dir}/R ${install_dir}/examples cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver mkdir -p ${install_dir}/driver @@ -111,8 +113,9 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector fi cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + cp -r ${connector_dir}/python ${install_dir}/connector/ + cp -r ${connector_dir}/go ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note # cp ${script_dir}/release_note ${install_dir} diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 796f39ccc0..3958cff53b 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -113,6 +113,8 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/python ${install_dir}/examples cp -r ${examples_dir}/R ${install_dir}/examples cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver mkdir -p ${install_dir}/driver @@ -122,10 +124,11 @@ cp ${lib_files} ${install_dir}/driver connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + cp -r ${connector_dir}/python ${install_dir}/connector/ + cp -r ${connector_dir}/go ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note # cp ${script_dir}/release_note ${install_dir} From 456f65575af261ca8ec427e91ed8d4f756676122 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 05:55:25 +0000 Subject: [PATCH 38/63] TD-1571 --- src/mnode/src/mnodeInt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index 91c8dcb6e5..fb1b8741a9 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -39,6 +39,11 @@ void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) { } int32_t mnodeInitMsg(SMnodeMsg *pMsg) { + if (pMsg->pUser != NULL) { + mDebug("app:%p:%p, user info already inited", pMsg->rpcMsg.ahandle, pMsg); + return TSDB_CODE_SUCCESS; + } + pMsg->pUser = mnodeGetUserFromConn(pMsg->rpcMsg.handle); if (pMsg->pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; From f82c9afcbcde398aaabaea6f7b05b24073f326a8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 06:41:22 +0000 Subject: [PATCH 39/63] hotfix for invalid table id --- src/mnode/src/mnodeInt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index 91c8dcb6e5..fb1b8741a9 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -39,6 +39,11 @@ void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) { } int32_t mnodeInitMsg(SMnodeMsg *pMsg) { + if (pMsg->pUser != NULL) { + mDebug("app:%p:%p, user info already inited", pMsg->rpcMsg.ahandle, pMsg); + return TSDB_CODE_SUCCESS; + } + pMsg->pUser = mnodeGetUserFromConn(pMsg->rpcMsg.handle); if (pMsg->pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; From 9c7aecc3d0ea8f4e9637b09835911712ba78299b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 27 Sep 2020 15:00:35 +0800 Subject: [PATCH 40/63] [td-1619] --- src/tsdb/src/tsdbRead.c | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a3bc0de272..6c1602a857 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -697,22 +697,41 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); if (pCheckInfo->pDataCols == NULL) { - tsdbError("%p failed to malloc buf, %p", pQueryHandle, pQueryHandle->qinfo); + tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return terrno; + goto _error; } } STSchema* pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); - tdInitDataCols(pCheckInfo->pDataCols, pSchema); - tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); - tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); + int32_t code = tdInitDataCols(pCheckInfo->pDataCols, pSchema); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _error; + } + + code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %p", pQueryHandle, pQueryHandle->qinfo); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _error; + } + + code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %p", pQueryHandle, pQueryHandle->qinfo); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _error; + } int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); if (ret != TSDB_CODE_SUCCESS) { - return terrno; + int32_t c = terrno; + assert(c != TSDB_CODE_SUCCESS); + goto _error; } SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; @@ -729,10 +748,16 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p int64_t elapsedTime = (taosGetTimestampUs() - st); pQueryHandle->cost.blockLoadTime += elapsedTime; - tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p", + tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, %p", pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo); - return TSDB_CODE_SUCCESS; + +_error: + pBlock->numOfRows = 0; + + tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, %p", + pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qinfo); + return terrno; } static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); From 251fdf2c9711388e9eccf4b203595b6dc8beee48 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 11:12:38 +0000 Subject: [PATCH 41/63] TD-1617 defect found by coverity scan --- src/dnode/src/dnodeModule.c | 4 ++-- src/mnode/src/mnodeTable.c | 4 ++-- src/os/src/detail/osDir.c | 3 ++- src/plugins/http/src/httpParser.c | 25 +++++++++++++++---------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 0a5b9b550c..18a293d415 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -147,13 +147,13 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { } bool dnodeCheckMnodeStarting() { - if (tsModuleStatus & TSDB_MOD_MNODE) return false; + if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) return false; SDMMnodeInfos *mnodes = dnodeGetMnodeInfos(); for (int32_t i = 0; i < mnodes->nodeNum; ++i) { SDMMnodeInfo *node = &mnodes->nodeInfos[i]; if (node->nodeId == dnodeGetDnodeId()) { - uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);; + uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE); dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus); dnodeProcessModuleStatus(moduleStatus); return true; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 43b80f65c8..1bc328800e 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2093,11 +2093,11 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->superTable) { + if (pTable->superTable != NULL) { tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); } - if (pTable->info.type == TSDB_CHILD_TABLE) { + if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) { pMeta->sversion = htons(pTable->superTable->sversion); pMeta->tversion = htons(pTable->superTable->tversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 30fd05ffc7..d3f0fda1a5 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -126,12 +126,13 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { dstFp = gzdopen(fd, "wb6f"); if (dstFp == NULL) { ret = -3; + close(fd); goto cmp_end; } while (!feof(srcFp)) { len = (int32_t)fread(data, 1, COMPRESS_STEP_SIZE, srcFp); - gzwrite(dstFp, data, len); + (void)gzwrite(dstFp, data, len); } cmp_end: diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 0c3204687a..b844834537 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -153,10 +153,10 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - httpAppendString(pParser->path + i, pStart, strlen(pStart)); + (void)httpAppendString(pParser->path + i, pStart, strlen(pStart)); break; } else { - httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); + (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); } pStart = pSeek + 1; } @@ -485,11 +485,11 @@ void httpClearParser(HttpParser *parser) { } void httpDestroyParser(HttpParser *parser) { + if (!parser) return; + HttpContext *pContext = parser->pContext; httpTrace("context:%p, fd:%d, destroy parser", pContext, pContext->fd); - if (!parser) return; - free(parser->method); parser->method = NULL; free(parser->target); parser->target = NULL; free(parser->version); parser->version = NULL; @@ -684,23 +684,28 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, break; } - if (c!='0' && c!='1') { + if (c != '0' && c != '1' && c != '2') { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } - - if (c == '0') parser->httpVersion = HTTP_VERSION_10; - else if (c == '1') parser->httpVersion = HTTP_VERSION_11; - else if (c == '2') parser->httpVersion = HTTP_VERSION_12; - else parser->httpVersion = HTTP_INVALID_VERSION; + + if (c == '0') + parser->httpVersion = HTTP_VERSION_10; + else if (c == '1') + parser->httpVersion = HTTP_VERSION_11; + else if (c == '2') + parser->httpVersion = HTTP_VERSION_12; + else { + } parser->version = strdup(parser->str.str); if (!parser->version) { From 24c5f02cc9df41c8643fa66ab38837b36f215ff9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 13:47:58 +0000 Subject: [PATCH 42/63] TD-1617 --- src/sync/inc/syncInt.h | 40 ++++++++++++++--------------- src/sync/src/syncMain.c | 3 +-- src/sync/src/tarbitrator.c | 52 +++++++++++++++++++------------------- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index cd1252f4b4..94df664219 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -114,26 +114,26 @@ typedef struct { } SSyncFwds; typedef struct SsyncPeer { - int32_t nodeId; - uint32_t ip; - uint16_t port; - char fqdn[TSDB_FQDN_LEN]; // peer ip string - char id[TSDB_EP_LEN+16]; // peer vgId + end point - int8_t role; - int8_t sstatus; // sync status - uint64_t version; - uint64_t sversion; // track the peer version in retrieve process - int syncFd; - int peerFd; // forward FD - int numOfRetrieves; // number of retrieves tried - int fileChanged; // a flag to indicate file is changed during retrieving process - void *timer; - void *pConn; - int notifyFd; - int watchNum; - int *watchFd; - int8_t refCount; // reference count - struct SSyncNode *pSyncNode; + int32_t nodeId; + uint32_t ip; + uint16_t port; + char fqdn[TSDB_FQDN_LEN]; // peer ip string + char id[TSDB_EP_LEN + 32]; // peer vgId + end point + int8_t role; + int8_t sstatus; // sync status + uint64_t version; + uint64_t sversion; // track the peer version in retrieve process + int syncFd; + int peerFd; // forward FD + int numOfRetrieves; // number of retrieves tried + int fileChanged; // a flag to indicate file is changed during retrieving process + void * timer; + void * pConn; + int notifyFd; + int watchNum; + int * watchFd; + int8_t refCount; // reference count + struct SSyncNode *pSyncNode; } SSyncPeer; typedef struct SSyncNode { diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 9b73558ba8..b8300dc75e 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -671,7 +671,6 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne int8_t selfOldRole = nodeRole; int8_t i, syncRequired = 0; - pNode->peerInfo[pNode->selfIndex]->version = nodeVersion; pPeer->role = newRole; sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]); @@ -923,7 +922,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { static int syncProcessPeerMsg(void *param, void *buffer) { SSyncPeer *pPeer = param; SSyncHead head; - char * cont = (char *)buffer; + char * cont = buffer; SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&(pNode->mutex)); diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index b704b1ecae..360ea93f6c 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -27,29 +27,29 @@ #include "tsync.h" #include "syncInt.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(void *param); -static int arbProcessPeerMsg(void *param, void *buffer); +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); +static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp); +static void arbProcessBrokenLink(void *param); +static int arbProcessPeerMsg(void *param, void *buffer); static tsem_t tsArbSem; static ttpool_h tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN+24]; + char id[TSDB_EP_LEN + 24]; int nodeFd; void *pConn; } SNodeConn; int main(int argc, char *argv[]) { - char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; + char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; - for (int i=1; i TSDB_FILENAME_LEN) continue; + } else if (strcmp(argv[i], "-g") == 0 && i < argc - 1) { + if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue; tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath)); } else { printf("\nusage: %s [options] \n", argv[0]); @@ -62,8 +62,8 @@ int main(int argc, char *argv[]) { } sDebugFlag = debugFlag; - - if (tsem_init(&tsArbSem, 0, 0) != 0) { + + if (tsem_init(&tsArbSem, 0, 0) != 0) { printf("failed to create exit semphore\n"); exit(EXIT_FAILURE); } @@ -91,10 +91,10 @@ int main(int argc, char *argv[]) { info.processIncomingMsg = arbProcessPeerMsg; info.processIncomingConn = arbProcessIncommingConnection; tsArbTcpPool = taosOpenTcpThreadPool(&info); - + if (tsArbTcpPool == NULL) { - sDebug("failed to open TCP thread pool, exit..."); - return -1; + sDebug("failed to open TCP thread pool, exit..."); + return -1; } sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); @@ -108,9 +108,8 @@ int main(int argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) -{ - char ipstr[24]; +static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) { + char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -121,15 +120,16 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) return; } - SNodeConn *pNode = (SNodeConn *) calloc(sizeof(SNodeConn), 1); + SNodeConn *pNode = (SNodeConn *)calloc(sizeof(SNodeConn), 1); if (pNode == NULL) { sError("failed to allocate memory(%s)", strerror(errno)); taosCloseSocket(connFd); return; } - snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); - if (firstPkt.syncHead.vgId) { + firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0; + snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); + if (firstPkt.syncHead.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); taosTFree(pNode); taosCloseSocket(connFd); @@ -151,10 +151,10 @@ static void arbProcessBrokenLink(void *param) { } static int arbProcessPeerMsg(void *param, void *buffer) { - SNodeConn * pNode = param; - SSyncHead head; - int bytes = 0; - char *cont = (char *)buffer; + SNodeConn *pNode = param; + SSyncHead head; + int bytes = 0; + char * cont = (char *)buffer; int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); if (hlen != sizeof(head)) { From 6d04062576b7519a136a59d0736445f615ce08de Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 13:50:44 +0000 Subject: [PATCH 43/63] TD-1617 --- src/sync/src/syncMain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b8300dc75e..ec5ffbfca7 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -622,7 +622,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - pNode->peerInfo[pNode->selfIndex]->role = nodeRole; + //pNode->peerInfo[pNode->selfIndex]->role = nodeRole; (*pNode->notifyRole)(pNode->ahandle, nodeRole); sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } @@ -671,6 +671,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne int8_t selfOldRole = nodeRole; int8_t i, syncRequired = 0; + //pNode->peerInfo[pNode->selfIndex]->version = nodeVersion; pPeer->role = newRole; sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]); From 2caeccf623584feeacadb178c59a1d63359b170d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 27 Sep 2020 15:34:44 +0000 Subject: [PATCH 44/63] TD-1617 --- src/sync/inc/syncInt.h | 1 - src/sync/inc/taosTcpPool.h | 1 - src/sync/src/syncMain.c | 34 +++--- src/sync/src/syncRestore.c | 79 ++++++------- src/sync/src/syncRetrieve.c | 179 ++++++++++++++--------------- src/sync/src/taosTcpPool.c | 81 ++++++------- src/sync/test/syncClient.c | 90 +++++++-------- src/sync/test/syncServer.c | 219 +++++++++++++++++------------------- 8 files changed, 336 insertions(+), 348 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 94df664219..f681810646 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -171,7 +171,6 @@ void syncBroadcastStatus(SSyncNode *pNode); void syncAddPeerRef(SSyncPeer *pPeer); int syncDecPeerRef(SSyncPeer *pPeer); - #ifdef __cplusplus } #endif diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 5f7ca9ede5..261d190ad3 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -38,7 +38,6 @@ void taosCloseTcpThreadPool(ttpool_h); void *taosAllocateTcpConn(void *, void *ahandle, int connFd); void taosFreeTcpConn(void *); - #ifdef __cplusplus } #endif diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ec5ffbfca7..c7b8959926 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -134,7 +134,7 @@ void syncCleanUp() { void *syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; - SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1); + SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1); if (pNode == NULL) { sError("no memory to allocate syncNode"); terrno = TAOS_SYSTEM_ERROR(errno); @@ -168,7 +168,7 @@ void *syncStart(const SSyncInfo *pInfo) { } syncAddNodeRef(pNode); - + if (pNode->selfIndex < 0) { sInfo("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; @@ -176,11 +176,12 @@ void *syncStart(const SSyncInfo *pInfo) { return NULL; } - nodeVersion = pInfo->version; // set the initial version + nodeVersion = pInfo->version; // set the initial version nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER; - sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]); + sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, + syncRole[nodeRole]); - pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo*sizeof(SFwdInfo), 1); + pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo * sizeof(SFwdInfo), 1); if (pNode->pSyncFwds == NULL) { sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); terrno = TAOS_SYSTEM_ERROR(errno); @@ -443,9 +444,7 @@ static void syncAddArbitrator(SSyncNode *pNode) { pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo); } -static void syncAddNodeRef(SSyncNode *pNode) { - atomic_add_fetch_8(&pNode->refCount, 1); -} +static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); } static void syncDecNodeRef(SSyncNode *pNode) { if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) { @@ -456,9 +455,7 @@ static void syncDecNodeRef(SSyncNode *pNode) { } } -void syncAddPeerRef(SSyncPeer *pPeer) { - atomic_add_fetch_8(&pPeer->refCount, 1); -} +void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { @@ -501,6 +498,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn)); pPeer->ip = ip; pPeer->port = pInfo->nodePort; + pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%d", pNode->vgId, pPeer->fqdn, pPeer->port); pPeer->peerFd = -1; @@ -573,10 +571,10 @@ static void syncChooseMaster(SSyncNode *pNode) { replica = pNode->replica + 1; } - if (index < 0 && onlineNum > replica/2.0) { + if (index < 0 && onlineNum > replica / 2.0) { // over half of nodes are online for (int i = 0; i < pNode->replica; ++i) { - //slave with highest version shall be master + // slave with highest version shall be master pPeer = pNode->peerInfo[i]; if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) { if (index < 0 || pPeer->version > pNode->peerInfo[index]->version) { @@ -622,7 +620,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - //pNode->peerInfo[pNode->selfIndex]->role = nodeRole; + // pNode->peerInfo[pNode->selfIndex]->role = nodeRole; (*pNode->notifyRole)(pNode->ahandle, nodeRole); sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } @@ -648,7 +646,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { static int syncValidateMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int code = 0; + int code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); @@ -671,7 +669,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne int8_t selfOldRole = nodeRole; int8_t i, syncRequired = 0; - //pNode->peerInfo[pNode->selfIndex]->version = nodeVersion; + // pNode->peerInfo[pNode->selfIndex]->version = nodeVersion; pPeer->role = newRole; sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]); @@ -877,8 +875,6 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version); } } - - return; } static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { @@ -1066,7 +1062,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { return; } - int32_t vgId = firstPkt.syncHead.vgId; + int32_t vgId = firstPkt.syncHead.vgId; SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 2a0bee3726..ebb6c3a0a9 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -23,10 +23,10 @@ #include "tsync.h" #include "syncInt.h" -static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eindex) { - char name[TSDB_FILENAME_LEN*2] = {0}; - char fname[TSDB_FILENAME_LEN*3] = {0}; - uint32_t magic; +static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) { + char name[TSDB_FILENAME_LEN * 2] = {0}; + char fname[TSDB_FILENAME_LEN * 3] = {0}; + uint32_t magic; uint64_t fversion; int64_t size; uint32_t index = sindex; @@ -40,12 +40,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind if (magic == 0) break; snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); - remove(fname); + (void)remove(fname); sDebug("%s, %s is removed", pPeer->id, fname); index++; if (index > eindex) break; - } + } } static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { @@ -62,35 +62,36 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { while (1) { // read file info int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); - if (ret < 0 ) break; + if (ret < 0) break; // if no more file from master, break; if (minfo.name[0] == 0 || minfo.magic == 0) { sDebug("%s, no more files to restore", pPeer->id); // remove extra files after the current index - syncRemoveExtraFile(pPeer, sinfo.index+1, TAOS_SYNC_MAX_INDEX); - code = 0; + syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); + code = 0; break; } - + // remove extra files on slave between the current and last index - syncRemoveExtraFile(pPeer, pindex+1, minfo.index-1); + syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); pindex = minfo.index; // check the file info sinfo = minfo; sDebug("%s, get file info:%s", pPeer->id, minfo.name); - sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); + sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, + &sinfo.fversion); // if file not there or magic is not the same, file shall be synced memset(&fileAck, 0, sizeof(fileAck)); - fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1:0; + fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0; // send file ack ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret <0) break; - + if (ret < 0) break; + // if sync is not required, continue if (fileAck.sync == 0) { sDebug("%s, %s is the same", pPeer->id, minfo.name); @@ -99,10 +100,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // if sync is required, open file, receive from master, and write to file // get the full path to file + minfo.name[sizeof(minfo.name) - 1] = 0; snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - if ( dfd < 0 ) { + if (dfd < 0) { sError("%s, failed to open file:%s", pPeer->id, name); break; } @@ -110,16 +112,15 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size); fsync(dfd); close(dfd); - if (ret<0) break; + if (ret < 0) break; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); - } if (code == 0 && (minfo.fversion != sinfo.fversion)) { - // data file is changed, code shall be set to 1 + // data file is changed, code shall be set to 1 *fversion = minfo.fversion; - code = 1; + code = 1; } if (code < 0) { @@ -130,8 +131,8 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { } static int syncRestoreWal(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - int ret, code = -1; + SSyncNode *pNode = pPeer->pSyncNode; + int ret, code = -1; void *buffer = calloc(1024000, 1); // size for one record if (buffer == NULL) return -1; @@ -140,18 +141,21 @@ static int syncRestoreWal(SSyncPeer *pPeer) { while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret <0) break; + if (ret < 0) break; + + if (pHead->len == 0) { + code = 0; + break; + } // wal sync over - if (pHead->len == 0) {code = 0; break;} // wal sync over - ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret <0) break; + if (ret < 0) break; sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version); (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL); } - if (code<0) { + if (code < 0) { sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno)); } @@ -159,10 +163,9 @@ static int syncRestoreWal(SSyncPeer *pPeer) { return code; } -static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) -{ +static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; - SWalHead *pHead = (SWalHead *) offset; + SWalHead * pHead = (SWalHead *)offset; (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD); offset += pHead->len + sizeof(SWalHead); @@ -171,7 +174,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) } static int syncProcessBufferedFwd(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; + SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; int forwards = 0; @@ -182,7 +185,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) { offset = syncProcessOneBufferedFwd(pPeer, offset); forwards++; } - + pthread_mutex_lock(&pNode->mutex); while (forwards < pRecv->forwards && pRecv->code == 0) { @@ -199,7 +202,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) { } int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { - SSyncNode *pNode = pPeer->pSyncNode; + SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; if (pRecv == NULL) return -1; @@ -259,9 +262,9 @@ static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { return -1; } - // if code > 0, data file is changed, notify app, and pass the version + // if code > 0, data file is changed, notify app, and pass the version if (code > 0 && pNode->notifyFileSynced) { - if ( (*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0 ) { + if ((*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0) { sError("%s, app not in ready state", pPeer->id); return -1; } @@ -296,8 +299,8 @@ void *syncRestoreData(void *param) { if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer", pPeer->id); - } else { - if ( syncRestoreDataStepByStep(pPeer) == 0) { + } else { + if (syncRestoreDataStepByStep(pPeer) == 0) { sInfo("%s, it is synced successfully", pPeer->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; syncBroadcastStatus(pNode); @@ -311,7 +314,7 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; - taosClose(pPeer->syncFd) + taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); syncDecPeerRef(pPeer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 8aa317b1ac..6d0b847afe 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -38,13 +38,13 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles); + memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); } int *wd = pPeer->watchFd + pPeer->watchNum; @@ -64,7 +64,7 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum); } - pPeer->watchNum = (pPeer->watchNum +1) % tsMaxWatchFiles; + pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles; return 0; } @@ -72,20 +72,20 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { static int syncAreFilesModified(SSyncPeer *pPeer) { if (pPeer->notifyFd <= 0) return 0; - char buf[2048]; - int len = read(pPeer->notifyFd, buf, sizeof(buf)); + char buf[2048]; + int len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - - int code = 0; - if (len > 0) { + + int code = 0; + if (len > 0) { const struct inotify_event *event; char *ptr; for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) { - event = (const struct inotify_event *) ptr; - if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) { + event = (const struct inotify_event *)ptr; + if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) { sDebug("%s, processed file is changed", pPeer->id); pPeer->fileChanged = 1; code = 1; @@ -98,11 +98,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) { } static int syncRetrieveFile(SSyncPeer *pPeer) { - SSyncNode * pNode = pPeer->pSyncNode; - SFileInfo fileInfo; - SFileAck fileAck; - int code = -1; - char name[TSDB_FILENAME_LEN * 2] = {0}; + SSyncNode *pNode = pPeer->pSyncNode; + SFileInfo fileInfo; + SFileAck fileAck; + int code = -1; + char name[TSDB_FILENAME_LEN * 2] = {0}; memset(&fileInfo, 0, sizeof(fileInfo)); memset(&fileAck, 0, sizeof(fileAck)); @@ -110,17 +110,19 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { while (1) { // retrieve file info fileInfo.name[0] = 0; - fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); - //fileInfo.size = htonl(size); + fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, + &fileInfo.size, &fileInfo.fversion); + // fileInfo.size = htonl(size); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); - if (ret < 0 ) break; + if (ret < 0) break; // if no file anymore, break - if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { - sDebug("%s, no more files to sync", pPeer->id); - code = 0; break; + if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { + sDebug("%s, no more files to sync", pPeer->id); + code = 0; + break; } // wait for the ack from peer @@ -132,29 +134,29 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { // get the full path to file snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); - + // add the file into watch list - if ( syncAddIntoWatchList(pPeer, name) <0) break; + if (syncAddIntoWatchList(pPeer, name) < 0) break; // if sync is not required, continue if (fileAck.sync == 0) { - fileInfo.index++; - sDebug("%s, %s is the same", pPeer->id, fileInfo.name); - continue; + fileInfo.index++; + sDebug("%s, %s is the same", pPeer->id, fileInfo.name); + continue; } // send the file to peer int sfd = open(name, O_RDONLY); if (sfd < 0) break; - ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); + ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) break; - sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); - fileInfo.index++; + sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); + fileInfo.index++; - // check if processed files are modified + // check if processed files are modified if (syncAreFilesModified(pPeer) != 0) break; } @@ -201,15 +203,15 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles); + memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); int *wd = pPeer->watchFd; - + *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE); if (*wd == -1) { sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno)); @@ -219,8 +221,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { return 0; } -static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { - char buf[2048]; +static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { + char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); @@ -231,26 +233,29 @@ static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { struct inotify_event *event; for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) { - event = (struct inotify_event *) ptr; + event = (struct inotify_event *)ptr; if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY; if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE; } - if (pEvent != 0) - sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent); + if (pEvent != 0) sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent); return 0; } static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { - SWalHead *pHead = (SWalHead *) malloc(640000); - int code = -1; - int32_t bytes = 0; - int sfd; + SWalHead *pHead = malloc(640000); + int code = -1; + int32_t bytes = 0; + int sfd; sfd = open(name, O_RDONLY); - if (sfd < 0) return -1; - lseek(sfd, offset, SEEK_SET); + if (sfd < 0) { + free(pHead); + return -1; + } + + (void)lseek(sfd, offset, SEEK_SET); sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); while (1) { @@ -263,34 +268,34 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); - if ( ret != wsize ) break; + if (ret != wsize) break; pPeer->sversion = pHead->version; bytes += wsize; - + if (pHead->version >= fversion && fversion > 0) { - code = 0; - bytes = 0; + code = 0; + bytes = 0; break; } } free(pHead); - taosClose(sfd); + close(sfd); if (code == 0) return bytes; return -1; } static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { - SSyncNode *pNode = pPeer->pSyncNode; - int code = -1; - char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file + SSyncNode *pNode = pPeer->pSyncNode; + int code = -1; + char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file if (syncAreFilesModified(pPeer) != 0) return -1; while (1) { - int32_t once = 0; // last WAL has once ever been processed + int32_t once = 0; // last WAL has once ever been processed int64_t offset = 0; uint64_t fversion = 0; uint32_t event = 0; @@ -300,48 +305,48 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname); // monitor last wal - if (syncMonitorLastWal(pPeer, fname) <0) break; + if (syncMonitorLastWal(pPeer, fname) < 0) break; while (1) { int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event); if (bytes < 0) break; // check file changes - if (syncCheckLastWalChanges(pPeer, &event) <0) break; + if (syncCheckLastWalChanges(pPeer, &event) < 0) break; // if file is not updated or updated once, set the fversion and sstatus if (((event & IN_MODIFY) == 0) || once) { if (fversion == 0) { pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt - fversion = nodeVersion; // must read data to fversion + fversion = nodeVersion; // must read data to fversion } } // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { - code = 0; + code = 0; sDebug("%s, data up to fversion:%ld has been read out, bytes:%d", pPeer->id, fversion, bytes); break; - } + } // if all data are read out, and no update if ((bytes == 0) && ((event & IN_MODIFY) == 0)) { // wal file is closed, break - if (event & IN_CLOSE_WRITE) { - code = 0; + if (event & IN_CLOSE_WRITE) { + code = 0; sDebug("%s, current wal is closed", pPeer->id); break; } - + // wal not closed, it means some data not flushed to disk, wait for a while usleep(10000); } - // if bytes>0, file is updated, or fversion is not reached but file still open, read again + // if bytes>0, file is updated, or fversion is not reached but file still open, read again once = 1; - offset += bytes; + offset += bytes; sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes); - event = event & (~IN_MODIFY); // clear IN_MODIFY flag + event = event & (~IN_MODIFY); // clear IN_MODIFY flag } if (code < 0) break; @@ -356,7 +361,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { break; } - // current last wal is closed, there is a new one + // current last wal is closed, there is a new one sDebug("%s, last wal is closed, try new one", pPeer->id); } @@ -377,14 +382,14 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); + code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); if (code < 0) break; // error if (wname[0] == 0) { // no wal file sDebug("%s, no wal file", pPeer->id); break; - } - - if (code == 0) { // last wal + } + + if (code == 0) { // last wal code = syncProcessLastWal(pPeer, wname, index); break; } @@ -392,26 +397,26 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { // get the full path to wal file snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - // send wal file, + // send wal file, // inotify is not required, old wal file won't be modified, even remove is ok if (stat(fname, &fstat) < 0) break; size = fstat.st_size; - sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); + sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); int sfd = open(fname, O_RDONLY); if (sfd < 0) break; - code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); - close(sfd); - if (code <0) break; + code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); + close(sfd); + if (code < 0) break; - index++; + index++; - if (syncAreFilesModified(pPeer) != 0) break; + if (syncAreFilesModified(pPeer) != 0) break; } if (code == 0) { - sDebug("%s, wal retrieve is finished", pPeer->id); + sDebug("%s, wal retrieve is finished", pPeer->id); pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); @@ -433,12 +438,12 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - if (write(pPeer->syncFd, (char *) &firstPkt, sizeof(firstPkt)) < 0) { + if (write(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { sError("%s, failed to send syncCmd", pPeer->id); return -1; } - pPeer->sversion = 0; + pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; sDebug("%s, start to retrieve file", pPeer->id); if (syncRetrieveFile(pPeer) < 0) { @@ -447,8 +452,7 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } // if no files are synced, there must be wal to sync, sversion must be larger than one - if (pPeer->sversion == 0) - pPeer->sversion = 1; + if (pPeer->sversion == 0) pPeer->sversion = 1; sDebug("%s, start to retrieve wal", pPeer->id); if (syncRetrieveWal(pPeer) < 0) { @@ -460,8 +464,8 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { - SSyncPeer * pPeer = (SSyncPeer *)param; - SSyncNode *pNode = pPeer->pSyncNode; + SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); pPeer->fileChanged = 0; @@ -470,7 +474,7 @@ void *syncRetrieveData(void *param) { sError("%s, failed to open socket to sync", pPeer->id); } else { sInfo("%s, sync tcp is setup", pPeer->id); - + if (syncRetrieveDataStepByStep(pPeer) == 0) { sDebug("%s, sync retrieve process is successful", pPeer->id); } else { @@ -482,12 +486,11 @@ void *syncRetrieveData(void *param) { if (pPeer->fileChanged) { // if file is changed 3 times continuously, start flow control pPeer->numOfRetrieves++; - if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) + if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2)); } else { pPeer->numOfRetrieves = 0; - if (pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); } pPeer->fileChanged = 0; diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 2f064ceb36..eda822b1ec 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -45,8 +45,8 @@ typedef struct { static void *taosAcceptPeerTcpConnection(void *argv); static void *taosProcessTcpData(void *param); +static void taosStopPoolThread(SThreadObj *pThread); static SThreadObj *taosGetTcpThread(SPoolObj *pPool); -static void taosStopPoolThread(SThreadObj* pThread); void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; @@ -58,8 +58,8 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { } pPool->info = *pInfo; - - pPool->pThread = (SThreadObj **) calloc(sizeof(SThreadObj *), pInfo->numOfThreads); + + pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads); if (pPool->pThread == NULL) { uError("TCP server, no enough memory"); free(pPool); @@ -68,17 +68,19 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port); if (pPool->acceptFd < 0) { - free(pPool->pThread); free(pPool); + free(pPool->pThread); + free(pPool); uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); return NULL; } pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pPool->thread), &thattr, (void *) taosAcceptPeerTcpConnection, pPool) != 0) { + if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) { uError("TCP server, failed to create accept thread, reason:%s", strerror(errno)); close(pPool->acceptFd); - free(pPool->pThread); free(pPool); + free(pPool->pThread); + free(pPool); return NULL; } @@ -89,29 +91,30 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { } void taosCloseTcpThreadPool(void *param) { - SPoolObj *pPool = (SPoolObj *)param; - SThreadObj *pThread; + SPoolObj * pPool = (SPoolObj *)param; + SThreadObj *pThread; - shutdown(pPool->acceptFd, SHUT_RD); + shutdown(pPool->acceptFd, SHUT_RD); pthread_join(pPool->thread, NULL); for (int i = 0; i < pPool->info.numOfThreads; ++i) { pThread = pPool->pThread[i]; - if (pThread) taosStopPoolThread(pThread); + if (pThread) taosStopPoolThread(pThread); } + uDebug("%p TCP pool is closed", pPool); + taosTFree(pPool->pThread); free(pPool); - uDebug("%p TCP pool is closed", pPool); } void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { struct epoll_event event; SPoolObj *pPool = (SPoolObj *)param; - SConnObj *pConn = (SConnObj *) calloc(sizeof(SConnObj), 1); + SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1); if (pConn == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -131,7 +134,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { uError("failed to add fd:%d(%s)", connFd, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); free(pConn); pConn = NULL; } else { @@ -143,8 +146,8 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { } void taosFreeTcpConn(void *param) { - SConnObj * pConn = (SConnObj *)param; - SThreadObj *pThread = pConn->pThread; + SConnObj * pConn = (SConnObj *)param; + SThreadObj *pThread = pConn->pThread; uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); pConn->closedByApp = 1; @@ -153,9 +156,9 @@ void taosFreeTcpConn(void *param) { static void taosProcessBrokenLink(SConnObj *pConn) { SThreadObj *pThread = pConn->pThread; - SPoolObj *pPool = pThread->pPool; - SPoolInfo *pInfo = &pPool->info; - + SPoolObj * pPool = pThread->pPool; + SPoolInfo * pInfo = &pPool->info; + if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); (*pInfo->processBrokenLink)(pConn->ahandle); @@ -169,24 +172,24 @@ static void taosProcessBrokenLink(SConnObj *pConn) { #define maxEvents 10 static void *taosProcessTcpData(void *param) { - SThreadObj *pThread = (SThreadObj *) param; - SPoolObj *pPool = pThread->pPool; - SPoolInfo *pInfo = &pPool->info; - SConnObj *pConn = NULL; + SThreadObj *pThread = (SThreadObj *)param; + SPoolObj * pPool = pThread->pPool; + SPoolInfo * pInfo = &pPool->info; + SConnObj * pConn = NULL; struct epoll_event events[maxEvents]; void *buffer = malloc(pInfo->bufferSize); taosBlockSIGPIPE(); while (1) { - if (pThread->stop) break; + if (pThread->stop) break; int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThread->stop) { uDebug("%p TCP epoll thread is exiting...", pThread); break; } - if (fdNum < 0) { + if (fdNum < 0) { uError("epoll_wait failed (%s)", strerror(errno)); continue; } @@ -215,27 +218,28 @@ static void *taosProcessTcpData(void *param) { taosFreeTcpConn(pConn); continue; } - } + } } } + uDebug("%p TCP epoll thread exits", pThread); + close(pThread->pollFd); free(pThread); free(buffer); - uDebug("%p TCP epoll thread exits", pThread); - return NULL; + return NULL; } static void *taosAcceptPeerTcpConnection(void *argv) { - SPoolObj *pPool = (SPoolObj *)argv; - SPoolInfo *pInfo = &pPool->info; + SPoolObj * pPool = (SPoolObj *)argv; + SPoolInfo *pInfo = &pPool->info; taosBlockSIGPIPE(); while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int connFd = accept(pPool->acceptFd, (struct sockaddr *) &clientAddr, &addrlen); + int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { if (errno == EINVAL) { uDebug("%p TCP server accept is exiting...", pPool); @@ -246,7 +250,7 @@ static void *taosAcceptPeerTcpConnection(void *argv) { } } - //uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); + // uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); taosKeepTcpAlive(connFd); (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } @@ -260,7 +264,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { if (pThread) return pThread; - pThread = (SThreadObj *) calloc(1, sizeof(SThreadObj)); + pThread = (SThreadObj *)calloc(1, sizeof(SThreadObj)); if (pThread == NULL) return NULL; pThread->pPool = pPool; @@ -273,7 +277,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int ret = pthread_create(&(pThread->thread), &thattr, (void *) taosProcessTcpData, pThread); + int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); pthread_attr_destroy(&thattr); if (ret != 0) { @@ -290,20 +294,20 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { return pThread; } -static void taosStopPoolThread(SThreadObj* pThread) { +static void taosStopPoolThread(SThreadObj *pThread) { pThread->stop = true; - + if (pThread->thread == pthread_self()) { pthread_detach(pthread_self()); return; } - // save thread ID into a local variable, since pThread is freed when the thread exits + // save thread ID into a local variable, since pThread is freed when the thread exits pthread_t thread = pThread->thread; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; + struct epoll_event event = {.events = EPOLLIN}; eventfd_t fd = eventfd(1, 0); if (fd == -1) { // failed to create eventfd, call pthread_cancel instead, which may result in data corruption @@ -319,4 +323,3 @@ static void taosStopPoolThread(SThreadObj* pThread) { pthread_join(thread, NULL); taosClose(fd); } - diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c index 16053d1088..23264dc8a0 100644 --- a/src/sync/test/syncClient.c +++ b/src/sync/test/syncClient.c @@ -25,31 +25,32 @@ typedef struct { int num; int numOfReqs; int msgSize; - tsem_t rspSem; - tsem_t *pOverSem; + tsem_t rspSem; + tsem_t * pOverSem; pthread_t thread; - void *pRpc; + void * pRpc; } SInfo; void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); + uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); - tsem_post(&pInfo->rspSem); + tsem_post(&pInfo->rspSem); } int tcount = 0; void *sendRequest(void *param) { - SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg = {0}; - + SInfo * pInfo = (SInfo *)param; + SRpcMsg rpcMsg = {0}; + uDebug("thread:%d, start to send request", pInfo->index); - while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); rpcMsg.contLen = pInfo->msgSize; @@ -57,8 +58,9 @@ void *sendRequest(void *param) { rpcMsg.msgType = 1; uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg); - if ( pInfo->num % 20000 == 0 ) + if (pInfo->num % 20000 == 0) { uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + } tsem_wait(&pInfo->rspSem); } @@ -72,12 +74,12 @@ int main(int argc, char *argv[]) { SRpcInit rpcInit; SRpcEpSet epSet; char secret[TSDB_KEY_LEN] = "mypassword"; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - struct timeval systemTime; - int64_t startTime, endTime; + int msgSize = 128; + int numOfReqs = 0; + int appThreads = 1; + char serverIp[40] = "127.0.0.1"; + struct timeval systemTime; + int64_t startTime, endTime; pthread_attr_t thattr; // server info @@ -102,30 +104,30 @@ int main(int argc, char *argv[]) { rpcInit.spi = 1; rpcInit.connType = TAOS_CONN_CLIENT; - for (int i=1; iindex = i; pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; @@ -177,18 +179,16 @@ int main(int argc, char *argv[]) { do { usleep(1); - } while ( tcount < appThreads); + } while (tcount < appThreads); gettimeofday(&systemTime, NULL); - endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime)/1000.0; // mseconds + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0; // mseconds - uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); - uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); taosCloseLog(); return 0; } - - diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index b802905038..8e769a461e 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -24,28 +24,27 @@ #include "twal.h" #include "tsync.h" -int msgSize = 128; -int commit = 0; -int dataFd = -1; -void *qhandle = NULL; -int walNum = 0; +int msgSize = 128; +int commit = 0; +int dataFd = -1; +void * qhandle = NULL; +int walNum = 0; uint64_t tversion = 0; -void *syncHandle; -int role; -int nodeId; -char path[256]; -int numOfWrites ; +void * syncHandle; +int role; +int nodeId; +char path[256]; +int numOfWrites; SSyncInfo syncInfo; SSyncCfg *pCfg; -int writeIntoWal(SWalHead *pHead) -{ +int writeIntoWal(SWalHead *pHead) { if (dataFd < 0) { - char walName[280]; + char walName[280]; snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum); - remove(walName); - dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) { + (void)remove(walName); + dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + if (dataFd < 0) { uInfo("failed to open wal file:%s(%s)", walName, strerror(errno)); return -1; } else { @@ -67,54 +66,52 @@ int writeIntoWal(SWalHead *pHead) dataFd = -1; numOfWrites = 0; } - + return 0; } -void confirmForward(void *ahandle, void *mhandle, int32_t code) -{ - SRpcMsg *pMsg = (SRpcMsg *)mhandle; +void confirmForward(void *ahandle, void *mhandle, int32_t code) { + SRpcMsg * pMsg = (SRpcMsg *)mhandle; SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); uDebug("ver:%" PRIu64 ", confirm is received", pHead->version); rpcFreeCont(pMsg->pCont); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = rpcMallocCont(msgSize); rpcMsg.contLen = msgSize; rpcMsg.handle = pMsg->handle; rpcMsg.code = code; rpcSendResponse(&rpcMsg); - taosFreeQitem(mhandle); + taosFreeQitem(mhandle); } int processRpcMsg(void *item) { - SRpcMsg *pMsg = (SRpcMsg *)item; - SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); - int code = -1; + SRpcMsg * pMsg = (SRpcMsg *)item; + SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); + int code = -1; if (role != TAOS_SYNC_ROLE_MASTER) { uError("not master, write failed, role:%s", syncRole[role]); } else { - pHead->version = ++tversion; pHead->msgType = pMsg->msgType; pHead->len = pMsg->contLen; uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version); - writeIntoWal(pHead); + writeIntoWal(pHead); syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC); code = 0; } - if (pCfg->quorum <= 1) { - taosFreeQitem(item); + if (pCfg->quorum <= 1) { rpcFreeCont(pMsg->pCont); + taosFreeQitem(item); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = rpcMallocCont(msgSize); rpcMsg.contLen = msgSize; rpcMsg.handle = pMsg->handle; @@ -126,7 +123,6 @@ int processRpcMsg(void *item) { } int processFwdMsg(void *item) { - SWalHead *pHead = (SWalHead *)item; if (pHead->version <= tversion) { @@ -142,11 +138,11 @@ int processFwdMsg(void *item) { // write into cache -/* - if (pHead->handle) { - syncSendFwdAck(syncHandle, pHead->handle, 0); - } -*/ + /* + if (pHead->handle) { + syncSendFwdAck(syncHandle, pHead->handle, 0); + } + */ taosFreeQitem(item); @@ -154,7 +150,6 @@ int processFwdMsg(void *item) { } int processWalMsg(void *item) { - SWalHead *pHead = (SWalHead *)item; if (pHead->version <= tversion) { @@ -168,11 +163,11 @@ int processWalMsg(void *item) { // write into cache -/* - if (pHead->handle) { - syncSendFwdAck(syncHandle, pHead->handle, 0); - } -*/ + /* + if (pHead->handle) { + syncSendFwdAck(syncHandle, pHead->handle, 0); + } + */ taosFreeQitem(item); @@ -180,15 +175,15 @@ int processWalMsg(void *item) { } void *processWriteQueue(void *param) { - int type; - void *item; + int type; + void *item; while (1) { int ret = taosReadQitem(qhandle, &type, &item); if (ret <= 0) { usleep(1000); continue; - } + } if (type == TAOS_QTYPE_RPC) { processRpcMsg(item); @@ -196,8 +191,7 @@ void *processWriteQueue(void *param) { processWalMsg(item); } else if (type == TAOS_QTYPE_FWD) { processFwdMsg(item); - } - + } } return NULL; @@ -224,21 +218,19 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char } void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - + uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen); - taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); + taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); } -uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) -{ - uint32_t magic; - struct stat fstat; - char aname[280]; +uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { + uint32_t magic; + struct stat fstat; + char aname[280]; if (*index == 2) { uInfo("wait for a while ....."); @@ -246,15 +238,15 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex } if (name[0] == 0) { - // find the file + // find the file snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index); - sprintf(name, "data/data.%d", *index); + sprintf(name, "data/data.%d", *index); } else { snprintf(aname, sizeof(aname), "%s/%s", path, name); } uInfo("get file info:%s", aname); - if ( stat(aname, &fstat) < 0 ) return 0; + if (stat(aname, &fstat) < 0) return 0; *size = fstat.st_size; magic = fstat.st_size; @@ -262,24 +254,22 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex return magic; } -int getWalInfo(void *ahandle, char *name, uint32_t *index) { - - struct stat fstat; - char aname[280]; +int getWalInfo(void *ahandle, char *name, uint32_t *index) { + struct stat fstat; + char aname[280]; name[0] = 0; - if (*index + 1> walNum) return 0; + if (*index + 1 > walNum) return 0; snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index); - sprintf(name, "wal/wal.%d", *index); + sprintf(name, "wal/wal.%d", *index); uInfo("get wal info:%s", aname); - if ( stat(aname, &fstat) < 0 ) return -1; + if (stat(aname, &fstat) < 0) return -1; - if (*index >= walNum-1) return 0; // no more + if (*index >= walNum - 1) return 0; // no more return 1; - } int writeToCache(void *ahandle, void *data, int type) { @@ -290,24 +280,19 @@ int writeToCache(void *ahandle, void *data, int type) { int msgSize = pHead->len + sizeof(SWalHead); void *pMsg = taosAllocateQitem(msgSize); memcpy(pMsg, pHead, msgSize); - taosWriteQitem(qhandle, type, pMsg); + taosWriteQitem(qhandle, type, pMsg); return 0; } -void confirmFwd(void *ahandle, int64_t version) { - - return; -} +void confirmFwd(void *ahandle, int64_t version) { return; } void notifyRole(void *ahandle, int8_t r) { role = r; printf("current role:%s\n", syncRole[role]); } - void initSync() { - pCfg->replica = 1; pCfg->quorum = 1; syncInfo.vgId = 1; @@ -339,20 +324,18 @@ void initSync() { taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn); } -void doSync() -{ - for (int i=0; i<5; ++i) { - if (tsSyncPort == pCfg->nodeInfo[i].nodePort) - nodeId = pCfg->nodeInfo[i].nodeId; +void doSync() { + for (int i = 0; i < 5; ++i) { + if (tsSyncPort == pCfg->nodeInfo[i].nodePort) nodeId = pCfg->nodeInfo[i].nodeId; } snprintf(path, sizeof(path), "/root/test/d%d", nodeId); - strcpy(syncInfo.path, path); + tstrncpy(syncInfo.path, path, sizeof(syncInfo.path)); - if ( syncHandle == NULL) { - syncHandle = syncStart(&syncInfo); + if (syncHandle == NULL) { + syncHandle = syncStart(&syncInfo); } else { - if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL; + if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL; } uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort); @@ -361,39 +344,39 @@ void doSync() int main(int argc, char *argv[]) { SRpcInit rpcInit; char dataName[20] = "server.data"; - pCfg = &syncInfo.syncCfg; + pCfg = &syncInfo.syncCfg; initSync(); memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7000; - rpcInit.label = "SER"; + rpcInit.localPort = 7000; + rpcInit.label = "SER"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer*1500; - rpcInit.afp = retrieveAuthInfo; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = tsShellActivityTimer * 1500; + rpcInit.afp = retrieveAuthInfo; - for (int i=1; ireplica = atoi(argv[++i]); - } else if (strcmp(argv[i], "-q")==0 && i < argc-1) { + } else if (strcmp(argv[i], "-q") == 0 && i < argc - 1) { pCfg->quorum = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d")==0 && i < argc-1) { + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { rpcDebugFlag = atoi(argv[++i]); } else { printf("\nusage: %s [options] \n", argv[0]); @@ -411,10 +394,10 @@ int main(int argc, char *argv[]) { exit(0); } } - + uDebugFlag = rpcDebugFlag; - dDebugFlag = rpcDebugFlag; - //tmrDebugFlag = rpcDebugFlag; + dDebugFlag = rpcDebugFlag; + // tmrDebugFlag = rpcDebugFlag; tsAsyncLog = 0; taosInitLog("server.log", 1000000, 10); @@ -443,35 +426,39 @@ int main(int argc, char *argv[]) { SNodesRole nroles; while (1) { - char c = getchar(); + int c = getchar(); - switch(c) { + switch (c) { case '1': - pCfg->replica = 1; doSync(); - break; + pCfg->replica = 1; + doSync(); + break; case '2': - pCfg->replica = 2; doSync(); + pCfg->replica = 2; + doSync(); break; case '3': - pCfg->replica = 3; doSync(); + pCfg->replica = 3; + doSync(); break; case '4': - pCfg->replica = 4; doSync(); + pCfg->replica = 4; + doSync(); break; case '5': - pCfg->replica = 5; doSync(); + pCfg->replica = 5; + doSync(); break; case 's': syncGetNodesRole(syncHandle, &nroles); - for (int i=0; ireplica; ++i) + for (int i = 0; i < pCfg->replica; ++i) printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]); break; default: break; } - if (c=='q') break; - + if (c == 'q') break; } syncStop(syncHandle); @@ -483,5 +470,3 @@ int main(int argc, char *argv[]) { return 0; } - - From 2811c96b307b977236514b66822dbc5ee742de05 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 10:29:49 +0800 Subject: [PATCH 45/63] TD-1530 --- cmake/define.inc | 12 ++++- src/client/src/TSDBJNIConnector.c | 10 ++-- src/client/src/tscServer.c | 4 +- src/client/src/tscStream.c | 4 +- src/common/src/tdataformat.c | 8 +-- src/connector/go | 2 +- src/os/inc/os.h | 4 ++ src/os/inc/osArm32.h | 90 +++++++++++++++++++++++++++++++ src/os/inc/osWindows.h | 4 +- src/os/src/detail/osSysinfo.c | 1 - src/query/src/qExecutor.c | 4 +- src/rpc/src/rpcMain.c | 2 +- src/rpc/src/rpcUdp.c | 2 +- src/util/src/hash.c | 2 +- 14 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 src/os/inc/osArm32.h diff --git a/cmake/define.inc b/cmake/define.inc index 84df6f5405..98b54bfb50 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -48,6 +48,7 @@ ENDIF () IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) + MESSAGE(STATUS "linux64 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () @@ -55,6 +56,7 @@ ENDIF () IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) + MESSAGE(STATUS "linux32 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () @@ -63,22 +65,26 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_64_) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) + MESSAGE(STATUS "arm64 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_32_) ADD_DEFINITIONS(-D_TD_ARM_) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + MESSAGE(STATUS "arm32 is defined") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64_) + MESSAGE(STATUS "mips64 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32_) + MESSAGE(STATUS "mips32 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () @@ -86,6 +92,7 @@ IF (TD_APLHINE) SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") link_libraries(/usr/lib/libargp.a) ADD_DEFINITIONS(-D_ALPINE) + MESSAGE(STATUS "aplhine is defined") ENDIF () IF (TD_LINUX) @@ -118,6 +125,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-DDARWIN) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) + MESSAGE(STATUS "darwin64 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") @@ -147,11 +155,13 @@ IF (TD_WINDOWS_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) + MESSAGE(STATUS "windows64 is defined") ENDIF () IF (TD_WINDOWS_32) ADD_DEFINITIONS(-D_TD_WINDOWS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) + MESSAGE(STATUS "windows32 is defined") ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index bd980b75a3..9368faa0ee 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -149,7 +149,7 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *e JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv *env, jobject jobj, jint optionIndex, jstring optionValue) { if (optionValue == NULL) { - jniDebug("option index:%d value is null", optionIndex); + jniDebug("option index:%d value is null", (int32_t)optionIndex); return 0; } @@ -183,7 +183,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv } (*env)->ReleaseStringUTFChars(env, optionValue, tz1); } else { - jniError("option index:%d is not found", optionIndex); + jniError("option index:%d is not found", (int32_t)optionIndex); } return res; @@ -227,10 +227,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, (uint16_t)jport); if (ret == 0) { jniError("jobj:%p, conn:%p, connect to database failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, jport); + (char *)host, (char *)user, (char *)dbname, (int32_t)jport); } else { jniDebug("jobj:%p, conn:%p, connect to database succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, jport); + (char *)host, (char *)user, (char *)dbname, (int32_t)jport); } if (host != NULL) (*env)->ReleaseStringUTFChars(env, jhost, host); @@ -385,7 +385,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsIm } jint ret = taos_affected_rows((SSqlObj *)res); - jniDebug("jobj:%p, conn:%p, sql:%p, res: %p, affect rows:%d", jobj, tscon, (void *)con, (void *)res, ret); + jniDebug("jobj:%p, conn:%p, sql:%p, res: %p, affect rows:%d", jobj, tscon, (void *)con, (void *)res, (int32_t)ret); return ret; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0a5cfef80a..e3bd30fb71 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -642,14 +642,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (int64_t)numOfSrcCols, tscGetNumOfColumns(pTableMeta)); return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->interval.interval < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->interval.interval); + tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, pSql, (int64_t)pQueryInfo->interval.interval); return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 81b8cf7359..d01ede279a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -398,8 +398,8 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { - tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream, - pQueryInfo->interval.interval, minIntervalTime); + tscWarn("%p stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream, + (int64_t)pQueryInfo->interval.interval, minIntervalTime); pQueryInfo->interval.interval = minIntervalTime; } diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index e7f40442a0..f669803263 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -313,13 +313,13 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); if (pCols == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", sizeof(SDataCols), strerror(errno)); + uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); return NULL; } pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); if (pCols->cols == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", sizeof(SDataCol) * maxCols, strerror(errno)); + uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); tdFreeDataCols(pCols); return NULL; } @@ -331,7 +331,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", sizeof(SDataCol) * maxCols, strerror(errno)); + uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); tdFreeDataCols(pCols); return NULL; } @@ -716,4 +716,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); return row; -} \ No newline at end of file +} diff --git a/src/connector/go b/src/connector/go index 8d7bf74385..06ec30a0f1 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b +Subproject commit 06ec30a0f1762e8169bf6b9045c82bcaa52bcdf0 diff --git a/src/os/inc/os.h b/src/os/inc/os.h index d4b71173a0..8ce49d6750 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -28,6 +28,10 @@ extern "C" { #include "osArm64.h" #endif +#ifdef _TD_ARM_32_ +#include "osArm32.h" +#endif + #ifdef _TD_LINUX_64 #include "osLinux64.h" #endif diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h new file mode 100644 index 0000000000..9a2b8c1b55 --- /dev/null +++ b/src/os/inc/osArm32.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_ARM32_H +#define TDENGINE_OS_ARM32_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TAOS_OS_FUNC_LZ4 +#define BUILDIN_CLZL(val) __builtin_clzll(val) +#define BUILDIN_CTZL(val) __builtin_ctzll(val) +#define BUILDIN_CLZ(val) __builtin_clz(val) +#define BUILDIN_CTZ(val) __builtin_ctz(val) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index d4f3d6d2af..dc1da35037 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -51,8 +51,6 @@ extern "C" { #endif -#define TAOS_OS_FUNC_ATOMIC - #define TAOS_OS_FUNC_LZ4 int32_t BUILDIN_CLZL(uint64_t val); int32_t BUILDIN_CLZ(uint32_t val); @@ -351,4 +349,4 @@ void wordfree(wordexp_t *pwordexp); #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 83ecd85809..f6470fc3e1 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -569,7 +569,6 @@ int taosSystem(const char *cmd) { } } -int _sysctl(struct __sysctl_args *args ); void taosSetCoreDump() { if (0 == tsEnableCoreFile) { return; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f2d324e376..d4c1c8560b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5050,8 +5050,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } qDebug( - "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%" PRIzu ", %"PRId64" points returned, total:%"PRId64", offset:%" PRId64, - pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, + "QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 " points returned, total:%" PRId64 ", offset:%" PRId64, + pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 50b1507a56..f0b8c996c5 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -240,7 +240,7 @@ void *rpcOpen(const SRpcInit *pInit) { size_t size = sizeof(SRpcConn) * pRpc->sessions; pRpc->connList = (SRpcConn *)calloc(1, size); if (pRpc->connList == NULL) { - tError("%s failed to allocate memory for taos connections, size:%ld", pRpc->label, size); + tError("%s failed to allocate memory for taos connections, size:%" PRId64, pRpc->label, (int64_t)size); rpcClose(pRpc); return NULL; } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 8e24aed8f7..4ea47582b9 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -211,7 +211,7 @@ static void *taosRecvUdpData(void *param) { char *tmsg = malloc(dataLen + tsRpcOverhead); if (NULL == tmsg) { - tError("%s failed to allocate memory, size:%ld", pConn->label, dataLen); + tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; } else { tDebug("UDP malloc mem: %p", tmsg); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 7427a2e4f3..625d4af1ac 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -738,7 +738,7 @@ void taosHashTableResize(SHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); - uDebug("hash table resize completed, new capacity:%"PRId64", load factor:%f, elapsed time:%fms", pHashObj->capacity, + uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } From 077c8ff41290320adf1100ba1200f3218f8b61f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 02:55:59 +0000 Subject: [PATCH 46/63] TD-1530 --- src/os/inc/osArm32.h | 99 +++++++++++++++++++++++++++++++++++++++++-- src/os/inc/osNingsi.h | 2 - 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 9a2b8c1b55..51d305bb9a 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -78,10 +78,101 @@ extern "C" { #include #define TAOS_OS_FUNC_LZ4 -#define BUILDIN_CLZL(val) __builtin_clzll(val) -#define BUILDIN_CTZL(val) __builtin_ctzll(val) -#define BUILDIN_CLZ(val) __builtin_clz(val) -#define BUILDIN_CTZ(val) __builtin_ctz(val) + #define BUILDIN_CLZL(val) __builtin_clzll(val) + #define BUILDIN_CTZL(val) __builtin_ctzll(val) + #define BUILDIN_CLZ(val) __builtin_clz(val) + #define BUILDIN_CTZ(val) __builtin_ctz(val) + +#define TAOS_OS_FUNC_ATOMIC + #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + + //specify for arm32 + #define atomic_store_8(ptr, val) (*(ptr)=(val)) + + #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap + + #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + // specify for arm32 + #define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) + + #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + // specify for arm32 + #define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) + + #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) #ifdef __cplusplus } diff --git a/src/os/inc/osNingsi.h b/src/os/inc/osNingsi.h index da7f796b59..d88d279745 100644 --- a/src/os/inc/osNingsi.h +++ b/src/os/inc/osNingsi.h @@ -129,8 +129,6 @@ void* atomic_exchange_ptr_impl( void **ptr, void *val ); #define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val)) #define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val)) - - #ifdef __cplusplus } #endif From ac004dc40adb59ddadcabadce07b16bffb846b70 Mon Sep 17 00:00:00 2001 From: Bo Xiao <69349626+boxiaobj@users.noreply.github.com> Date: Mon, 28 Sep 2020 11:19:32 +0800 Subject: [PATCH 47/63] Add alter user privilledge in Document add alter user privilledge --- documentation20/webdocs/markdowndocs/administrator-ch.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 50b388650b..ee978aa79a 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -233,6 +233,12 @@ ALTER USER PASS <'password'>; 修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 +``` +ALTER USER PRIVILEDGE <'super'|'write'|'read'>; +``` + +修改用户权限为：super/write/read。 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 + ``` SHOW USERS; ``` From 6a1825905b50005d8e1b3978c84814d4421db8f7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 03:22:31 +0000 Subject: [PATCH 48/63] minor changes --- src/os/inc/osArm32.h | 91 -------------------------------------- src/os/src/linux/aarch32.c | 34 ++++++++++++++ 2 files changed, 34 insertions(+), 91 deletions(-) create mode 100644 src/os/src/linux/aarch32.c diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 51d305bb9a..17b4d2dbd5 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -83,97 +83,6 @@ extern "C" { #define BUILDIN_CLZ(val) __builtin_clz(val) #define BUILDIN_CTZ(val) __builtin_ctz(val) -#define TAOS_OS_FUNC_ATOMIC - #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - - //specify for arm32 - #define atomic_store_8(ptr, val) (*(ptr)=(val)) - - #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - // specify for arm32 - #define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) - - #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - // specify for arm32 - #define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) - - #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #ifdef __cplusplus } #endif diff --git a/src/os/src/linux/aarch32.c b/src/os/src/linux/aarch32.c new file mode 100644 index 0000000000..62ef437685 --- /dev/null +++ b/src/os/src/linux/aarch32.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifdef _TD_ARM_32_ + +int8_t atomic_store_8(void *ptr, int8_t val) {} + return __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) +} + +int8_t atomic_fetch_sub_8(void *ptr, int8_t val) {} + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +} + +int8_t atomic_fetch_add_8(void *ptr, int8_t val) {} + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +} + +#endif + From ca028a3d17ce4cc15d34fe61ff032ef2416f2760 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 04:20:40 +0000 Subject: [PATCH 49/63] TD-1530 --- src/os/src/detail/CMakeLists.txt | 4 ++++ src/os/src/linux/aarch32.c | 34 -------------------------------- 2 files changed, 4 insertions(+), 34 deletions(-) delete mode 100644 src/os/src/linux/aarch32.c diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index afb8935453..cbb8d46da8 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -9,3 +9,7 @@ SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w) ADD_LIBRARY(osdetail ${SRC}) TARGET_LINK_LIBRARIES(osdetail os) + +IF (TD_ARM_64) + TARGET_LINK_LIBRARIES(osdetail atomic) +ENDIF () \ No newline at end of file diff --git a/src/os/src/linux/aarch32.c b/src/os/src/linux/aarch32.c deleted file mode 100644 index 62ef437685..0000000000 --- a/src/os/src/linux/aarch32.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#ifdef _TD_ARM_32_ - -int8_t atomic_store_8(void *ptr, int8_t val) {} - return __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) -} - -int8_t atomic_fetch_sub_8(void *ptr, int8_t val) {} - return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) -} - -int8_t atomic_fetch_add_8(void *ptr, int8_t val) {} - return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -} - -#endif - From e32e507e1f949443cb9eafe05cdf352cb402d92f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 14:29:23 +0800 Subject: [PATCH 50/63] TD-1530 --- cmake/define.inc | 6 +++--- src/common/src/ttypes.c | 8 ++++---- src/common/src/tvariant.c | 16 ++++++++-------- src/inc/taosdef.h | 2 +- src/os/inc/os.h | 6 +++--- src/os/src/detail/CMakeLists.txt | 4 ++-- src/os/src/linux/ningsi.c | 2 +- src/plugins/http/src/httpGcJson.c | 2 +- src/plugins/http/src/httpJson.c | 10 +++++----- src/sync/src/syncRetrieve.c | 2 +- src/sync/test/syncServer.c | 2 +- src/wal/test/waltest.c | 4 ++-- tests/comparisonTest/tdengine/tdengineTest.c | 2 +- tests/examples/c/demo.c | 3 ++- tests/tsim/src/simExe.c | 4 ++-- 15 files changed, 37 insertions(+), 36 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 98b54bfb50..5a5a667cd8 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -62,7 +62,7 @@ ENDIF () IF (TD_ARM_64) ADD_DEFINITIONS(-D_M_X64) - ADD_DEFINITIONS(-D_TD_ARM_64_) + ADD_DEFINITIONS(-D_TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") @@ -70,7 +70,7 @@ IF (TD_ARM_64) ENDIF () IF (TD_ARM_32) - ADD_DEFINITIONS(-D_TD_ARM_32_) + ADD_DEFINITIONS(-D_TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) MESSAGE(STATUS "arm32 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") @@ -102,7 +102,7 @@ IF (TD_LINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) IF (TD_NINGSI_60) - ADD_DEFINITIONS(-D_TD_NINGSI_60_) + ADD_DEFINITIONS(-D_TD_NINGSI_60) MESSAGE(STATUS "set ningsi macro to true") ENDIF () diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index ff417b6cde..45ec20ce45 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -235,7 +235,7 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num double csum = 0; csum = GET_DOUBLE_VAL(sum); csum += dsum; -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 SET_DOUBLE_VAL_ALIGN(sum, &csum); SET_DOUBLE_VAL_ALIGN(max, &fmax); SET_DOUBLE_VAL_ALIGN(min, &fmin); @@ -282,7 +282,7 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num csum += dsum; -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 SET_DOUBLE_VAL_ALIGN(sum, &csum); SET_DOUBLE_VAL_ALIGN(max, &dmax); SET_DOUBLE_VAL_ALIGN(min, &dmin); @@ -494,7 +494,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { break; } case TSDB_DATA_TYPE_FLOAT: { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 float fv = GET_FLOAT_VAL(src); SET_FLOAT_VAL_ALIGN(val, &fv); #else @@ -503,7 +503,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { break; }; case TSDB_DATA_TYPE_DOUBLE: { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 double dv = GET_DOUBLE_VAL(src); SET_DOUBLE_VAL_ALIGN(val, &dv); #else diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 069d8eb251..ba118d2ccb 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -709,7 +709,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 //memcpy(&payload, &value, sizeof(float)); float fv = (float)value; SET_FLOAT_VAL_ALIGN(payload, &fv); @@ -718,7 +718,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu #endif } } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 //memcpy(&payload, &pVariant->i64Key, sizeof(float)); float fv = (float)pVariant->i64Key; SET_FLOAT_VAL_ALIGN(payload, &fv); @@ -726,7 +726,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu *((float *)payload) = (float)pVariant->i64Key; #endif } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 //memcpy(&payload, &pVariant->dKey, sizeof(float)); float fv = (float)pVariant->dKey; SET_FLOAT_VAL_ALIGN(payload, &fv); @@ -738,7 +738,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return 0; } -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 float fv = GET_FLOAT_VAL(payload); if (isinf(fv) || isnan(fv) || fv > FLT_MAX || fv < -FLT_MAX) { return -1; @@ -765,21 +765,21 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 SET_DOUBLE_VAL_ALIGN(payload, &value); #else *((double *)payload) = value; #endif } } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 double dv = (double)(pVariant->i64Key); SET_DOUBLE_VAL_ALIGN(payload, &dv); #else *((double *)payload) = (double)pVariant->i64Key; #endif } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 double dv = (double)(pVariant->dKey); SET_DOUBLE_VAL_ALIGN(payload, &dv); #else @@ -790,7 +790,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return 0; } -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 double dv = GET_DOUBLE_VAL(payload); if (isinf(dv) || isnan(dv) || dv > DBL_MAX || dv < -DBL_MAX) { return -1; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 228aba2a5e..f636cde8e5 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -131,7 +131,7 @@ do { \ #define GET_INT16_VAL(x) (*(int16_t *)(x)) #define GET_INT32_VAL(x) (*(int32_t *)(x)) #define GET_INT64_VAL(x) (*(int64_t *)(x)) -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 #define GET_FLOAT_VAL(x) taos_align_get_float(x) #define GET_DOUBLE_VAL(x) taos_align_get_double(x) diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 8ce49d6750..86e16db8b1 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -24,11 +24,11 @@ extern "C" { #include "osDarwin.h" #endif -#ifdef _TD_ARM_64_ +#ifdef _TD_ARM_64 #include "osArm64.h" #endif -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 #include "osArm32.h" #endif @@ -44,7 +44,7 @@ extern "C" { #include "osAlpine.h" #endif -#ifdef _TD_NINGSI_60_ +#ifdef _TD_NINGSI_60 #include "osNingsi.h" #endif diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index cbb8d46da8..3c6810fcd5 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -10,6 +10,6 @@ SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w) ADD_LIBRARY(osdetail ${SRC}) TARGET_LINK_LIBRARIES(osdetail os) -IF (TD_ARM_64) +IF (TD_ARM_32) TARGET_LINK_LIBRARIES(osdetail atomic) -ENDIF () \ No newline at end of file +ENDIF () diff --git a/src/os/src/linux/ningsi.c b/src/os/src/linux/ningsi.c index 793ccac84a..2514c38ece 100644 --- a/src/os/src/linux/ningsi.c +++ b/src/os/src/linux/ningsi.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#ifdef _TD_NINGSI_60_ +#ifdef _TD_NINGSI_60 void* atomic_exchange_ptr_impl(void** ptr, void* val ) { void *old; do { diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index a291641dc3..fde8ae2176 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -145,7 +145,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d,", fields[i].name, *((int32_t *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%ld", fields[i].name, *((int64_t *)row[i])); + len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%" PRId64, fields[i].name, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%.5f", fields[i].name, *((float *)row[i])); diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 7600fb3e43..1aa6cfac4b 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -113,7 +113,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%lx\r\n", srcLen); + int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen); httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); @@ -267,9 +267,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { ptm = localtime(&tt); int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { - length += snprintf(ts + length, 8, ".%06ld", t % precision); + length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { - length += snprintf(ts + length, 5, ".%03ld", t % precision); + length += snprintf(ts + length, 5, ".%03" PRId64, t % precision); } httpJsonString(buf, ts, length); @@ -287,9 +287,9 @@ void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { ptm = localtime(&tt); int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); if (us) { - length += snprintf(ts + length, 8, ".%06ld", t % precision); + length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { - length += snprintf(ts + length, 5, ".%03ld", t % precision); + length += snprintf(ts + length, 5, ".%03" PRId64, t % precision); } length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 6d0b847afe..60625d75ec 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -325,7 +325,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { code = 0; - sDebug("%s, data up to fversion:%ld has been read out, bytes:%d", pPeer->id, fversion, bytes); + sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); break; } diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 8e769a461e..aa07779845 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -386,7 +386,7 @@ int main(int argc, char *argv[]) { printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); - printf(" [-v version]: initial node version, default is:%ld\n", syncInfo.version); + printf(" [-v version]: initial node version, default is:%" PRId64 "\n", syncInfo.version); printf(" [-r replica]: replicacation number, default is:%d\n", pCfg->replica); printf(" [-q quorum]: quorum, default is:%d\n", pCfg->quorum); printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 073dbf72af..bbee1347b8 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -71,7 +71,7 @@ int main(int argc, char *argv[]) { printf(" [-t total]: total wal files, default is:%d\n", total); printf(" [-r rows]: rows of records per wal file, default is:%d\n", rows); printf(" [-k keep]: keep the wal after closing, default is:%d\n", keep); - printf(" [-v version]: initial version, default is:%ld\n", ver); + printf(" [-v version]: initial version, default is:%" PRId64 "\n", ver); printf(" [-d debugFlag]: debug flag, default:%d\n", dDebugFlag); printf(" [-h help]: print out this help\n\n"); exit(0); @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { exit(-1); } - printf("version starts from:%ld\n", ver); + printf("version starts from:%" PRId64 "\n", ver); int contLen = sizeof(SWalHead) + size; SWalHead *pHead = (SWalHead *) malloc(contLen); diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index 3d78a3d0a5..237403f525 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -278,7 +278,7 @@ void writeData() { free(threads); - printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs); + printf("---- Spent %f seconds to insert %" PRId64 " records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs); } void readDataImp(void *param) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index f0e970c332..d64c0de1ce 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -19,6 +19,7 @@ #include #include #include +#include #include // TAOS header file int main(int argc, char *argv[]) { @@ -67,7 +68,7 @@ int main(int argc, char *argv[]) { // insert 10 records int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%ld, %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); if (taos_query(taos, qstr)) { printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 463dc33c7c..4bb0cbe86f 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -751,7 +751,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT:{ -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 float fv = 0; *(int32_t*)(&fv) = *(int32_t*)row[i]; sprintf(value, "%.5f", fv); @@ -761,7 +761,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } break; case TSDB_DATA_TYPE_DOUBLE: { -#ifdef _TD_ARM_32_ +#ifdef _TD_ARM_32 double dv = 0; *(int64_t*)(&dv) = *(int64_t*)row[i]; sprintf(value, "%.9lf", dv); From 3b724adeb3cad451d30a6643deb92bd33b757785 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 07:26:39 +0000 Subject: [PATCH 51/63] TD-1530 --- src/plugins/http/src/httpContext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 6abd382f8e..fb8ab48498 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -130,7 +130,7 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { uint64_t handleVal = (uint64_t)ptr; - HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(HttpContext *)); + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(uint64_t)); if (ppContext) { HttpContext *pContext = *ppContext; From dec6288006b3cc66124d09f8f09b579f22112062 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 07:59:35 +0000 Subject: [PATCH 52/63] TD-1530 --- src/plugins/http/src/httpContext.c | 10 +++++----- src/util/inc/tcache.h | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index fb8ab48498..106ba9a772 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -67,7 +67,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_CACHE_PTR_KEY, 2, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -117,8 +117,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->state = HTTP_CONTEXT_STATE_READY; pContext->parser = httpCreateParser(pContext); - uint64_t handleVal = (uint64_t)pContext; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(int64_t), &pContext, sizeof(int64_t), 3000); + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext; + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, TSDB_CACHE_PTR_LEN, &pContext, TSDB_CACHE_PTR_LEN, 3000); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); @@ -129,8 +129,8 @@ HttpContext *httpCreateContext(int32_t fd) { } HttpContext *httpGetContext(void *ptr) { - uint64_t handleVal = (uint64_t)ptr; - HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(uint64_t)); + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)ptr; + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, TSDB_CACHE_PTR_LEN); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index af5f30c7c3..6ef02b63d7 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -24,6 +24,16 @@ extern "C" { #include "tlockfree.h" #include "hash.h" +#if defined(_TD_ARM_32) + #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT + #define TSDB_CACHE_PTR_TYPE int32_t + #define TSDB_CACHE_PTR_LEN sizeof(int32_t) +#else + #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT + #define TSDB_CACHE_PTR_TYPE int64_t + #define TSDB_CACHE_PTR_LEN sizeof(int64_t) +#endif + typedef void (*__cache_free_fn_t)(void*); typedef struct SCacheStatis { From f4900138fe8e8e00c0be22a34c84d15ab29e3dfd Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 28 Sep 2020 17:09:07 +0800 Subject: [PATCH 53/63] [TD-1502] --- tests/examples/go/taosdemo.go | 409 ++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 tests/examples/go/taosdemo.go diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go new file mode 100644 index 0000000000..b42e1e6d70 --- /dev/null +++ b/tests/examples/go/taosdemo.go @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package main + +import ( + "database/sql" + "fmt" + _ "github.com/taosdata/driver-go/taosSql" + "os" + "sync" + "runtime" + "strconv" + "time" + "flag" + "math/rand" + //"golang.org/x/sys/unix" +) + +const ( + maxLocationSize = 32 + maxSqlBufSize = 65480 +) + +var locations = [maxLocationSize]string { + "Beijing", "Shanghai", "Guangzhou", "Shenzhen", + "HangZhou", "Tianjin", "Wuhan", "Changsha", + "Nanjing", "Xian"} + +type config struct { + hostName string + serverPort int + user string + password string + dbName string + supTblName string + tablePrefix string + numOftables int + numOfRecordsPerTable int + numOfRecordsPerReq int + numOfThreads int + startTimestamp string + startTs int64 + + keep int + days int +} + +var configPara config +var taosDriverName = "taosSql" +var url string + +func init() { + flag.StringVar(&configPara.hostName, "h", "127.0.0.1","The host to connect to TDengine server.") + flag.IntVar(&configPara.serverPort, "p", 6030, "The TCP/IP port number to use for the connection to TDengine server.") + flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") + flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") + flag.StringVar(&configPara.dbName, "d", "test", "Destination database.") + flag.StringVar(&configPara.tablePrefix, "m", "d", "Table prefix name.") + flag.IntVar(&configPara.numOftables, "t", 2, "The number of tables.") + flag.IntVar(&configPara.numOfRecordsPerTable, "n", 10, "The number of records per table.") + flag.IntVar(&configPara.numOfRecordsPerReq, "r", 3, "The number of records per request.") + flag.IntVar(&configPara.numOfThreads, "T", 1, "The number of threads.") + flag.StringVar(&configPara.startTimestamp, "s", "2020-10-01 08:00:00", "The start timestamp for one table.") + flag.Parse() + + configPara.keep = 365 * 20 + configPara.days = 30 + configPara.supTblName = "meters" + + startTs, err := time.ParseInLocation("2006-01-02 15:04:05", configPara.startTimestamp, time.Local) + if err==nil { + configPara.startTs = startTs.UnixNano() / 1e6 + } +} + +func printAllArgs() { + fmt.Printf("\n============= args parse result: =============\n") + fmt.Printf("dbName: %v\n", configPara.hostName) + fmt.Printf("serverPort: %v\n", configPara.serverPort) + fmt.Printf("usr: %v\n", configPara.user) + fmt.Printf("password: %v\n", configPara.password) + fmt.Printf("dbName: %v\n", configPara.dbName) + fmt.Printf("tablePrefix: %v\n", configPara.tablePrefix) + fmt.Printf("numOftables: %v\n", configPara.numOftables) + fmt.Printf("numOfRecordsPerTable: %v\n", configPara.numOfRecordsPerTable) + fmt.Printf("numOfRecordsPerReq: %v\n", configPara.numOfRecordsPerReq) + fmt.Printf("numOfThreads: %v\n", configPara.numOfThreads) + fmt.Printf("startTimestamp: %v[%v]\n", configPara.startTimestamp, configPara.startTs) + fmt.Printf("================================================\n") +} + +func main() { + printAllArgs() + fmt.Printf("Please press enter key to continue....\n") + fmt.Scanln() + + url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/" + //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) + // open connect to taos server + //db, err := sql.Open(taosDriverName, url) + //if err != nil { + // fmt.Println("Open database error: %s\n", err) + // os.Exit(1) + //} + //defer db.Close() + + createDatabase(configPara.dbName, configPara.supTblName) + fmt.Printf("======== create database success! ========\n\n") + + //create_table(db, stblName) + multiThreadCreateTable(configPara.numOfThreads, configPara.numOftables, configPara.dbName, configPara.tablePrefix) + fmt.Printf("======== create super table and child tables success! ========\n\n") + + //insert_data(db, demot) + multiThreadInsertData(configPara.numOfThreads, configPara.numOftables, configPara.dbName, configPara.tablePrefix) + fmt.Printf("======== insert data into child tables success! ========\n\n") + + //select_data(db, demot) + selectTest(configPara.dbName, configPara.tablePrefix, configPara.supTblName) + fmt.Printf("======== select data success! ========\n\n") + + fmt.Printf("======== end demo ========\n") +} + +func createDatabase(dbName string, supTblName string) { + db, err := sql.Open(taosDriverName, url) + if err != nil { + fmt.Println("Open database error: %s\n", err) + os.Exit(1) + } + defer db.Close() + + // drop database if exists + sqlStr := "drop database if exists " + dbName + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) + + time.Sleep(time.Second) + + // create database + sqlStr = "create database " + dbName + " keep " + strconv.Itoa(configPara.keep) + " days " + strconv.Itoa(configPara.days) + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) + + // use database + //sqlStr = "use " + dbName + //_, err = db.Exec(sqlStr) + //checkErr(err, sqlStr) + + sqlStr = "create table if not exists " + dbName + "." + supTblName + " (ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int);" + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) +} + +func multiThreadCreateTable(threads int, ntables int, dbName string, tablePrefix string) { + st := time.Now().UnixNano() + + if (threads < 1) { + threads = 1; + } + + a := ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + b := ntables % threads; + + last := 0; + endTblId := 0 + wg := sync.WaitGroup{} + for i := 0; i < threads; i++ { + startTblId := last + if (i < b ) { + endTblId = last + a + } else { + endTblId = last + a - 1 + } + last = endTblId + 1 + wg.Add(1) + go createTable(dbName, tablePrefix, startTblId, endTblId, &wg) + } + wg.Wait() + + et := time.Now().UnixNano() + fmt.Printf("create tables spent duration: %6.6fs\n", (float32(et-st))/1e9) +} + +func createTable(dbName string, childTblPrefix string, startTblId int, endTblId int, wg *sync.WaitGroup) { + //fmt.Printf("subThread[%d]: create table from %d to %d \n", unix.Gettid(), startTblId, endTblId) + // windows.GetCurrentThreadId() + + db, err := sql.Open(taosDriverName, url) + if err != nil { + fmt.Println("Open database error: %s\n", err) + os.Exit(1) + } + defer db.Close() + + for i := startTblId; i <= endTblId; i++ { + sqlStr := "create table if not exists " + dbName + "." + childTblPrefix + strconv.Itoa(i) + " using " + dbName + ".meters tags('" + locations[i%maxLocationSize] + "', " + strconv.Itoa(i) + ");" + //fmt.Printf("sqlStr: %v\n", sqlStr) + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) + } + wg.Done() + runtime.Goexit() +} + +func generateRowData(ts int64) string { + voltage := rand.Int() % 1000 + current := 200 + rand.Float32() + phase := rand.Float32() + values := "( " + strconv.FormatInt(ts, 10) + ", " + strconv.FormatFloat(float64(current), 'f', 6, 64) + ", " + strconv.Itoa(voltage) + ", " + strconv.FormatFloat(float64(phase), 'f', 6, 64) + " ) " + return values +} +func insertData(dbName string, childTblPrefix string, startTblId int, endTblId int, wg *sync.WaitGroup) { + //fmt.Printf("subThread[%d]: insert data to table from %d to %d \n", unix.Gettid(), startTblId, endTblId) + // windows.GetCurrentThreadId() + + db, err := sql.Open(taosDriverName, url) + if err != nil { + fmt.Println("Open database error: %s\n", err) + os.Exit(1) + } + defer db.Close() + + tmpTs := configPara.startTs; + //rand.New(rand.NewSource(time.Now().UnixNano())) + for tID := startTblId; tID <= endTblId; tID++{ + totalNum := 0 + for { + sqlStr := "insert into " + dbName + "." + childTblPrefix + strconv.Itoa(tID) + " values " + currRowNum := 0 + for { + tmpTs += 1000 + valuesOfRow := generateRowData(tmpTs) + currRowNum += 1 + totalNum += 1 + + sqlStr = fmt.Sprintf("%s %s", sqlStr, valuesOfRow) + + if (currRowNum >= configPara.numOfRecordsPerReq || totalNum >= configPara.numOfRecordsPerTable) { + break + } + } + + res, err := db.Exec(sqlStr) + checkErr(err, sqlStr) + + count, err := res.RowsAffected() + checkErr(err, "rows affected") + + if (count != int64(currRowNum)) { + fmt.Printf("insert data, expect affected:%d, actual:%d\n", currRowNum, count) + os.Exit(1) + } + + if (totalNum >= configPara.numOfRecordsPerTable) { + break + } + } + } + + wg.Done() + runtime.Goexit() +} +func multiThreadInsertData(threads int, ntables int, dbName string, tablePrefix string) { + st := time.Now().UnixNano() + + if (threads < 1) { + threads = 1; + } + + a := ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + b := ntables % threads; + + last := 0; + endTblId := 0 + wg := sync.WaitGroup{} + for i := 0; i < threads; i++ { + startTblId := last + if (i < b ) { + endTblId = last + a + } else { + endTblId = last + a - 1 + } + last = endTblId + 1 + wg.Add(1) + go insertData(dbName, tablePrefix, startTblId , endTblId, &wg) + } + wg.Wait() + + et := time.Now().UnixNano() + fmt.Printf("insert data spent duration: %6.6fs\n", (float32(et-st))/1e9) +} +func selectTest(dbName string, tbPrefix string, supTblName string){ + db, err := sql.Open(taosDriverName, url) + if err != nil { + fmt.Println("Open database error: %s\n", err) + os.Exit(1) + } + defer db.Close() + + // select sql 1 + limit := 3 + offset := 0 + sqlStr := "select * from " + dbName + "." + supTblName + " limit " + strconv.Itoa(limit) + " offset " + strconv.Itoa(offset) + rows, err := db.Query(sqlStr) + checkErr(err, sqlStr) + + defer rows.Close() + fmt.Printf("query sql: %s\n", sqlStr) + for rows.Next() { + var ( + ts string + current float32 + voltage int + phase float32 + location string + groupid int + ) + err := rows.Scan(&ts, ¤t, &voltage, &phase, &location, &groupid) + if err != nil { + checkErr(err, "rows scan fail") + } + + fmt.Printf("ts:%s\t current:%f\t voltage:%d\t phase:%f\t location:%s\t groupid:%d\n", ts, current, voltage, phase, location, groupid) + } + // check iteration error + if rows.Err() != nil { + checkErr(err, "rows next iteration error") + } + + // select sql 2 + sqlStr = "select avg(voltage), min(voltage), max(voltage) from " + dbName + "." + tbPrefix + strconv.Itoa( rand.Int() % configPara.numOftables) + rows, err = db.Query(sqlStr) + checkErr(err, sqlStr) + + defer rows.Close() + fmt.Printf("\nquery sql: %s\n", sqlStr) + for rows.Next() { + var ( + voltageAvg float32 + voltageMin int + voltageMax int + ) + err := rows.Scan(&voltageAvg, &voltageMin, &voltageMax) + if err != nil { + checkErr(err, "rows scan fail") + } + + fmt.Printf("avg(voltage):%f\t min(voltage):%d\t max(voltage):%d\n", voltageAvg, voltageMin, voltageMax) + } + // check iteration error + if rows.Err() != nil { + checkErr(err, "rows next iteration error") + } + + // select sql 3 + sqlStr = "select last(*) from " + dbName + "." + supTblName + rows, err = db.Query(sqlStr) + checkErr(err, sqlStr) + + defer rows.Close() + fmt.Printf("\nquery sql: %s\n", sqlStr) + for rows.Next() { + var ( + lastTs string + lastCurrent float32 + lastVoltage int + lastPhase float32 + ) + err := rows.Scan(&lastTs, &lastCurrent, &lastVoltage, &lastPhase) + if err != nil { + checkErr(err, "rows scan fail") + } + + fmt.Printf("last(ts):%s\t last(current):%f\t last(voltage):%d\t last(phase):%f\n", lastTs, lastCurrent, lastVoltage, lastPhase) + } + // check iteration error + if rows.Err() != nil { + checkErr(err, "rows next iteration error") + } +} +func checkErr(err error, prompt string) { + if err != nil { + fmt.Printf("%s\n", prompt) + panic(err) + } +} From 17190c6e3b653fa65c426e9e02ba4162f384378d Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 28 Sep 2020 17:09:47 +0800 Subject: [PATCH 54/63] [del old go demo file] --- tests/examples/go/src/taosapp/taosapp.go | 302 ----------------------- 1 file changed, 302 deletions(-) delete mode 100644 tests/examples/go/src/taosapp/taosapp.go diff --git a/tests/examples/go/src/taosapp/taosapp.go b/tests/examples/go/src/taosapp/taosapp.go deleted file mode 100644 index 30126ea571..0000000000 --- a/tests/examples/go/src/taosapp/taosapp.go +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package main - -import ( - "database/sql" - "time" - "log" - "fmt" - _ "github.com/taosdata/driver-go/taosSql" -) - -func main() { - taosDriverName := "taosSql" - demodb := "demodb" - demot := "demot" - - fmt.Printf("\n======== start demo test ========\n") - // open connect to taos server - db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/") - if err != nil { - log.Fatalf("Open database error: %s\n", err) - } - defer db.Close() - - drop_database(db, demodb) - create_database(db, demodb) - use_database(db, demodb) - create_table(db, demot) - insert_data(db, demot) - select_data(db, demot) - - fmt.Printf("\n======== start stmt mode test ========\n") - - demodbStmt := "demodbStmt" - demotStmt := "demotStmt" - drop_database_stmt(db, demodbStmt) - create_database_stmt(db, demodbStmt) - use_database_stmt(db, demodbStmt) - create_table_stmt(db, demotStmt) - insert_data_stmt(db, demotStmt) - select_data_stmt(db, demotStmt) - - fmt.Printf("\n======== end demo test ========\n") -} - -func drop_database(db *sql.DB, demodb string) { - st := time.Now().Nanosecond() - res, err := db.Exec("drop database if exists " + demodb) - checkErr(err, "drop database if exists " + demodb) - - affectd, err := res.RowsAffected() - checkErr(err, "drop db, res.RowsAffected") - - et := time.Now().Nanosecond() - - fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func create_database(db *sql.DB, demodb string) { - st := time.Now().Nanosecond() - // create database - res, err := db.Exec("create database " + demodb) - checkErr(err, "create db, db.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "create db, res.RowsAffected") - - et := time.Now().Nanosecond() - - fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) - - return -} - -func use_database(db *sql.DB, demodb string) { - st := time.Now().Nanosecond() - // use database - res, err := db.Exec("use " + demodb) // notes: must no quote to db name - checkErr(err, "use db db.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "use db, res.RowsAffected") - - et := time.Now().Nanosecond() - - fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func create_table(db *sql.DB, demot string) { - st := time.Now().Nanosecond() - // create table - res, err := db.Exec("create table " + demot + " (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)") - checkErr(err, "create table db.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "create table res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func insert_data(db *sql.DB, demot string) { - st := time.Now().Nanosecond() - // insert data - res, err := db.Exec("insert into " + demot + - " values (now, 100, 'beijing', 10, true, 'one', 123.456, 123.456)" + - " (now+1s, 101, 'shanghai', 11, true, 'two', 789.123, 789.123)" + - " (now+2s, 102, 'shenzhen', 12, false, 'three', 456.789, 456.789)") - - checkErr(err, "insert data, db.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "insert data res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func select_data(db *sql.DB, demot string) { - st := time.Now().Nanosecond() - - rows, err := db.Query("select * from ? " , demot) // go text mode - checkErr(err, "select db.Query") - - fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %4s%s %5s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv") - var affectd int - for rows.Next() { - var ts string - var name string - var id int - var len int8 - var flag bool - var notes string - var fv float32 - var dv float64 - - err = rows.Scan(&ts, &id, &name, &len, &flag, ¬es, &fv, &dv) - checkErr(err, "select rows.Scan") - - fmt.Printf("%s\t", ts) - fmt.Printf("%d\t",id) - fmt.Printf("%10s\t",name) - fmt.Printf("%d\t",len) - fmt.Printf("%t\t",flag) - fmt.Printf("%s\t",notes) - fmt.Printf("%06.3f\t",fv) - fmt.Printf("%09.6f\n",dv) - - affectd++ - } - - et := time.Now().Nanosecond() - fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) - fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func drop_database_stmt(db *sql.DB,demodb string) { - st := time.Now().Nanosecond() - // drop test db - res, err := db.Exec("drop database if exists " + demodb) - checkErr(err, "drop database " + demodb) - - affectd, err := res.RowsAffected() - checkErr(err, "drop db, res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func create_database_stmt(db *sql.DB,demodb string) { - st := time.Now().Nanosecond() - // create database - //var stmt interface{} - stmt, err := db.Prepare("create database ?") - checkErr(err, "create db, db.Prepare") - - //var res driver.Result - res, err := stmt.Exec(demodb) - checkErr(err, "create db, stmt.Exec") - - //fmt.Printf("Query OK, %d row(s) affected()", res.RowsAffected()) - affectd, err := res.RowsAffected() - checkErr(err, "create db, res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func use_database_stmt (db *sql.DB,demodb string) { - st := time.Now().Nanosecond() - // create database - //var stmt interface{} - stmt, err := db.Prepare("use " + demodb) - checkErr(err, "use db, db.Prepare") - - res, err := stmt.Exec() - checkErr(err, "use db, stmt.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "use db, res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func create_table_stmt (db *sql.DB,demot string) { - st := time.Now().Nanosecond() - // create table - // (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double) - stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(10), ? tinyint, ? bool, ? binary(8), ? float, ? double)") - checkErr(err, "create table db.Prepare") - - res, err := stmt.Exec(demot, "ts", "id", "name", "len", "flag", "notes", "fv", "dv") - checkErr(err, "create table stmt.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "create table res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func insert_data_stmt(db *sql.DB,demot string) { - st := time.Now().Nanosecond() - // insert data into table - stmt, err := db.Prepare("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?)") - checkErr(err, "insert db.Prepare") - - res, err := stmt.Exec(demot, "now" , 1000, "'haidian'" , 6, true, "'AI world'", 6987.654, 321.987, - "now+1s", 1001, "'changyang'" , 7, false, "'DeepMode'", 12356.456, 128634.456, - "now+2s", 1002, "'chuangping'" , 8, true, "'database'", 3879.456, 65433478.456,) - checkErr(err, "insert data, stmt.Exec") - - affectd, err := res.RowsAffected() - checkErr(err, "res.RowsAffected") - - et := time.Now().Nanosecond() - fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func select_data_stmt(db *sql.DB, demot string) { - st := time.Now().Nanosecond() - - stmt, err := db.Prepare("select ?, ?, ?, ?, ?, ?, ?, ? from ?" ) // go binary mode - checkErr(err, "db.Prepare") - - rows, err := stmt.Query("ts", "id","name","len", "flag","notes", "fv", "dv", demot) - checkErr(err, "stmt.Query") - - fmt.Printf("%10s%s%8s %5s %8s%s %s %10s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv") - var affectd int - for rows.Next() { - var ts string - var name string - var id int - var len int8 - var flag bool - var notes string - var fv float32 - var dv float64 - - err = rows.Scan(&ts, &id, &name, &len, &flag, ¬es, &fv, &dv) - //fmt.Println("start scan fields from row.rs, &fv:", &fv) - //err = rows.Scan(&fv) - checkErr(err, "rows.Scan") - - fmt.Printf("%s\t", ts) - fmt.Printf("%d\t",id) - fmt.Printf("%10s\t",name) - fmt.Printf("%d\t",len) - fmt.Printf("%t\t",flag) - fmt.Printf("%s\t",notes) - fmt.Printf("%06.3f\t",fv) - fmt.Printf("%09.6f\n",dv) - - affectd++ - - } - - et := time.Now().Nanosecond() - fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9) -} - -func checkErr(err error, prompt string) { - if err != nil { - fmt.Printf("%s\n", prompt) - panic(err) - } -} From fed24001bc1a191f3f126a8bd3c61f03db2453fd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 18:36:10 +0800 Subject: [PATCH 55/63] TD1530 --- src/plugins/http/src/httpContext.c | 5 +++-- src/util/inc/tcache.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 106ba9a772..ec60b984b2 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -118,7 +118,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->parser = httpCreateParser(pContext); TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, TSDB_CACHE_PTR_LEN, &pContext, TSDB_CACHE_PTR_LEN, 3000); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &pContext, + sizeof(TSDB_CACHE_PTR_TYPE), 3000); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); @@ -130,7 +131,7 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)ptr; - HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, TSDB_CACHE_PTR_LEN); + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 6ef02b63d7..efd51f90ce 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -27,11 +27,9 @@ extern "C" { #if defined(_TD_ARM_32) #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT #define TSDB_CACHE_PTR_TYPE int32_t - #define TSDB_CACHE_PTR_LEN sizeof(int32_t) #else #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT #define TSDB_CACHE_PTR_TYPE int64_t - #define TSDB_CACHE_PTR_LEN sizeof(int64_t) #endif typedef void (*__cache_free_fn_t)(void*); From 05687d1eba69af601ed9c741cdb332538f92ad76 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Sep 2020 22:30:34 +0800 Subject: [PATCH 56/63] TD-1530 minor changes --- cmake/define.inc | 2 +- src/client/src/tscServer.c | 2 +- src/common/src/tdataformat.c | 6 +++--- src/os/src/detail/CMakeLists.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5a5a667cd8..7cc8dc7a26 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -57,7 +57,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e3bd30fb71..2e92b67181 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -642,7 +642,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (int64_t)numOfSrcCols, + tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (uint64_t)numOfSrcCols, tscGetNumOfColumns(pTableMeta)); return TSDB_CODE_TSC_INVALID_SQL; diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f669803263..9c241c5c43 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -313,13 +313,13 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); if (pCols == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); return NULL; } pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); if (pCols->cols == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); tdFreeDataCols(pCols); return NULL; } @@ -331,7 +331,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { - uDebug("malloc failure, size:%"PRId64" failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); tdFreeDataCols(pCols); return NULL; } diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index 3c6810fcd5..0d5c130d6e 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -10,6 +10,6 @@ SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w) ADD_LIBRARY(osdetail ${SRC}) TARGET_LINK_LIBRARIES(osdetail os) -IF (TD_ARM_32) +IF (TD_ARM_32 OR TD_LINUX_32) TARGET_LINK_LIBRARIES(osdetail atomic) ENDIF () From dcd2eb4bf63383fb8dbdfaed52ae0327cd922523 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 08:46:22 +0800 Subject: [PATCH 57/63] Update administrator-ch.md --- .../webdocs/markdowndocs/administrator-ch.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index ee978aa79a..828b91f083 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -130,9 +130,21 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - statusInterval: dnode向mnode报告状态时长。单位为秒，默认值：1。 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 - maxVgroupsPerDb: 每个数据库中能够使用的最大vnode个数。 -- arbitrator: 系统中裁决器的end point，缺省为空 +- arbitrator: 系统中裁决器的end point，缺省为空。 - timezone、locale、charset 的配置见客户端配置。 +为方便调试，可通过SQL语句临时调整每个dnode的日志配置，系统重启后会失效： + +```mysql +ALTER DNODE +``` + +- dnode_id: 可以通过SQL语句"SHOW DNODES"命令获取 +- config: 要调整的日志参数，在如下列表中取值 + > resetlog 截断旧日志文件，创建一个新日志文件 + > debugFlag < 131 | 135 | 143 > 设置debugFlag为131、135或者143 + + ## 客户端配置 TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 @@ -392,5 +404,5 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 - +## From f191236d958dddabe2c50b3f7fdaa095b252a7b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 08:49:55 +0800 Subject: [PATCH 58/63] Update administrator-ch.md --- documentation20/webdocs/markdowndocs/administrator-ch.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 828b91f083..d1ad107db6 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -144,6 +144,10 @@ ALTER DNODE > resetlog 截断旧日志文件，创建一个新日志文件 > debugFlag < 131 | 135 | 143 > 设置debugFlag为131、135或者143 +例如： +``` + alter dnode 1 debugFlag 135; +``` ## 客户端配置 From 66b25f1a3f5c90179002d8d107e6d7df5a6c035a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 08:50:33 +0800 Subject: [PATCH 59/63] Update administrator-ch.md From 226c826a54fd8ba4abfcedf71a1203f379cef319 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 08:55:59 +0800 Subject: [PATCH 60/63] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 7bbf7531c8..a47f315490 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -108,4 +108,8 @@ Connection = DriverManager.getConnection(url, properties); 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 -为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。也可以通过执行SQL语句 +``` + alter dnode debugFlag 135; +``` +临时设置taosd的日志级别。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 From f9c9699ec93fdcd8c416c835ff0f7dafbaf7a309 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 09:46:42 +0800 Subject: [PATCH 61/63] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index a47f315490..6a4d4be118 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -108,8 +108,8 @@ Connection = DriverManager.getConnection(url, properties); 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 -为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。也可以通过执行SQL语句 +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。也可以通过如下SQL语句，临时设置taosd的日志级别。 ``` alter dnode debugFlag 135; ``` -临时设置taosd的日志级别。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 +但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 From fe1cbc31af1eccbbdf572bb78b0ff50f2c9a61d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Sep 2020 10:39:19 +0800 Subject: [PATCH 62/63] Update tsdbRead.c --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 1db2f20a42..a3bc0de272 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2551,7 +2551,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co } CATCH( code ) { CLEANUP_EXECUTE(); terrno = code; - //tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases + tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases goto _error; // TODO: more error handling From 2f346de95ebd7c4d0186cbfa8a2cf657ed516182 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Sep 2020 03:16:47 +0000 Subject: [PATCH 63/63] Fix TD-1635 --- src/inc/taoserror.h | 1 + src/mnode/src/mnodeSdb.c | 6 ++++- src/wal/src/walMain.c | 48 ++++++++++++++++++++++++++++++++-------- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 67e2d43c98..17b2d24e90 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -246,6 +246,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync modul // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted") // http TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 646c17b2b8..4c672eb557 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -185,7 +185,11 @@ static int32_t sdbInitWal() { } sdbInfo("open sdb wal for restore"); - walRestore(tsSdbObj.wal, NULL, sdbWrite); + int code = walRestore(tsSdbObj.wal, NULL, sdbWrite); + if (code != TSDB_CODE_SUCCESS) { + sdbError("failed to open wal for restore, reason:%s", tstrerror(code)); + return -1; + } return 0; } diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index bebad69f32..4987ba2116 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -347,9 +347,10 @@ static void walRelease(SWal *pWal) { static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { char *name = pWal->name; + int size = 1024 * 1024; // default 1M buffer size terrno = 0; - char *buffer = malloc(1024000); // size for one record + char *buffer = malloc(size); if (buffer == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; @@ -357,7 +358,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { SWalHead *pHead = (SWalHead *)buffer; - int fd = open(name, O_RDONLY); + int fd = open(name, O_RDWR); if (fd < 0) { wError("wal:%s, failed to open for restore(%s)", name, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -367,29 +368,58 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { wDebug("wal:%s, start to restore", name); + size_t offset = 0; while (1) { int ret = taosTRead(fd, pHead, sizeof(SWalHead)); - if ( ret == 0) break; + if (ret == 0) break; - if (ret != sizeof(SWalHead)) { - wWarn("wal:%s, failed to read head, skip, ret:%d(%s)", name, ret, strerror(errno)); + if (ret < 0) { + wError("wal:%s, failed to read wal head part since %s", name, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); break; } + if (ret < sizeof(SWalHead)) { + wError("wal:%s, failed to read head, ret:%d, skip the rest of file", name, ret); + taosFtruncate(fd, offset); + fsync(fd); + break; + } + if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wWarn("wal:%s, cksum is messed up, skip the rest of file", name); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(false); break; - } + } + + if (pHead->len > size - sizeof(SWalHead)) { + size = sizeof(SWalHead) + pHead->len; + buffer = realloc(buffer, size); + if (buffer == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + + pHead = (SWalHead *)buffer; + } ret = taosTRead(fd, pHead->cont, pHead->len); - if ( ret != pHead->len) { - wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret); + if (ret < 0) { + wError("wal:%s failed to read wal body part since %s", name, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); break; } + if (ret < pHead->len) { + wError("wal:%s, failed to read body, len:%d ret:%d, skip the rest of file", name, pHead->len, ret); + taosFtruncate(fd, offset); + fsync(fd); + break; + } + + offset = offset + sizeof(SWalHead) + pHead->len; + if (pWal->keep) pWal->version = pHead->version; (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL); }