From 4918783d5a3e1f5c6b8c6743e96a87db5ebe232a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 9 Nov 2019 22:30:27 +0800 Subject: [PATCH 01/69] refactor the header file format in client module --- src/os/linux/inc/os.h | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 004896960e..1a10bc3fe2 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -1,17 +1,17 @@ /* -* Copyright (c) 2019 TAOS Data, Inc. -* -* This program is free software: you can use, redistribute, and/or modify -* it under the terms of the GNU Affero General Public License, version 3 -* or later ("AGPL"), as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, but WITHOUT -* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -* FITNESS FOR A PARTICULAR PURPOSE. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #ifndef TDENGINE_PLATFORM_LINUX_H #define TDENGINE_PLATFORM_LINUX_H @@ -25,10 +25,12 @@ extern "C" { #include #include +#include #include #include #include #include +#include #include #include #include @@ -37,7 +39,8 @@ extern "C" { #include #include #include -#include +#include +#include #include #include #include @@ -45,21 +48,19 @@ extern "C" { #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include #include #include #include -#include -#include #define taosCloseSocket(x) \ { \ @@ -81,7 +82,7 @@ extern "C" { #define __sync_sub_and_fetch_32 __sync_sub_and_fetch int32_t __sync_val_load_32(int32_t *ptr); -void __sync_val_restore_32(int32_t *ptr, int32_t newval); +void __sync_val_restore_32(int32_t *ptr, int32_t newval); #define SWAP(a, b, c) \ do { \ @@ -139,9 +140,9 @@ bool taosSkipSocketCheck(); int64_t str2int64(char *str); #define BUILDIN_CLZL(val) __builtin_clzl(val) -#define BUILDIN_CLZ(val) __builtin_clz(val) +#define BUILDIN_CLZ(val) __builtin_clz(val) #define BUILDIN_CTZL(val) __builtin_ctzl(val) -#define BUILDIN_CTZ(val) __builtin_ctz(val) +#define BUILDIN_CTZ(val) __builtin_ctz(val) #ifdef __cplusplus } From c0d6cb743fd1dbfded9bafd5de6fe05231f65d8c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 9 Nov 2019 22:38:22 +0800 Subject: [PATCH 02/69] refactor the header file format in client module --- src/client/src/TSDBJNIConnector.c | 2 -- src/client/src/tscAst.c | 6 ------ src/client/src/tscAsync.c | 3 +-- src/client/src/tscCache.c | 9 +-------- src/client/src/tscFunctionImpl.c | 10 ---------- src/client/src/tscJoinProcess.c | 7 ------- src/client/src/tscLocal.c | 4 +--- src/client/src/tscParseInsert.c | 16 +--------------- src/client/src/tscPrepare.c | 3 --- src/client/src/tscProfile.c | 3 --- src/client/src/tscSQLParserImpl.c | 7 ------- src/client/src/tscSchemaUtil.c | 4 ---- src/client/src/tscSecondaryMerge.c | 7 +------ src/client/src/tscServer.c | 8 +------- src/client/src/tscSql.c | 3 --- src/client/src/tscSub.c | 2 +- src/client/src/tscSyntaxtreefunction.c | 5 +---- src/client/src/tscSystem.c | 9 --------- src/client/src/tscUtil.c | 4 ---- 19 files changed, 8 insertions(+), 104 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index fb175618c0..e98acfc9a6 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include - #include "os.h" #include "com_taosdata_jdbc_TSDBJNIConnector.h" #include "taos.h" diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index 845a9fd36d..fe0921c64d 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tast.h" diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 6e0e1ae7c5..aad0295959 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include -#include +#include "os.h" #include "tlog.h" #include "trpc.h" diff --git a/src/client/src/tscCache.c b/src/client/src/tscCache.c index 866b6e7dbc..f508857ce1 100644 --- a/src/client/src/tscCache.c +++ b/src/client/src/tscCache.c @@ -13,14 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tglobalcfg.h" #include "tlog.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0f9be90412..35011a259b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -15,16 +15,6 @@ #pragma GCC diagnostic ignored "-Wincompatible-pointer-types" -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tast.h" diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index 4965498eff..dfeebc8c61 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tcache.h" #include "tscJoinProcess.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 9afb74fec2..d217d09588 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "tcache.h" diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 37a32e9ebb..c8ce728de1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -21,22 +21,8 @@ #pragma GCC diagnostic ignored "-Woverflow" #pragma GCC diagnostic ignored "-Wunused-variable" -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "ihash.h" #include "os.h" +#include "ihash.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index e956d6159e..03f644ac36 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "taos.h" #include "tsclient.h" #include "tsql.h" diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 770e61e278..7d74f62dcc 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "os.h" #include "tlog.h" #include "tsclient.h" diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index 7e1b4a7cf1..4fefe1665c 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tglobalcfg.h" #include "tsql.h" diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 9728811ae7..5e50c27ab1 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -13,10 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tschemautil.h" diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index ac2638f635..398cd1cca6 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -13,13 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include "tlosertree.h" +#include "os.h" #include "tlosertree.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6edec3d477..73e612962a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #include "os.h" #include "tcache.h" #include "trpc.h" @@ -3613,7 +3607,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) { code = tscDoGetMeterMeta(pSql, meterId, 0); // todo ?? } else { - tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", pSql, + tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%lld, addr:%p", pSql, pMeterMetaInfo->pMeterMeta->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, pMeterMetaInfo->pMeterMeta); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e6714bb4cb..1727840a98 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "os.h" #include "tcache.h" #include "tlog.h" diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index dee8f02118..bcbcaba4c4 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +#include "os.h" #include "shash.h" #include "taos.h" diff --git a/src/client/src/tscSyntaxtreefunction.c b/src/client/src/tscSyntaxtreefunction.c index 00781919e4..bbd30c1fa9 100644 --- a/src/client/src/tscSyntaxtreefunction.c +++ b/src/client/src/tscSyntaxtreefunction.c @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include +#include "os.h" #include "tscSyntaxtreefunction.h" #include "tsql.h" diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 7ebd43cd19..708ceee405 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -13,15 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tcache.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ec5c980612..5793656477 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -13,10 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include - #include "os.h" #include "ihash.h" #include "taosmsg.h" From d04354a8ac2f7dd9ba521d755e5d484a203783d9 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 10:19:40 +0800 Subject: [PATCH 03/69] [TBASE-816] --- src/client/src/tscFunctionImpl.c | 75 ++++++++++++++++++++++--------- src/client/src/tscProfile.c | 2 +- src/system/detail/src/vnodeRead.c | 3 +- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0f9be90412..7eeb87832a 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3786,10 +3786,11 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, double *min, double *max, double *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = DBL_MAX; - *max = -DBL_MAX; - *minIndex = 0; - *maxIndex = 0; + float fmin = DBL_MAX; + float fmax = -DBL_MAX; + float fminIndex = 0; + float fmaxIndex = 0; + double dsum = 0; assert(numOfRow <= INT16_MAX); @@ -3799,15 +3800,19 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou continue; } - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; + float fv = 0; + *(int32_t*)(&fv) = *(int32_t*)(&(data[i])); + + //*sum += data[i]; + dsum += fv; + if (fmin > fv) { + fmin = fv; + fminIndex = i; } - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + if (fmax < fv) { + fmax = fv; + fmaxIndex = i; } // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { @@ -3819,14 +3824,26 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou // lastVal = data[i]; // } } + + double csum = 0; + *(int64_t*)(&csum) = *(int64_t*)sum; + csum += dsum; + *(int64_t*)(sum) = *(int64_t*)(&csum); + + *(int32_t*)max = *(int32_t*)(&fmax); + *(int32_t*)min = *(int32_t*)(&fmin); + *(int32_t*)minIndex = *(int32_t*)(&fminIndex); + *(int32_t*)maxIndex = *(int32_t*)(&fmaxIndex); + } static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, double *min, double *max, double *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = DBL_MAX; - *max = -DBL_MAX; - *minIndex = 0; - *maxIndex = 0; + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dminIndex = 0; + double dmaxIndex = 0; + double dsum = 0; assert(numOfRow <= INT16_MAX); @@ -3839,15 +3856,19 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do continue; } - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; + double dv = 0; + *(int64_t*)(&dv) = *(int64_t*)(&(data[i])); + + //*sum += data[i]; + dsum += dv; + if (dmin > dv) { + dmin = dv; + dminIndex = i; } - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + if (dmax < dv) { + dmax = dv; + dmaxIndex = i; } // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { @@ -3859,6 +3880,16 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do // lastVal = data[i]; // } } + + double csum = 0; + *(int64_t*)(&csum) = *(int64_t*)sum; + csum += dsum; + *(int64_t*)(sum) = *(int64_t*)(&csum); + + *(int64_t*)max = *(int64_t*)(&dmax); + *(int64_t*)min = *(int64_t*)(&dmin); + *(int64_t*)minIndex = *(int64_t*)(&dminIndex); + *(int64_t*)maxIndex = *(int64_t*)(&dmaxIndex); } void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max, diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 770e61e278..7472a9dac3 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -73,7 +73,7 @@ void tscSaveSlowQuery(SSqlObj *pSql) { const static int64_t SLOW_QUERY_INTERVAL = 3000000L; if (pSql->res.useconds < SLOW_QUERY_INTERVAL) return; - tscTrace("%p query time:%ld sql:%s", pSql, pSql->res.useconds, pSql->sqlstr); + tscTrace("%p query time:%lld sql:%s", pSql, pSql->res.useconds, pSql->sqlstr); char *sql = malloc(200); int len = snprintf(sql, 200, "insert into %s.slowquery values(now, '%s', %lld, %lld, '", tsMonitorDbName, diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 3e580b747f..f35567e720 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -866,7 +866,8 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { pQInfo->pointsRead); if (pQInfo->over == 0) { - dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); + //dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); + dTrace("QInfo:%p set query flag, oldSig:%p", pQInfo, pQInfo->signature); uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); /* From 85c644ae6652d3d189e56a1818d97e94009b6348 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 10:56:53 +0800 Subject: [PATCH 04/69] [TBASE-816] --- CMakeLists.txt | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcad242a30..ff5da13254 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,8 +38,9 @@ IF (NOT DEFINED TD_CLUSTER) # Set macro definitions according to os platform SET(TD_LINUX_64 FALSE) SET(TD_LINUX_32 FALSE) - SET(TD_ARM_64 FALSE) - SET(TD_ARM_32 FALSE) + SET(_TD_ARM_ FALSE) + SET(_TD_ARM_64 FALSE) + SET(_TD_ARM_32 FALSE) SET(TD_MIPS_64 FALSE) SET(TD_DARWIN_64 FALSE) SET(TD_WINDOWS_64 FALSE) @@ -50,9 +51,18 @@ IF (NOT DEFINED TD_CLUSTER) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) ADD_DEFINITIONS(-D_M_X64) MESSAGE(STATUS "The current platform is Linux 64-bit") - ELSE () + ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) + IF (_TD_ARM_) SET(TD_LINUX_32 TRUE) - MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, not supported yet") + SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) + #ADD_DEFINITIONS(-D_M_IX86) + MESSAGE(STATUS "The current platform is Linux 32-bit") + ELSE () + MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet") + EXIT () + ENDIF () + ELSE () + MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet") EXIT () ENDIF () ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") @@ -102,13 +112,27 @@ IF (NOT DEFINED TD_CLUSTER) IF (TD_LINUX_64) SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") - IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + IF (NOT _TD_ARM_) + IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () ADD_DEFINITIONS(-DLINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + ELSEIF (TD_LINUX_32) + IF (NOT _TD_ARM_) + EXIT () + ENDIF () + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ADD_DEFINITIONS(-DLINUX) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + ADD_DEFINITIONS(-DUSE_LIBICONV) ELSEIF (TD_WINDOWS_64) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") @@ -169,6 +193,14 @@ IF (NOT DEFINED TD_CLUSTER) INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") + ELSEIF (TD_LINUX_32) + IF (NOT _TD_ARM_) + EXIT () + ENDIF () + SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") ELSEIF (TD_WINDOWS_64) SET(CMAKE_INSTALL_PREFIX C:/TDengine) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) From 15cb828bea3257d2222acebb02653351aaea38bd Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 11:11:25 +0800 Subject: [PATCH 05/69] [TBASE-816] --- src/client/CMakeLists.txt | 2 +- src/kit/shell/CMakeLists.txt | 2 +- src/kit/taosdemo/CMakeLists.txt | 2 +- src/kit/taosdump/CMakeLists.txt | 2 +- src/modules/http/CMakeLists.txt | 2 +- src/modules/monitor/CMakeLists.txt | 2 +- src/os/linux/CMakeLists.txt | 2 +- src/rpc/CMakeLists.txt | 2 +- src/sdb/CMakeLists.txt | 4 ++-- src/system/detail/CMakeLists.txt | 2 +- src/system/lite/CMakeLists.txt | 2 +- src/util/CMakeLists.txt | 2 +- src/util/src/version.c | 4 ++-- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3bb2f74868..de03125438 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -7,7 +7,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) # set the static lib name diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 20aa0c9d31..96dcec64c8 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) ADD_EXECUTABLE(shell ${SRC}) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 38a28e3079..5384fce62d 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) TARGET_LINK_LIBRARIES(taosdemo taos_static) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index 2bcc020654..49940add9a 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdump ${SRC}) TARGET_LINK_LIBRARIES(taosdump taos_static) diff --git a/src/modules/http/CMakeLists.txt b/src/modules/http/CMakeLists.txt index 911c5a3515..4805271d66 100644 --- a/src/modules/http/CMakeLists.txt +++ b/src/modules/http/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) diff --git a/src/modules/monitor/CMakeLists.txt b/src/modules/monitor/CMakeLists.txt index ff8ddb1c2d..baa5e6218c 100644 --- a/src/modules/monitor/CMakeLists.txt +++ b/src/modules/monitor/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) diff --git a/src/os/linux/CMakeLists.txt b/src/os/linux/CMakeLists.txt index 8a4cc56f72..14e7e06b23 100644 --- a/src/os/linux/CMakeLists.txt +++ b/src/os/linux/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index b3a761e3df..febb3eb83f 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(./src SRC) ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/thaship.c) diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt index 76b407fded..08cdb4baef 100644 --- a/src/sdb/CMakeLists.txt +++ b/src/sdb/CMakeLists.txt @@ -5,11 +5,11 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(sdb ${SRC}) TARGET_LINK_LIBRARIES(sdb trpc) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(sdb sdb_cluster) ENDIF () -ENDIF () \ No newline at end of file +ENDIF () diff --git a/src/system/detail/CMakeLists.txt b/src/system/detail/CMakeLists.txt index 9233d6c71b..16468e7984 100644 --- a/src/system/detail/CMakeLists.txt +++ b/src/system/detail/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) diff --git a/src/system/lite/CMakeLists.txt b/src/system/lite/CMakeLists.txt index 965f7666b0..e9257b2388 100644 --- a/src/system/lite/CMakeLists.txt +++ b/src/system/lite/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 7e54759f7e..b36ae3a4b5 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -IF (TD_LINUX_64) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread os m rt) diff --git a/src/util/src/version.c b/src/util/src/version.c index dc2c3c5480..96e7ad4ead 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ char version[64] = "1.6.4.0"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "869171d2331eb25ba0901e88d33ae627bf5a9d91"; -char buildinfo[512] = "Built by ubuntu at 2019-11-07 22:31"; +char gitinfo[128] = "d04354a8ac2f7dd9ba521d755e5d484a203783d9"; +char buildinfo[512] = "Built by root at 2019-11-11 10:23"; From 7ed580d93b6de7f248d8db3ae6b5f6de80a8b1a8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 11:55:49 +0800 Subject: [PATCH 06/69] fix some memory leaks --- src/system/detail/src/mgmtMeter.c | 1 + src/system/detail/src/vnodeQueryImpl.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index bccf9a06c8..2692e39131 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -833,6 +833,7 @@ static void removeMeterFromMetricIndex(STabObj *pMetric, STabObj *pMeter) { } } + tSkipListDestroyKey(&key); if (num != 0) { free(pRes); } diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 187269d30b..9c0d7883a5 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -3436,9 +3436,18 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // only the function of interp needs the corresponding information + if (pCtx->functionId != TSDB_FUNC_INTERP) { + continue; + } + + pCtx->numOfParams = 4; + SInterpInfo * pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; // for primary timestamp column, set the flag From 5400e49c7ede3a2d60eaa4b3adc39a0fdfb29370 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 11:58:20 +0800 Subject: [PATCH 07/69] refactor some code, fix the used time too big bug. --- src/client/src/tscServer.c | 4 ++-- src/inc/ttimer.h | 2 +- src/system/detail/src/vnodeShell.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 73e612962a..49368e4d17 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2842,7 +2842,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql) { return msgLen; } -int tscEstimateBuildHeartBeatMsgLength(SSqlObj *pSql) { +int tscEstimateHeartBeatMsgLength(SSqlObj *pSql) { int size = 0; STscObj *pObj = pSql->pTscObj; @@ -2875,7 +2875,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql) { pthread_mutex_lock(&pObj->mutex); - size = tscEstimateBuildHeartBeatMsgLength(pSql); + size = tscEstimateHeartBeatMsgLength(pSql); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for heartbeat msg", pSql); return -1; diff --git a/src/inc/ttimer.h b/src/inc/ttimer.h index ddfd3f1fdf..5424881e26 100644 --- a/src/inc/ttimer.h +++ b/src/inc/ttimer.h @@ -41,7 +41,7 @@ extern int taosTmrThreads; tprintf("TMR ", tmrDebugFlag, __VA_ARGS__); \ } } while(0) -#define MAX_NUM_OF_TMRCTL 32 +#define MAX_NUM_OF_TMRCTL 512 #define MSECONDS_PER_TICK 5 void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 190c1d6ee6..353b2668ee 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -419,7 +419,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { if (code == TSDB_CODE_SUCCESS) { pRsp->offset = htobe64(vnodeGetOffsetVal(pRetrieve->qhandle)); - pRsp->useconds = ((SQInfo *)(pRetrieve->qhandle))->useconds; + pRsp->useconds = htobe64(((SQInfo *)(pRetrieve->qhandle))->useconds); } else { pRsp->offset = 0; pRsp->useconds = 0; From 1f78d9b859c26daf5127bae0c790ff83208af38f Mon Sep 17 00:00:00 2001 From: localvar Date: Mon, 11 Nov 2019 03:28:55 +0000 Subject: [PATCH 08/69] make(rollback) max timer controller number to 512 and make it configurable. --- src/inc/tglobalcfg.h | 2 ++ src/inc/ttimer.h | 2 +- src/util/src/tglobalcfg.c | 5 +++++ src/util/src/ttimer.c | 21 ++++++++++++++++----- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index e83c89593f..026d48ba08 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -169,6 +169,8 @@ extern uint32_t debugFlag; extern uint32_t odbcdebugFlag; extern uint32_t qdebugFlag; +extern uint32_t taosMaxTmrCtrl; + extern int tsRpcTimer; extern int tsRpcMaxTime; extern int tsUdpDelay; diff --git a/src/inc/ttimer.h b/src/inc/ttimer.h index 5424881e26..b9bbbb04dd 100644 --- a/src/inc/ttimer.h +++ b/src/inc/ttimer.h @@ -25,6 +25,7 @@ typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern uint32_t tmrDebugFlag; extern int taosTmrThreads; +extern uint32_t taosMaxTmrCtrl; #define tmrError(...) \ do { if (tmrDebugFlag & DEBUG_ERROR) { \ @@ -41,7 +42,6 @@ extern int taosTmrThreads; tprintf("TMR ", tmrDebugFlag, __VA_ARGS__); \ } } while(0) -#define MAX_NUM_OF_TMRCTL 512 #define MSECONDS_PER_TICK 5 void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 9abe60220f..586f329001 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -536,6 +536,11 @@ void tsInitGlobalConfig() { 0, 2, 0, TSDB_CFG_UTYPE_NONE); // 0-any, 1-mgmt, 2-dnode + // timer + tsInitConfigOption(cfg++, "maxTmrCtrl", &taosMaxTmrCtrl, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 8, 2048, 0, TSDB_CFG_UTYPE_NONE); + // time tsInitConfigOption(cfg++, "monitorInterval", &tsMonitorInterval, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 798e63a64a..e6d98a3734 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -82,13 +82,16 @@ typedef struct time_wheel_t { } time_wheel_t; uint32_t tmrDebugFlag = DEBUG_ERROR | DEBUG_WARN | DEBUG_FILE; +uint32_t taosMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tmrCtrlMutex; -static tmr_ctrl_t tmrCtrls[MAX_NUM_OF_TMRCTL]; +static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; -void* tmrQhandle; -int taosTmrThreads = 1; +static void* tmrQhandle; +static int numOfTmrCtrl = 0; + +int taosTmrThreads = 1; static uintptr_t nextTimerId = 0; @@ -481,7 +484,13 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, } static void taosTmrModuleInit(void) { - for (int i = 0; i < tListLen(tmrCtrls) - 1; ++i) { + tmrCtrls = malloc(sizeof(tmr_ctrl_t) * taosMaxTmrCtrl); + if (tmrCtrls == NULL) { + tmrError("failed to allocate memory for timer controllers."); + return; + } + + for (int i = 0; i < taosMaxTmrCtrl - 1; ++i) { tmr_ctrl_t* ctrl = tmrCtrls + i; ctrl->next = ctrl + 1; } @@ -526,6 +535,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab tmr_ctrl_t* ctrl = unusedTmrCtrl; if (ctrl != NULL) { unusedTmrCtrl = ctrl->next; + numOfTmrCtrl++; } pthread_mutex_unlock(&tmrCtrlMutex); @@ -536,7 +546,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab strncpy(ctrl->label, label, sizeof(ctrl->label)); ctrl->label[sizeof(ctrl->label) - 1] = 0; - tmrTrace("timer controller[label=%s] is initialized.", label); + tmrTrace("timer controller[label=%s] is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); return ctrl; } @@ -549,6 +559,7 @@ void taosTmrCleanUp(void* handle) { pthread_mutex_lock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; + numOfTmrCtrl--; unusedTmrCtrl = ctrl; pthread_mutex_unlock(&tmrCtrlMutex); } From 48c60fd01a86373e5c656186c5235b22763bfa88 Mon Sep 17 00:00:00 2001 From: localvar Date: Mon, 11 Nov 2019 04:08:14 +0000 Subject: [PATCH 09/69] update timer log format --- src/util/src/ttimer.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index e6d98a3734..97e2189153 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -132,7 +132,7 @@ static void unlockTimerList(timer_list_t* list) { int64_t tid = taosGetPthreadId(); if (__sync_val_compare_and_swap_64(&(list->lockedBy), tid, 0) != tid) { assert(false); - tmrError("trying to unlock a timer list not locked by current thread."); + tmrError("%d trying to unlock a timer list not locked by current thread.", tid); } } @@ -259,13 +259,13 @@ static void processExpiredTimer(void* handle, void* arg) { timer->executedBy = taosGetPthreadId(); uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] execution start."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] execution start."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); (*timer->fp)(timer->param, (tmr_h)timer->id); atomic_store_8(&timer->state, TIMER_STATE_STOPPED); - fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] execution end."; + fmt = "%s timer[id=%lld, fp=%p, param=%p] execution end."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } removeTimer(timer->id); @@ -273,18 +273,21 @@ static void processExpiredTimer(void* handle, void* arg) { } static void addToExpired(tmr_obj_t* head) { - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] expired"; + const char* fmt = "%s adding expired timer[id=%lld, fp=%p, param=%p] to queue."; while (head != NULL) { - tmrTrace(fmt, head->ctrl->label, head->id, head->fp, head->param); - + uintptr_t id = head->id; tmr_obj_t* next = head->next; + tmrTrace(fmt, head->ctrl->label, id, head->fp, head->param); + SSchedMsg schedMsg; schedMsg.fp = NULL; schedMsg.tfp = processExpiredTimer; schedMsg.ahandle = head; schedMsg.thandle = NULL; taosScheduleTask(tmrQhandle, &schedMsg); + + tmrTrace("timer[id=%lld] has been added to queue.", id); head = next; } } @@ -298,7 +301,7 @@ static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int msecon timer->ctrl = ctrl; addTimer(timer); - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] started"; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] started"; tmrTrace(fmt, ctrl->label, timer->id, timer->fp, timer->param); if (mseconds == 0) { @@ -321,7 +324,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle tmr_obj_t* timer = (tmr_obj_t*)calloc(1, sizeof(tmr_obj_t)); if (timer == NULL) { - tmrError("failed to allocated memory for new timer object."); + tmrError("%s failed to allocated memory for new timer object.", ctrl->label); return NULL; } @@ -392,7 +395,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { // we cannot guarantee the thread safety of the timr in all other cases. reusable = true; } - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] is cancelled."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] is cancelled."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } else if (state != TIMER_STATE_EXPIRED) { // timer already stopped or cancelled, has nothing to do in this case @@ -403,7 +406,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { } else { assert(timer->executedBy != taosGetPthreadId()); - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] fired, waiting..."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] fired, waiting..."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); for (int i = 1; atomic_load_8(&timer->state) != TIMER_STATE_STOPPED; i++) { @@ -412,7 +415,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { } } - fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] stopped."; + fmt = "%s timer[id=%lld, fp=%p, param=%p] stopped."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } @@ -451,7 +454,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, bool stopped = false; tmr_obj_t* timer = findTimer(id); if (timer == NULL) { - tmrTrace("timer[id=%lld] does not exist", id); + tmrTrace("%s timer[id=%lld] does not exist", ctrl->label, id); } else { uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); if (!doStopTimer(timer, state)) { @@ -466,7 +469,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, return stopped; } - tmrTrace("timer[id=%lld] is reused", timer->id); + tmrTrace("%s timer[id=%lld] is reused", ctrl->label, timer->id); // wait until there's no other reference to this timer, // so that we can reuse this timer safely. @@ -540,13 +543,13 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab pthread_mutex_unlock(&tmrCtrlMutex); if (ctrl == NULL) { - tmrError("too many timer controllers, failed to create timer controller[label=%s].", label); + tmrError("%s too many timer controllers, failed to create timer controller.", label); return NULL; } strncpy(ctrl->label, label, sizeof(ctrl->label)); ctrl->label[sizeof(ctrl->label) - 1] = 0; - tmrTrace("timer controller[label=%s] is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); + tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); return ctrl; } @@ -554,7 +557,7 @@ void taosTmrCleanUp(void* handle) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; assert(ctrl != NULL && ctrl->label[0] != 0); - tmrTrace("timer controller[label=%s] is cleaned up.", ctrl->label); + tmrTrace("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; pthread_mutex_lock(&tmrCtrlMutex); From db26cf1e8e54b9d6f7cf22324603453d33591d69 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Mon, 11 Nov 2019 12:39:48 +0800 Subject: [PATCH 10/69] [TBASE-1047] Go connector prints all the SQL. This will harm the performance and consume unnecessary hard disk space. Modified to print only the SQLs which returns error. --- src/connector/go/src/taosSql/taosSqlCgo.go | 57 +++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/connector/go/src/taosSql/taosSqlCgo.go b/src/connector/go/src/taosSql/taosSqlCgo.go index e240ac44ec..fcef14045f 100755 --- a/src/connector/go/src/taosSql/taosSqlCgo.go +++ b/src/connector/go/src/taosSql/taosSqlCgo.go @@ -29,46 +29,47 @@ import ( "unsafe" ) -func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error){ +func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) { cuser := C.CString(user) cpass := C.CString(pass) - cip := C.CString(ip) - cdb := C.CString(db) + cip := C.CString(ip) + cdb := C.CString(db) defer C.free(unsafe.Pointer(cip)) defer C.free(unsafe.Pointer(cuser)) defer C.free(unsafe.Pointer(cpass)) defer C.free(unsafe.Pointer(cdb)) taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.int)(port)) - if taosObj == nil { - return nil, errors.New("taos_connect() fail!") - } + if taosObj == nil { + return nil, errors.New("taos_connect() fail!") + } - return (unsafe.Pointer)(taosObj), nil -} + return (unsafe.Pointer)(taosObj), nil +} func (mc *taosConn) taosQuery(sqlstr string) (int, error) { - taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) + //taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - csqlstr := C.CString(sqlstr) + csqlstr := C.CString(sqlstr) defer C.free(unsafe.Pointer(csqlstr)) - code := int(C.taos_query(mc.taos, csqlstr)) + code := int(C.taos_query(mc.taos, csqlstr)) - if 0 != code { - mc.taos_error() - errStr := C.GoString(C.taos_errstr(mc.taos)) - taosLog.Println("taos_query() failed:", errStr) - return 0, errors.New(errStr) - } + if 0 != code { + mc.taos_error() + errStr := C.GoString(C.taos_errstr(mc.taos)) + taosLog.Println("taos_query() failed:", errStr) + taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) + return 0, errors.New(errStr) + } - // read result and save into mc struct - num_fields := int(C.taos_field_count(mc.taos)) - if 0 == num_fields { // there are no select and show kinds of commands - mc.affectedRows = int(C.taos_affected_rows(mc.taos)) - mc.insertId = 0 - } + // read result and save into mc struct + num_fields := int(C.taos_field_count(mc.taos)) + if 0 == num_fields { // there are no select and show kinds of commands + mc.affectedRows = int(C.taos_affected_rows(mc.taos)) + mc.insertId = 0 + } - return num_fields, nil + return num_fields, nil } func (mc *taosConn) taos_close() { @@ -76,8 +77,8 @@ func (mc *taosConn) taos_close() { } func (mc *taosConn) taos_error() { - // free local resouce: allocated memory/metric-meta refcnt - //var pRes unsafe.Pointer - pRes := C.taos_use_result(mc.taos) - C.taos_free_result(pRes) + // free local resouce: allocated memory/metric-meta refcnt + //var pRes unsafe.Pointer + pRes := C.taos_use_result(mc.taos) + C.taos_free_result(pRes) } From 2a11363e0c57af6e4060a0072283cd1736259078 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 11 Nov 2019 13:04:09 +0800 Subject: [PATCH 11/69] [TBASE-1060] --- src/client/src/tscProfile.c | 45 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 770e61e278..b6de98e712 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -23,6 +23,27 @@ #include "ttimer.h" #include "tutil.h" +void tscSaveSlowQueryFp(void *handle, void *tmrId); +void *tscSlowQueryConn = NULL; +bool tscSlowQueryConnInitialized = false; +TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, int port, void (*fp)(void *, TAOS_RES *, int), + void *param, void **taos); + +void tscInitConnCb(void *param, TAOS_RES *result, int code) { + char *sql = param; + if (code < 0) { + tscError("taos:%p, slow query connect failed, code:%d", tscSlowQueryConn, code); + taos_close(tscSlowQueryConn); + tscSlowQueryConn = NULL; + tscSlowQueryConnInitialized = false; + free(sql); + } else { + tscTrace("taos:%p, slow query connect success, code:%d", tscSlowQueryConn, code); + tscSlowQueryConnInitialized = true; + tscSaveSlowQueryFp(sql, NULL); + } +} + void tscAddIntoSqlList(SSqlObj *pSql) { static uint32_t queryId = 1; @@ -47,26 +68,28 @@ void tscAddIntoSqlList(SSqlObj *pSql) { void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { if (code < 0) { - tscError("failed to save slowquery, code:%d", code); + tscError("failed to save slow query, code:%d", code); + } else { + tscTrace("success to save slow query, code:%d", code); } } void tscSaveSlowQueryFp(void *handle, void *tmrId) { char *sql = handle; - static void *taos = NULL; - if (taos == NULL) { - taos = taos_connect(NULL, "monitor", tsInternalPass, NULL, 0); - if (taos == NULL) { - tscError("failed to save slow query, can't connect to server"); + if (!tscSlowQueryConnInitialized) { + if (tscSlowQueryConn == NULL) { + tscTrace("start to init slow query connect"); + taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, tscInitConnCb, sql, &tscSlowQueryConn); + } else { + tscError("taos:%p, slow query connect is already initialized", tscSlowQueryConn); free(sql); - return; } + } else { + tscTrace("taos:%p, save slow query:%s", tscSlowQueryConn, sql); + taos_query_a(tscSlowQueryConn, sql, tscSaveSlowQueryFpCb, NULL); + free(sql); } - - tscTrace("save slow query:sql", sql); - taos_query_a(taos, sql, tscSaveSlowQueryFpCb, NULL); - free(sql); } void tscSaveSlowQuery(SSqlObj *pSql) { From 1aa5f927f374186b3883f2560ba24ab5373d54ae Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 13:08:50 +0800 Subject: [PATCH 12/69] [TBASE-1098]fix failed to drop a db, of which name start with "log" issue #686 --- src/inc/tutil.h | 2 -- src/system/detail/inc/mgmtUtil.h | 1 + src/system/detail/src/mgmtDb.c | 16 +++++++++++++--- src/system/detail/src/mgmtMeter.c | 10 ++++++---- src/system/detail/src/mgmtUtil.c | 7 +++++++ src/util/src/tutil.c | 7 ------- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/inc/tutil.h b/src/inc/tutil.h index f63f8b6cec..884e448aae 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -169,8 +169,6 @@ int32_t taosInitTimer(void (*callback)(int), int32_t ms); */ uint32_t MurmurHash3_32(const void *key, int32_t len); -bool taosCheckDbName(char *db, char *monitordb); - bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len); bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); diff --git a/src/system/detail/inc/mgmtUtil.h b/src/system/detail/inc/mgmtUtil.h index aecb229dba..6a6b819bf2 100644 --- a/src/system/detail/inc/mgmtUtil.h +++ b/src/system/detail/inc/mgmtUtil.h @@ -30,6 +30,7 @@ char* mgmtMeterGetTag(STabObj* pMeter, int32_t col, SSchema* pTagColSchema); int32_t mgmtFindTagCol(STabObj * pMetric, const char * tagName); int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col); +bool mgmtCheckIsMonitorDB(char *db, char *monitordb); int32_t mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pInfo, int32_t tableIndex, tQueryResultset* pRes); int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes); diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index ae4c5bed7c..93ff1279c0 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -14,9 +14,11 @@ */ #define _DEFAULT_SOURCE +#include "os.h" + #include "mgmt.h" -#include #include "mgmtBalance.h" +#include "mgmtUtil.h" #include "tschemautil.h" void *dbSdb = NULL; @@ -373,10 +375,18 @@ int mgmtDropDbByName(SAcctObj *pAcct, char *name) { if (pDb == NULL) { mWarn("db:%s is not there", name); // return TSDB_CODE_INVALID_DB; - return 0; + return TSDB_CODE_SUCCESS; } - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; +// char dbName[TSDB_DB_NAME_LEN + 1] = {0}; +// extractDBName(pDb->name, dbName); +// if (strncasecmp(dbName, tsMonitorDbName, strlen(dbName)) == 0) { +// return TSDB_CODE_MONITOR_DB_FORBEIDDEN; +// } + + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + } return mgmtDropDb(pDb); } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 2692e39131..2db920fdd7 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -688,8 +688,10 @@ int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore) { pAcct = mgmtGetAcct(pDb->cfg.acct); - // 0.sys - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + // 0.log + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + } if (mgmtIsNormalMeter(pMeter)) { return dropMeterImp(pDb, pMeter, pAcct); @@ -719,8 +721,8 @@ int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter) { return TSDB_CODE_INVALID_TABLE; } - // 0.sys - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + // 0.log + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { if (!mgmtIsNormalMeter(pMeter) || !mgmtMeterCreateFromMetric(pMeter)) { diff --git a/src/system/detail/src/mgmtUtil.c b/src/system/detail/src/mgmtUtil.c index a427771e0f..be90180f68 100644 --- a/src/system/detail/src/mgmtUtil.c +++ b/src/system/detail/src/mgmtUtil.c @@ -87,3 +87,10 @@ int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col) { // length before col return len; } + +bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { + char dbName[TSDB_DB_NAME_LEN + 1] = {0}; + extractDBName(db, dbName); + + return (strncasecmp(dbName, monitordb, strlen(dbName)) == 0); +} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 805f27a381..0d1ea9ed50 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -406,13 +406,6 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP return rename(fullPath, *dstPath); } -bool taosCheckDbName(char *db, char *monitordb) { - char *pos = strchr(db, '.'); - if (pos == NULL) return false; - - return strncasecmp(pos + 1, monitordb, strlen(monitordb)) == 0; -} - bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef USE_LIBICONV iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); From d609fe319aff0b70aa48ebd31b9202542f8c7597 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 13:11:36 +0800 Subject: [PATCH 13/69] remove unused code --- src/system/detail/src/mgmtDb.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index 93ff1279c0..e87266aa4b 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -378,12 +378,6 @@ int mgmtDropDbByName(SAcctObj *pAcct, char *name) { return TSDB_CODE_SUCCESS; } -// char dbName[TSDB_DB_NAME_LEN + 1] = {0}; -// extractDBName(pDb->name, dbName); -// if (strncasecmp(dbName, tsMonitorDbName, strlen(dbName)) == 0) { -// return TSDB_CODE_MONITOR_DB_FORBEIDDEN; -// } - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { return TSDB_CODE_MONITOR_DB_FORBEIDDEN; } From ee112120ffe39f3260a57e5776e573fd095e163f Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 14:18:31 +0800 Subject: [PATCH 14/69] [TBASE-816] --- CMakeLists.txt | 18 +++++++++++------- src/client/CMakeLists.txt | 2 +- src/kit/shell/CMakeLists.txt | 2 +- src/kit/taosdemo/CMakeLists.txt | 2 +- src/kit/taosdump/CMakeLists.txt | 2 +- src/modules/http/CMakeLists.txt | 2 +- src/modules/monitor/CMakeLists.txt | 2 +- src/os/linux/CMakeLists.txt | 2 +- src/rpc/CMakeLists.txt | 2 +- src/sdb/CMakeLists.txt | 2 +- src/system/detail/CMakeLists.txt | 2 +- src/system/lite/CMakeLists.txt | 2 +- src/util/CMakeLists.txt | 2 +- src/util/src/tcrc32c.c | 2 +- 14 files changed, 24 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ff5da13254..9a3a406cc3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,12 +38,16 @@ IF (NOT DEFINED TD_CLUSTER) # Set macro definitions according to os platform SET(TD_LINUX_64 FALSE) SET(TD_LINUX_32 FALSE) - SET(_TD_ARM_ FALSE) - SET(_TD_ARM_64 FALSE) - SET(_TD_ARM_32 FALSE) + SET(TD_ARM FALSE) + SET(TD_ARM_64 FALSE) + SET(TD_ARM_32 FALSE) SET(TD_MIPS_64 FALSE) SET(TD_DARWIN_64 FALSE) SET(TD_WINDOWS_64 FALSE) + + IF (TD_ARM) + ADD_DEFINITIONS(-D_TD_ARM_) + ENDIF () IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) @@ -52,7 +56,7 @@ IF (NOT DEFINED TD_CLUSTER) ADD_DEFINITIONS(-D_M_X64) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) - IF (_TD_ARM_) + IF (TD_ARM) SET(TD_LINUX_32 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) #ADD_DEFINITIONS(-D_M_IX86) @@ -112,7 +116,7 @@ IF (NOT DEFINED TD_CLUSTER) IF (TD_LINUX_64) SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") - IF (NOT _TD_ARM_) + IF (NOT TD_ARM) IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ELSE () @@ -124,7 +128,7 @@ IF (NOT DEFINED TD_CLUSTER) ADD_DEFINITIONS(-DLINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ELSEIF (TD_LINUX_32) - IF (NOT _TD_ARM_) + IF (NOT TD_ARM) EXIT () ENDIF () SET(DEBUG_FLAGS "-O0 -DDEBUG") @@ -194,7 +198,7 @@ IF (NOT DEFINED TD_CLUSTER) INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") ELSEIF (TD_LINUX_32) - IF (NOT _TD_ARM_) + IF (NOT TD_ARM) EXIT () ENDIF () SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index de03125438..d3baf84d67 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -7,7 +7,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) # set the static lib name diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 96dcec64c8..7442367e91 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) ADD_EXECUTABLE(shell ${SRC}) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 5384fce62d..599875fa60 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) TARGET_LINK_LIBRARIES(taosdemo taos_static) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index 49940add9a..76b40d1c2d 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdump ${SRC}) TARGET_LINK_LIBRARIES(taosdump taos_static) diff --git a/src/modules/http/CMakeLists.txt b/src/modules/http/CMakeLists.txt index 4805271d66..d8dccb526a 100644 --- a/src/modules/http/CMakeLists.txt +++ b/src/modules/http/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) diff --git a/src/modules/monitor/CMakeLists.txt b/src/modules/monitor/CMakeLists.txt index baa5e6218c..44ac8aae0b 100644 --- a/src/modules/monitor/CMakeLists.txt +++ b/src/modules/monitor/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) diff --git a/src/os/linux/CMakeLists.txt b/src/os/linux/CMakeLists.txt index 14e7e06b23..a702cac759 100644 --- a/src/os/linux/CMakeLists.txt +++ b/src/os/linux/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index febb3eb83f..8d54d5fda5 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(./src SRC) ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/thaship.c) diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt index 08cdb4baef..6b29eefde6 100644 --- a/src/sdb/CMakeLists.txt +++ b/src/sdb/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(sdb ${SRC}) TARGET_LINK_LIBRARIES(sdb trpc) diff --git a/src/system/detail/CMakeLists.txt b/src/system/detail/CMakeLists.txt index 16468e7984..95cce3dfe6 100644 --- a/src/system/detail/CMakeLists.txt +++ b/src/system/detail/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) diff --git a/src/system/lite/CMakeLists.txt b/src/system/lite/CMakeLists.txt index e9257b2388..a22ed60563 100644 --- a/src/system/lite/CMakeLists.txt +++ b/src/system/lite/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index b36ae3a4b5..e73428353e 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND _TD_ARM_)) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread os m rt) diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 88841bcbc9..705ca5872a 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -1353,7 +1353,7 @@ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { #endif // #ifndef _TD_ARM_ void taosResolveCRC() { -#ifndef _TD_ARM_32 +#ifndef _TD_ARM_ int sse42; SSE42(sse42); crc32c = sse42 ? crc32c_hw : crc32c_sf; From 96289dc358bc7fc6ae9af47ba93d5502c849be90 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 14:44:10 +0800 Subject: [PATCH 15/69] [TBASE-816] --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a3a406cc3..fd361ae974 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,9 @@ IF (NOT DEFINED TD_CLUSTER) IF (TD_ARM) ADD_DEFINITIONS(-D_TD_ARM_) + IF (TD_ARM_32) + ADD_DEFINITIONS(-D_TD_ARM_32_) + ENDIF () ENDIF () IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") From 27a508b7f960f8488fdda21c252edc7c30c1f82c Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 14:49:18 +0800 Subject: [PATCH 16/69] [TBASE-816] --- src/kit/shell/src/shellEngine.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 324dde55d1..12c844d00d 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -445,7 +445,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BIGINT: printf("%*lld|", l[i], *((int64_t *)row[i])); break; - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ float fv = 0; //memcpy(&fv, row[i], sizeof(float)); @@ -454,8 +454,9 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else printf("%*.5f|", l[i], *((float *)row[i])); #endif + } break; - case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DOUBLE: { #ifdef _TD_ARM_32_ double dv = 0; //memcpy(&dv, row[i], sizeof(double)); @@ -464,6 +465,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else printf("%*.9f|", l[i], *((double *)row[i])); #endif + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -528,7 +530,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BIGINT: printf("%lld\n", *((int64_t *)row[i])); break; - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ float fv = 0; //memcpy(&fv, row[i], sizeof(float)); @@ -537,8 +539,9 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else printf("%.5f\n", *((float *)row[i])); #endif + } break; - case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DOUBLE: { #ifdef _TD_ARM_32_ double dv = 0; //memcpy(&dv, row[i], sizeof(double)); @@ -547,7 +550,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else printf("%.9f\n", *((double *)row[i])); #endif - break; + } + break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); @@ -614,7 +618,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BIGINT: fprintf(fp, "%lld", *((int64_t *)row[i])); break; - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ float fv = 0; //memcpy(&fv, row[i], sizeof(float)); @@ -623,8 +627,9 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else fprintf(fp, "%.5f", *((float *)row[i])); #endif + } break; - case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DOUBLE: { #ifdef _TD_ARM_32_ double dv = 0; //memcpy(&dv, row[i], sizeof(double)); @@ -633,6 +638,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { #else fprintf(fp, "%.9f", *((double *)row[i])); #endif + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: From e521288582012559dd54fc825512663847a90cd9 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 15:43:25 +0800 Subject: [PATCH 17/69] [TBASE-816] --- CMakeLists.txt | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fd361ae974..2daa360f2f 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,12 +44,30 @@ IF (NOT DEFINED TD_CLUSTER) SET(TD_MIPS_64 FALSE) SET(TD_DARWIN_64 FALSE) SET(TD_WINDOWS_64 FALSE) - + + # if generate ARM version: + # cmake -DARMVER=arm32 .. or cmake -DARMVER=arm64 + IF (${ARMVER} MATCHES "arm32") + SET(TD_ARM TRUE) + SET(TD_ARM_32 TRUE) + ADD_DEFINITIONS(-D_TD_ARM_) + ADD_DEFINITIONS(-D_TD_ARM_32_) + ELSEIF (${ARMVER} MATCHES "arm64") + SET(TD_ARM TRUE) + SET(TD_ARM_64 TRUE) + ADD_DEFINITIONS(-D_TD_ARM_) + ADD_DEFINITIONS(-D_TD_ARM_64_) + ENDIF () + IF (TD_ARM) ADD_DEFINITIONS(-D_TD_ARM_) IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_32_) - ENDIF () + ELSEIF (TD_ARM_64) + ADD_DEFINITIONS(-D_TD_ARM_64_) + ELSE () + EXIT () + ENDIF ENDIF () IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") From acce71dbf3620db5bf6dc54130a73a3cdabe38f6 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 16:08:49 +0800 Subject: [PATCH 18/69] [TBASE-1099] fix bug referred in issue #699, refactor some codes --- src/client/inc/tsclient.h | 1 + src/client/src/TSDBJNIConnector.c | 16 +++++++++------- src/client/src/tscAst.c | 9 ++++----- src/client/src/tscUtil.c | 14 ++++++++++++++ .../java/com/taosdata/jdbc/TSDBJNIConnector.java | 2 +- src/inc/taos.h | 3 --- src/inc/tutil.h | 2 +- src/system/detail/src/mgmtShell.c | 3 +-- 8 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 0d2c7dace3..44bfc978a3 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -479,6 +479,7 @@ void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql); void tscKillMetricQuery(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); int32_t tscBuildResultsForEmptyRetrieval(SSqlObj *pSql); +bool tscIsUpdateQuery(STscObj *pObj); // transfer SSqlInfo to SqlCmd struct int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index e98acfc9a6..d61621f9db 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -289,15 +289,17 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( return JNI_CONNECTION_NULL; } - int num_fields = taos_field_count(tscon); - if (num_fields != 0) { - jlong ret = (jlong)taos_use_result(tscon); - jniTrace("jobj:%p, taos:%p, get resultset:%p", jobj, tscon, (void *)ret); - return ret; + jlong ret = 0; + + if (tscIsUpdateQuery(tscon)) { + ret = 0; // for update query, no result pointer + jniTrace("jobj:%p, taos:%p, no result", jobj, tscon); + } else { + ret = (jlong) taos_use_result(tscon); + jniTrace("jobj:%p, taos:%p, get resultset:%p", jobj, tscon, (void *) ret); } - jniTrace("jobj:%p, taos:%p, no resultset", jobj, tscon); - return 0; + return ret; } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index fe0921c64d..527b1345e2 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -255,8 +255,7 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0 || t0.type == TK_RP) { - if (pLeft->nodeType != TSQL_NODE_EXPR) { - // if left is not the expr, it is not a legal expr + if (pLeft->nodeType != TSQL_NODE_EXPR) { // if left is not the expr, it is not a legal expr tSQLSyntaxNodeDestroy(pLeft, NULL); return NULL; } @@ -320,13 +319,13 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha pn->colId = -1; return pn; } else { - int32_t optr = getBinaryExprOptr(&t0); - if (optr <= 0) { + uint8_t localOptr = getBinaryExprOptr(&t0); + if (localOptr <= 0) { pError("not support binary operator:%d", t0.type); return NULL; } - return parseRemainStr(str, pBinExpr, pSchema, optr, numOfCols, i); + return parseRemainStr(str, pBinExpr, pSchema, localOptr, numOfCols, i); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5793656477..9a699b7bb0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1817,3 +1817,17 @@ int16_t tscGetJoinTagColIndexByUid(SSqlCmd* pCmd, uint64_t uid) { return pTagCond->joinInfo.right.tagCol; } } + +bool tscIsUpdateQuery(STscObj* pObj) { + if (pObj == NULL || pObj->signature != pObj) { + globalCode = TSDB_CODE_DISCONNECTED; + return TSDB_CODE_DISCONNECTED; + } + + SSqlCmd* pCmd = &(pObj->pSql->cmd); + if (pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) { + return 1; + } + + return 0; +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index c66a8322c1..3adb601822 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -134,7 +134,7 @@ public class TSDBJNIConnector { } } - // Try retrieving result set for the executed SQLusing the current connection pointer. If the executed + // Try retrieving result set for the executed SQL using the current connection pointer. If the executed // SQL is a DML/DDL which doesn't return a result set, then taosResultSetPointer should be 0L. Otherwise, // taosResultSetPointer should be a non-zero value. taosResultSetPointer = this.getResultSetImp(this.taos); diff --git a/src/inc/taos.h b/src/inc/taos.h index 94e99d582c..05acc91551 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -122,9 +122,6 @@ void taos_close_stream(TAOS_STREAM *tstr); int taos_load_table_info(TAOS *taos, const char* tableNameList); -// TODO: `configDir` should not be declared here -extern char configDir[]; // the path to global configuration - #ifdef __cplusplus } #endif diff --git a/src/inc/tutil.h b/src/inc/tutil.h index 884e448aae..7725bd2d1b 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -89,7 +89,7 @@ extern "C" { } else { \ return (x) < (y) ? -1 : 1; \ } \ - } while (0); + } while (0) #define GET_INT8_VAL(x) (*(int8_t *)(x)) #define GET_INT16_VAL(x) (*(int16_t *)(x)) diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 44b9ff286c..43b002992d 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1140,7 +1140,7 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { SAcctObj * pAcct = NULL; SUserObj * pUser = NULL; SDbObj * pDb = NULL; - char dbName[TSDB_METER_ID_LEN]; + char dbName[256] = {0}; pConnectMsg = (SConnectMsg *)pMsg; @@ -1158,7 +1158,6 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { pAcct = mgmtGetAcct(pUser->acct); if (pConnectMsg->db[0]) { - memset(dbName, 0, sizeof(dbName)); sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); pDb = mgmtGetDb(dbName); if (pDb == NULL) { From f740e8bc1ed943cae353de9cd9c52214f05fb223 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 11 Nov 2019 16:30:21 +0800 Subject: [PATCH 19/69] [TBASE-816] --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2daa360f2f..c89195a89a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,7 @@ IF (NOT DEFINED TD_CLUSTER) ADD_DEFINITIONS(-D_TD_ARM_64_) ELSE () EXIT () - ENDIF + ENDIF () ENDIF () IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") From 1ebd80ea69761b4137f62cba186caca1cf5f1c22 Mon Sep 17 00:00:00 2001 From: fangpanpan <53251079+fangpanpan@users.noreply.github.com> Date: Mon, 11 Nov 2019 16:38:32 +0800 Subject: [PATCH 20/69] Update connector-ch.md --- documentation/webdocs/markdowndocs/connector-ch.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index a8ceaa848f..4d1701d848 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -269,13 +269,14 @@ public Connection getConn() throws Exception{ 用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装： -​ `pip install src/connector/python/python2/` +​ `pip install src/connector/python/[linux|windows]/python2/` 或 -​ `pip install src/connector/python/python3/` +​ `pip install src/connector/python/[linux|windows]/python3/` 如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。 +对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。所有TDengine的连接器，均需依赖taos.dll。 ### Python客户端接口 From 1ca75c5a077d52425e0be7bd27927d13614fee35 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 11 Nov 2019 20:22:28 +0800 Subject: [PATCH 21/69] fix bugs in package script --- packaging/tools/install.sh | 4 +-- src/client/src/TSDBJNIConnector.c | 47 +++++++++++++++++++------------ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 6b5af3f9ef..acd6c6ccce 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -247,9 +247,9 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/taosd -V | cut -d ' ' -f 1) + curr_version=$(${bin_dir}/taosd -V | cut -d ' ' -f 2) - min_compatible_version=$(${script_dir}/bin/taosd -V | cut -d ' ' -f 2) + min_compatible_version=$(${script_dir}/bin/taosd -V | cut -d ' ' -f 4) vercomp $curr_version $min_compatible_version case $? in diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index fb175618c0..a15a6c0f40 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -208,10 +208,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, jport); if (ret == 0) { - jniError("jobj:%p, taos:%p, connect to tdengine failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, + jniError("jobj:%p, conn:%p, connect to database failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, (char *)host, (char *)user, (char *)dbname, jport); } else { - jniTrace("jobj:%p, taos:%p, connect to tdengine succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, + jniTrace("jobj:%p, conn:%p, connect to database succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, (char *)host, (char *)user, (char *)dbname, jport); } @@ -232,7 +232,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J } if (jsql == NULL) { - jniError("jobj:%p, taos:%p, sql is null", jobj, tscon); + jniError("jobj:%p, conn:%p, sql is null", jobj, tscon); return JNI_SQL_NULL; } @@ -250,7 +250,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J int code = taos_query(tscon, dst); if (code != 0) { - jniError("jobj:%p, taos:%p, code:%d, msg:%s, sql:%s", jobj, tscon, code, taos_errstr(tscon), dst); + jniError("jobj:%p, conn:%p, code:%d, msg:%s, sql:%s", jobj, tscon, code, taos_errstr(tscon), dst); free(dst); return JNI_TDENGINE_ERROR; } else { @@ -259,9 +259,9 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J if (pSql->cmd.command == TSDB_SQL_INSERT) { affectRows = taos_affected_rows(tscon); - jniTrace("jobj:%p, taos:%p, code:%d, affect rows:%d, sql:%s", jobj, tscon, code, affectRows, dst); + jniTrace("jobj:%p, conn:%p, code:%d, affect rows:%d, sql:%s", jobj, tscon, code, affectRows, dst); } else { - jniTrace("jobj:%p, taos:%p, code:%d, sql:%s", jobj, tscon, code, dst); + jniTrace("jobj:%p, conn:%p, code:%d, sql:%s", jobj, tscon, code, dst); } free(dst); @@ -291,11 +291,22 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( return JNI_CONNECTION_NULL; } +<<<<<<< Updated upstream int num_fields = taos_field_count(tscon); if (num_fields != 0) { jlong ret = (jlong)taos_use_result(tscon); jniTrace("jobj:%p, taos:%p, get resultset:%p", jobj, tscon, (void *)ret); return ret; +======= + jlong ret = 0; + + if (tscIsUpdateQuery(tscon)) { + ret = 0; // for update query, no result pointer + jniTrace("jobj:%p, conn:%p, no result", jobj, tscon); + } else { + ret = (jlong) taos_use_result(tscon); + jniTrace("jobj:%p, conn:%p, get resultset:%p", jobj, tscon, (void *) ret); +>>>>>>> Stashed changes } jniTrace("jobj:%p, taos:%p, no resultset", jobj, tscon); @@ -311,12 +322,12 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp( } if ((void *)res == NULL) { - jniError("jobj:%p, taos:%p, resultset is null", jobj, tscon); + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); return JNI_RESULT_SET_NULL; } taos_free_result((void *)res); - jniTrace("jobj:%p, taos:%p, free resultset:%p", jobj, tscon, (void *)res); + jniTrace("jobj:%p, conn:%p, free resultset:%p", jobj, tscon, (void *)res); return JNI_SUCCESS; } @@ -330,7 +341,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsIm jint ret = taos_affected_rows(tscon); - jniTrace("jobj:%p, taos:%p, affect rows:%d", jobj, tscon, (void *)con, ret); + jniTrace("jobj:%p, conn:%p, affect rows:%d", jobj, tscon, (void *)con, ret); return ret; } @@ -346,7 +357,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData TAOS_RES *result = (TAOS_RES *)res; if (result == NULL) { - jniError("jobj:%p, taos:%p, resultset is null", jobj, tscon); + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); return JNI_RESULT_SET_NULL; } @@ -356,10 +367,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData // jobject arrayListObj = (*env)->NewObject(env, g_arrayListClass, g_arrayListConstructFp, ""); if (num_fields == 0) { - jniError("jobj:%p, taos:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); + jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); return JNI_NUM_OF_FIELDS_0; } else { - jniTrace("jobj:%p, taos:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); + jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); for (int i = 0; i < num_fields; ++i) { jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); @@ -402,7 +413,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn TAOS_RES *result = (TAOS_RES *)res; if (result == NULL) { - jniError("jobj:%p, taos:%p, resultset is null", jobj, tscon); + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); return JNI_RESULT_SET_NULL; } @@ -410,7 +421,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn int num_fields = taos_num_fields(result); if (num_fields == 0) { - jniError("jobj:%p, taos:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); + jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, res, num_fields); return JNI_NUM_OF_FIELDS_0; } @@ -418,10 +429,10 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn if (row == NULL) { int tserrno = taos_errno(tscon); if (tserrno == 0) { - jniTrace("jobj:%p, taos:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, res, num_fields); + jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, res, num_fields); return JNI_FETCH_END; } else { - jniTrace("jobj:%p, taos:%p, interruptted query", jobj, tscon); + jniTrace("jobj:%p, conn:%p, interruptted query", jobj, tscon); return JNI_RESULT_SET_NULL; } } @@ -485,7 +496,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionIm jniError("jobj:%p, connection is closed", jobj); return JNI_CONNECTION_NULL; } else { - jniTrace("jobj:%p, taos:%p, close connection success", jobj, tscon); + jniTrace("jobj:%p, conn:%p, close connection success", jobj, tscon); taos_close(tscon); return JNI_SUCCESS; } @@ -640,7 +651,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTab } if (jsql == NULL) { - jniError("jobj:%p, taos:%p, sql is null", jobj, tscon); + jniError("jobj:%p, conn:%p, sql is null", jobj, tscon); return JNI_SQL_NULL; } From b2d01f72e0a008e6b9562731fa52eeaa9288059d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 21:43:48 +0800 Subject: [PATCH 22/69] add update check command type --- src/client/src/tscUtil.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9a699b7bb0..547171cffe 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1824,10 +1824,8 @@ bool tscIsUpdateQuery(STscObj* pObj) { return TSDB_CODE_DISCONNECTED; } - SSqlCmd* pCmd = &(pObj->pSql->cmd); - if (pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) { - return 1; - } + SSqlCmd* pCmd = &pObj->pSql->cmd; + return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || + TSDB_SQL_USE_DB == pCmd->command) ? 1 : 0; - return 0; } From 30c38cbafe4061859dc6eb0906910d28c80cccbc Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 11 Nov 2019 23:32:08 +0800 Subject: [PATCH 23/69] update some log format --- src/client/src/TSDBJNIConnector.c | 18 ------------------ src/client/src/tscServer.c | 2 +- src/client/src/tscSystem.c | 2 +- src/kit/taosdemo/taosdemo.c | 3 +++ src/modules/http/src/httpSystem.c | 2 +- src/modules/monitor/src/monitorSystem.c | 4 ++-- tests/examples/JDBC/readme.md | 2 +- 7 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 74c4e0a360..a19579b7d4 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -289,32 +289,14 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( return JNI_CONNECTION_NULL; } -<<<<<<< HEAD -<<<<<<< Updated upstream - int num_fields = taos_field_count(tscon); - if (num_fields != 0) { - jlong ret = (jlong)taos_use_result(tscon); - jniTrace("jobj:%p, taos:%p, get resultset:%p", jobj, tscon, (void *)ret); - return ret; -======= -======= ->>>>>>> origin/develop jlong ret = 0; if (tscIsUpdateQuery(tscon)) { ret = 0; // for update query, no result pointer -<<<<<<< HEAD jniTrace("jobj:%p, conn:%p, no result", jobj, tscon); } else { ret = (jlong) taos_use_result(tscon); jniTrace("jobj:%p, conn:%p, get resultset:%p", jobj, tscon, (void *) ret); ->>>>>>> Stashed changes -======= - jniTrace("jobj:%p, taos:%p, no result", jobj, tscon); - } else { - ret = (jlong) taos_use_result(tscon); - jniTrace("jobj:%p, taos:%p, get resultset:%p", jobj, tscon, (void *) ret); ->>>>>>> origin/develop } return ret; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 49368e4d17..aa80e505d5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -562,7 +562,7 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { void *taosres = tscKeepConn[command] ? pSql : NULL; code = pRes->code ? -pRes->code : pRes->numOfRows; - tscTrace("%p Async SQL result:%d taosres:%p", pSql, code, taosres); + tscTrace("%p Async SQL result:%d res:%p", pSql, code, taosres); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 708ceee405..5e383c9d0b 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -176,7 +176,7 @@ void taos_init_imp() { tscConnCache = taosOpenConnCache(tsMaxMeterConnections * 2, taosCloseRpcConn, tscTmr, tsShellActivityTimer * 1000); initialized = 1; - tscTrace("taos client is initialized successfully"); + tscTrace("client is initialized successfully"); tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg); } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index df29a72e86..f9fe9efa33 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -30,6 +30,9 @@ #include #include "taos.h" + +extern char configDir[]; + #pragma GCC diagnostic ignored "-Wmissing-braces" #define BUFFER_SIZE 65536 diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c index 5c0d9a69d6..ece7c17814 100644 --- a/src/modules/http/src/httpSystem.c +++ b/src/modules/http/src/httpSystem.c @@ -50,7 +50,7 @@ int httpInitSystem() { httpServer = (HttpServer *)malloc(sizeof(HttpServer)); memset(httpServer, 0, sizeof(HttpServer)); - strcpy(httpServer->label, "taosh"); + strcpy(httpServer->label, "rest"); strcpy(httpServer->serverIp, tsHttpIp); httpServer->serverPort = tsHttpPort; httpServer->cacheContext = tsHttpCacheSessions; diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index 013d86050e..78cee40f96 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -131,7 +131,7 @@ void monitorInitConn(void *para, void *unused) { void monitorInitConnCb(void *param, TAOS_RES *result, int code) { if (code < 0) { - monitorError("monitor:%p, connect to taosd failed, code:%d", monitor->conn, code); + monitorError("monitor:%p, connect to database failed, code:%d", monitor->conn, code); taos_close(monitor->conn); monitor->conn = NULL; monitor->state = MONITOR_STATE_UN_INIT; @@ -139,7 +139,7 @@ void monitorInitConnCb(void *param, TAOS_RES *result, int code) { return; } - monitorTrace("monitor:%p, connect to taosd success, code:%d", monitor->conn, code); + monitorTrace("monitor:%p, connect to database success, code:%d", monitor->conn, code); monitorInitDatabase(); } diff --git a/tests/examples/JDBC/readme.md b/tests/examples/JDBC/readme.md index 54a2e5c6ac..a91624a9e4 100644 --- a/tests/examples/JDBC/readme.md +++ b/tests/examples/JDBC/readme.md @@ -9,5 +9,5 @@ TDengine's JDBC driver jar is not yet published to maven center repo, so we need ## Compile the Demo Code and Run It To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute 
mvn clean assembly:single package
-The ``pom.xml`` is configured to package all the dependencies into one executable jar file. To run it, go to ``TDengine/tests/examples/JDBC/JDBCDemo/target`` and execute +The ``pom.xml`` is configured to package all the dependencies into one executable jar file. To run it, go to ``examples/JDBC/JDBCDemo/target`` and execute 
java -jar jdbcdemo-1.0-SNAPSHOT-jar-with-dependencies.jar
From 8833fc69ae3d01e7992c6c06f1c7754872dab7dc Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 12 Nov 2019 10:07:34 +0800 Subject: [PATCH 24/69] remove illegal usage of configure file path --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index df29a72e86..928b06cb61 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -176,7 +176,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - strcpy(configDir, full_path.we_wordv[0]); + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); wordfree(&full_path); break; case OPT_ABORT: From 79e1a7bcf8dcee26cd4d00e29807b0127d106d7e Mon Sep 17 00:00:00 2001 From: robot Date: Tue, 12 Nov 2019 10:40:44 +0800 Subject: [PATCH 25/69] Add support for continuous query in Lua connector. Malloc memory to store Lua state and callback function from Lua. Connector can support multiple connection by this way. Memory will be freed when connection isn't established successfully or connection is closed by lua call. Stream content in Lua is not the same one with stream content in connector. A loop in test code do the job for continuous query. record is inserted between query actions. --- tests/examples/lua/build.sh | 0 tests/examples/lua/lua_connector.c | 131 ++++++++++++++++++++++++++++- tests/examples/lua/test.lua | 69 ++++++++++++++- 3 files changed, 195 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/examples/lua/build.sh diff --git a/tests/examples/lua/build.sh b/tests/examples/lua/build.sh old mode 100644 new mode 100755 diff --git a/tests/examples/lua/lua_connector.c b/tests/examples/lua/lua_connector.c index f37657e822..f4065bb274 100644 --- a/tests/examples/lua/lua_connector.c +++ b/tests/examples/lua/lua_connector.c @@ -7,8 +7,15 @@ #include #include -static int l_connect(lua_State *L) -{ +struct cb_param{ + lua_State* state; + int callback; + void * stream; +}; + + + +static int l_connect(lua_State *L){ TAOS * taos; char *host = lua_tostring(L, 1); char *user = lua_tostring(L, 2); @@ -29,6 +36,7 @@ static int l_connect(lua_State *L) lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "conn"); }else{ printf("success to connect server\n"); lua_pushnumber(L, 0); @@ -49,7 +57,7 @@ static int l_query(lua_State *L){ lua_newtable(L); int table_index = lua_gettop(L); - printf("receive command:%s\r\n",s); + // printf("receive command:%s\r\n",s); if(taos_query(taos, s)!=0){ printf("failed, reason:%s\n", taos_errstr(taos)); lua_pushnumber(L, -1); @@ -78,8 +86,12 @@ static int l_query(lua_State *L){ TAOS_FIELD *fields = taos_fetch_fields(result); char temp[256]; + int affectRows = taos_affected_rows(taos); + // printf(" affect rows:%d\r\n", affectRows); lua_pushnumber(L, 0); lua_setfield(L, table_index, "code"); + lua_pushinteger(L, affectRows); + lua_setfield(L, table_index, "affected"); lua_newtable(L); while ((row = taos_fetch_row(result))) { @@ -95,7 +107,7 @@ static int l_query(lua_State *L){ } lua_pushstring(L,fields[i].name); - //printf("field name:%s,type:%d\n",fields[i].name,fields[i].type); + switch (fields[i].type) { case TSDB_DATA_TYPE_TINYINT: lua_pushinteger(L,*((char *)row[i])); @@ -142,6 +154,115 @@ static int l_query(lua_State *L){ return 1; } +void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ + + struct cb_param* p = (struct cb_param*) param; + TAOS_FIELD *fields = taos_fetch_fields(result); + int numFields = taos_num_fields(result); + + printf("\n\r-----------------------------------------------------------------------------------\n"); + + // printf("r:%d, L:%d\n",p->callback, p->state); + + lua_State *L = p->state; + lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); + + lua_newtable(L); + + for (int i = 0; i < numFields; ++i) { + if (row[i] == NULL) { + continue; + } + + lua_pushstring(L,fields[i].name); + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + lua_pushinteger(L,*((char *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + lua_pushinteger(L,*((short *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + lua_pushinteger(L,*((int *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + lua_pushnumber(L,*((float *)row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + lua_pushnumber(L,*((double *)row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + lua_pushstring(L,(char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + lua_pushinteger(L,*((char *)row[i])); + break; + default: + lua_pushnil(L); + break; + } + + lua_settable(L, -3); + } + + lua_call(L, 1, 0); + + printf("-----------------------------------------------------------------------------------\n\r"); +} + +static int l_open_stream(lua_State *L){ + int r = luaL_ref(L, LUA_REGISTRYINDEX); + TAOS * taos = lua_topointer(L,1); + char * sqlstr = lua_tostring(L,2); + int stime = luaL_checknumber(L,3); + + lua_newtable(L); + int table_index = lua_gettop(L); + + struct cb_param *p = malloc(sizeof(struct cb_param)); + p->state = L; + p->callback=r; + // printf("r:%d, L:%d\n",r,L); + void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); + if (s == NULL) { + printf("failed to open stream, reason:%s\n", taos_errstr(taos)); + free(p); + lua_pushnumber(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "stream"); + }else{ + // printf("success to open stream\n"); + lua_pushnumber(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + p->stream = s; + lua_pushlightuserdata(L,p); + lua_setfield(L, table_index, "stream");//stream has different content in lua and c. + } + + return 1; +} + +static int l_close_stream(lua_State *L){ + //TODO:get stream and free cb_param + struct cb_param *p = lua_touserdata(L,1); + taos_close_stream(p->stream); + free(p); + return 0; +} + static int l_close(lua_State *L){ TAOS * taos= lua_topointer(L,1); lua_newtable(L); @@ -166,6 +287,8 @@ static const struct luaL_Reg lib[] = { {"connect", l_connect}, {"query", l_query}, {"close", l_close}, + {"open_stream", l_open_stream}, + {"close_stream", l_close_stream}, {NULL, NULL} }; diff --git a/tests/examples/lua/test.lua b/tests/examples/lua/test.lua index f644b82dd4..38ae1c82f2 100644 --- a/tests/examples/lua/test.lua +++ b/tests/examples/lua/test.lua @@ -35,10 +35,12 @@ if res.code ~=0 then return end -res = driver.query(conn,"insert into m1 values (1592222222222,0,'robotspace'), (1592222222223,1,'Hilink'),(1592222222224,2,'Harmony')") +res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") if res.code ~=0 then print(res.error) return +else + print("insert successfully, affected:"..res.affected) end res = driver.query(conn,"select * from m1") @@ -55,4 +57,69 @@ else end end +res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") +if res.code ~=0 then + print(res.error) + return +end +res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") +if res.code ~=0 then + print(res.error) + return +end +res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") + +if res.code ~=0 then + print(res.error) + return +else + print("insert successfully, affected:"..res.affected) +end + +res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") +if res.code ~=0 then + print("select error:"..res.error) + return +else + print("in lua, result:") + for i = 1, #(res.item) do + print("res:"..res.item[i].count) + end +end + +function callback(t) + print("continuous query result:") + for key, value in pairs(t) do + print("key:"..key..", value:"..value) + end +end + +local stream +res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0,callback) +if res.code ~=0 then + print("open stream error:"..res.error) + return +else + print("openstream ok") + stream = res.stream +end + +--From now on we begin continous query in an definite (infinite if you want) loop. +local loop_index = 0 +while loop_index < 20 do + local t = os.time()*1000 + local v = loop_index + res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) + + if res.code ~=0 then + print(res.error) + return + else + print("insert successfully, affected:"..res.affected) + end + os.execute("sleep " .. 1) + loop_index = loop_index + 1 +end + +driver.close_stream(stream) driver.close(conn) From b12e12f8402bfcd8871349e559e539c0e4e75f00 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 12 Nov 2019 11:47:47 +0800 Subject: [PATCH 26/69] add some logs for release resource at vnode side --- src/inc/tsdb.h | 16 ++++++++-------- src/system/detail/src/vnodeShell.c | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index de4e430a46..4c43171283 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -221,20 +221,20 @@ enum _syncstatus { #define TSDB_MAX_RPC_THREADS 5 #define TSDB_QUERY_TYPE_QUERY 0 // normal query -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x1 // free qhandle at vnode +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01U // free qhandle at vnode /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x2 -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x4 // two-stage subquery for super table +#define TSDB_QUERY_TYPE_SUBQUERY 0x02U +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04U // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x8 // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10 // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20 // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40 // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80 // join sub query at the second stage +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08U // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10U // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20U // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40U // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80U // join sub query at the second stage #ifdef __cplusplus } diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 353b2668ee..bbe2f60274 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -385,14 +385,21 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { pRetrieve = (SRetrieveMeterMsg *)pMsg; pRetrieve->free = htons(pRetrieve->free); + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + dTrace("retrieve msg, handle:%p, free:%d", pRetrieve->qhandle, pRetrieve->free); + } else { + dTrace("retrieve msg to free resource from client, handle:%p, free:%d", pRetrieve->qhandle, pRetrieve->free); + } + /* * in case of server restart, apps may hold qhandle created by server before restart, * which is actually invalid, therefore, signature check is required. */ if (pRetrieve->qhandle == (uint64_t)pObj->qhandle) { // if free flag is set, client wants to clean the resources - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { code = vnodeRetrieveQueryInfo((void *)(pRetrieve->qhandle), &numOfRows, &rowSize, &timePrec); + } } else { dError("QInfo:%p, qhandle:%p is not matched with saved:%p", pObj->qhandle, pRetrieve->qhandle, pObj->qhandle); code = TSDB_CODE_INVALID_QHANDLE; From 9b754d70f4d20924500d354a44963a1dfb665714 Mon Sep 17 00:00:00 2001 From: localvar Date: Tue, 12 Nov 2019 06:38:32 +0000 Subject: [PATCH 27/69] add more atomic api --- src/client/src/TSDBJNIConnector.c | 6 +- src/client/src/tscJoinProcess.c | 10 +- src/client/src/tscSecondaryMerge.c | 7 +- src/client/src/tscServer.c | 10 +- src/client/src/tscStream.c | 2 +- src/modules/http/src/httpServer.c | 6 +- src/modules/http/src/httpSystem.c | 2 +- src/os/darwin/inc/os.h | 81 +++++++++--- src/os/darwin/src/tdarwin.c | 8 -- src/os/linux/inc/os.h | 81 +++++++++--- src/os/linux/src/tlinux.c | 8 -- src/os/windows/inc/os.h | 167 +++++++++++++++++++++---- src/os/windows/src/twindows.c | 130 +++++++++++++++++-- src/rpc/src/trpc.c | 8 +- src/system/detail/inc/vnode.h | 6 +- src/system/detail/src/dnodeSystem.c | 4 +- src/system/detail/src/mgmtShell.c | 16 +-- src/system/detail/src/vnodeCache.c | 6 +- src/system/detail/src/vnodeFile.c | 2 +- src/system/detail/src/vnodeImport.c | 4 +- src/system/detail/src/vnodeMeter.c | 4 +- src/system/detail/src/vnodeQueryImpl.c | 4 +- src/system/detail/src/vnodeShell.c | 4 +- src/system/detail/src/vnodeStore.c | 2 +- src/system/detail/src/vnodeUtil.c | 6 +- src/util/src/tcache.c | 16 +-- src/util/src/textbuffer.c | 2 +- src/util/src/tlog.c | 4 +- src/util/src/ttimer.c | 16 +-- 29 files changed, 462 insertions(+), 160 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index a19579b7d4..d958679544 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -61,13 +61,13 @@ jmethodID g_rowdataSetByteArrayFp; void jniGetGlobalMethod(JNIEnv *env) { // make sure init function executed once - switch (__sync_val_compare_and_swap_32(&__init, 0, 1)) { + switch (atomic_val_compare_exchange_32(&__init, 0, 1)) { case 0: break; case 1: do { taosMsleep(0); - } while (__sync_val_load_32(&__init) == 1); + } while (atomic_load_32(&__init) == 1); case 2: return; } @@ -107,7 +107,7 @@ void jniGetGlobalMethod(JNIEnv *env) { g_rowdataSetByteArrayFp = (*env)->GetMethodID(env, g_rowdataClass, "setByteArray", "(I[B)V"); (*env)->DeleteLocalRef(env, rowdataClass); - __sync_val_restore_32(&__init, 2); + atomic_store_32(&__init, 2); jniTrace("native method register finished"); } diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index dfeebc8c61..24d8ad902a 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -353,7 +353,7 @@ static void doQuitSubquery(SSqlObj* pParentSql) { } static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { - if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + if (atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { pSqlObj->res.code = abs(pSupporter->pState->code); tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); @@ -412,7 +412,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { taos_fetch_rows_a(tres, joinRetrieveCallback, param); } else if (numOfRows == 0) { // no data from this vnode anymore - if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + if (atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, tres, @@ -451,7 +451,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { tscError("%p retrieve failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); } - if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + if (atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { tscTrace("%p secondary retrieve completed, global code:%d", tres, pParentSql->res.code); if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { pParentSql->res.code = abs(pSupporter->pState->code); @@ -560,7 +560,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; - // if (__sync_add_and_fetch_32(pSupporter->numOfComplete, 1) >= + // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= // pSupporter->numOfTotal) { // SSqlObj *pParentObj = pSupporter->pObj; // @@ -605,7 +605,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { quitAllSubquery(pParentSql, pSupporter); } else { - if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + if (atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { tscSetupOutputColumnIndex(pParentSql); if (pParentSql->fp == NULL) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 398cd1cca6..c1fe0c8674 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -432,11 +432,10 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; // there is no more result, so we release all allocated resource - SLocalReducer *pLocalReducer = - (SLocalReducer *)__sync_val_compare_and_swap_64(&pRes->pLocalReducer, pRes->pLocalReducer, 0); + SLocalReducer *pLocalReducer = (SLocalReducer*)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); if (pLocalReducer != NULL) { int32_t status = 0; - while ((status = __sync_val_compare_and_swap_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, + while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_TOBE_FREED)) == TSC_LOCALREDUCE_IN_PROGRESS) { taosMsleep(100); tscTrace("%p waiting for delete procedure, status: %d", pSql, status); @@ -1328,7 +1327,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { // set the data merge in progress int32_t prevStatus = - __sync_val_compare_and_swap_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); + atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); if (prevStatus != TSC_LOCALREDUCE_READY || pLocalReducer == NULL) { assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already return TSDB_CODE_SUCCESS; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index aa80e505d5..511ccb23fa 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1023,13 +1023,13 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq tscProcessSql(pNew); return; } else { // reach the maximum retry count, abort - __sync_val_compare_and_swap_32(&trsupport->pState->code, TSDB_CODE_SUCCESS, numOfRows); + atomic_val_compare_exchange_32(&trsupport->pState->code, TSDB_CODE_SUCCESS, numOfRows); tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, numOfRows, idx, trsupport->pState->code); } } - if (__sync_add_and_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { + if (atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { return tscFreeSubSqlObj(trsupport, pSql); } @@ -1095,7 +1095,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); - __sync_add_and_fetch_64(&trsupport->pState->numOfRetrievedRows, numOfRows); + atomic_add_fetch_64(&trsupport->pState->numOfRetrievedRows, numOfRows); tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, pRes->numOfRows, trsupport->pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); @@ -1154,7 +1154,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); } - if (__sync_add_and_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { + if (atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { return tscFreeSubSqlObj(trsupport, pSql); } @@ -1283,7 +1283,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { if (code != TSDB_CODE_SUCCESS) { if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) { tscTrace("%p sub:%p reach the max retry count,set global code:%d", trsupport->pParentSqlObj, pSql, code); - __sync_val_compare_and_swap_32(&trsupport->pState->code, 0, code); + atomic_val_compare_exchange_32(&trsupport->pState->code, 0, code); } else { // does not reach the maximum retry count, go on tscTrace("%p sub:%p failed code:%d, retry:%d", trsupport->pParentSqlObj, pSql, code, trsupport->numOfRetry); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index ec8233c6f9..c9809be1e3 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -546,7 +546,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p void taos_close_stream(TAOS_STREAM *handle) { SSqlStream *pStream = (SSqlStream *)handle; - SSqlObj *pSql = (SSqlObj *)__sync_val_compare_and_swap_64(&pStream->pSql, pStream->pSql, 0); + SSqlObj *pSql = (SSqlObj *)atomic_exchange_ptr(&pStream->pSql, 0); if (pSql == NULL) { return; } diff --git a/src/modules/http/src/httpServer.c b/src/modules/http/src/httpServer.c index 8b981e0f84..49ff3562bf 100644 --- a/src/modules/http/src/httpServer.c +++ b/src/modules/http/src/httpServer.c @@ -72,7 +72,7 @@ void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext) { } bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { - return (__sync_val_compare_and_swap_32(&pContext->state, srcState, destState) == srcState); + return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); } void httpFreeContext(HttpServer *pServer, HttpContext *pContext); @@ -124,7 +124,7 @@ void httpCleanUpContextTimer(HttpContext *pContext) { void httpCleanUpContext(HttpContext *pContext) { httpTrace("context:%p, start the clean up operation", pContext); - __sync_val_compare_and_swap_64(&pContext->signature, pContext, 0); + atomic_val_compare_exchange_ptr(&pContext->signature, pContext, 0); if (pContext->signature != NULL) { httpTrace("context:%p is freed by another thread.", pContext); return; @@ -494,7 +494,7 @@ void httpProcessHttpData(void *param) { } else { if (httpReadData(pThread, pContext)) { (*(pThread->processData))(pContext); - __sync_fetch_and_add(&pThread->pServer->requestNum, 1); + atomic_fetch_add_32(&pThread->pServer->requestNum, 1); } } } diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c index ece7c17814..67be206842 100644 --- a/src/modules/http/src/httpSystem.c +++ b/src/modules/http/src/httpSystem.c @@ -148,7 +148,7 @@ void httpCleanUpSystem() { void httpGetReqCount(int32_t *httpReqestNum) { if (httpServer != NULL) { - *httpReqestNum = __sync_fetch_and_and(&httpServer->requestNum, 0); + *httpReqestNum = atomic_exchange_32(&httpServer->requestNum, 0); } else { *httpReqestNum = 0; } diff --git a/src/os/darwin/inc/os.h b/src/os/darwin/inc/os.h index 83a56483b8..eabd5cd221 100644 --- a/src/os/darwin/inc/os.h +++ b/src/os/darwin/inc/os.h @@ -73,28 +73,71 @@ #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -// TODO: update prefix of below macros to 'atomic' as '__' is reserved by compiler -// and GCC suggest new code to use '__atomic' builtins to replace '__sync' builtins. -#define __sync_val_compare_and_swap_64 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_32 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_16 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_8 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_ptr __sync_val_compare_and_swap +#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap -#define __sync_add_and_fetch_64 __sync_add_and_fetch -#define __sync_add_and_fetch_32 __sync_add_and_fetch -#define __sync_add_and_fetch_16 __sync_add_and_fetch -#define __sync_add_and_fetch_8 __sync_add_and_fetch -#define __sync_add_and_fetch_ptr __sync_add_and_fetch +#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define __sync_sub_and_fetch_64 __sync_sub_and_fetch -#define __sync_sub_and_fetch_32 __sync_sub_and_fetch -#define __sync_sub_and_fetch_16 __sync_sub_and_fetch -#define __sync_sub_and_fetch_8 __sync_sub_and_fetch -#define __sync_sub_and_fetch_ptr __sync_sub_and_fetch +#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -int32_t __sync_val_load_32(int32_t *ptr); -void __sync_val_restore_32(int32_t *ptr, int32_t newval); +#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) #define SWAP(a, b, c) \ do { \ diff --git a/src/os/darwin/src/tdarwin.c b/src/os/darwin/src/tdarwin.c index 2f97c7c376..eec252d660 100644 --- a/src/os/darwin/src/tdarwin.c +++ b/src/os/darwin/src/tdarwin.c @@ -416,11 +416,3 @@ int tsem_post(dispatch_semaphore_t *sem) { int tsem_destroy(dispatch_semaphore_t *sem) { return 0; } - -int32_t __sync_val_load_32(int32_t *ptr) { - return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); -} - -void __sync_val_restore_32(int32_t *ptr, int32_t newval) { - __atomic_store_n(ptr, newval, __ATOMIC_RELEASE); -} \ No newline at end of file diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index ee65965a5b..b9bb8b3e10 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -90,28 +90,71 @@ extern "C" { #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -// TODO: update prefix of below macros to 'atomic' as '__' is reserved by compiler -// and GCC suggest new code to use '__atomic' builtins to replace '__sync' builtins. -#define __sync_val_compare_and_swap_64 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_32 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_16 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_8 __sync_val_compare_and_swap -#define __sync_val_compare_and_swap_ptr __sync_val_compare_and_swap +#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap -#define __sync_add_and_fetch_64 __sync_add_and_fetch -#define __sync_add_and_fetch_32 __sync_add_and_fetch -#define __sync_add_and_fetch_16 __sync_add_and_fetch -#define __sync_add_and_fetch_8 __sync_add_and_fetch -#define __sync_add_and_fetch_ptr __sync_add_and_fetch +#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define __sync_sub_and_fetch_64 __sync_sub_and_fetch -#define __sync_sub_and_fetch_32 __sync_sub_and_fetch -#define __sync_sub_and_fetch_16 __sync_sub_and_fetch -#define __sync_sub_and_fetch_8 __sync_sub_and_fetch -#define __sync_sub_and_fetch_ptr __sync_sub_and_fetch +#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -int32_t __sync_val_load_32(int32_t *ptr); -void __sync_val_restore_32(int32_t *ptr, int32_t newval); +#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + +#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) #define SWAP(a, b, c) \ do { \ diff --git a/src/os/linux/src/tlinux.c b/src/os/linux/src/tlinux.c index b5271006e2..f78250cb52 100644 --- a/src/os/linux/src/tlinux.c +++ b/src/os/linux/src/tlinux.c @@ -340,11 +340,3 @@ bool taosSkipSocketCheck() { return false; } - -int32_t __sync_val_load_32(int32_t *ptr) { - return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); -} - -void __sync_val_restore_32(int32_t *ptr, int32_t newval) { - __atomic_store_n(ptr, newval, __ATOMIC_RELEASE); -} diff --git a/src/os/windows/inc/os.h b/src/os/windows/inc/os.h index 3f61ff12b5..a1a4bdfa5c 100644 --- a/src/os/windows/inc/os.h +++ b/src/os/windows/inc/os.h @@ -81,6 +81,10 @@ extern "C" { #if defined(_M_ARM) || defined(_M_ARM64) +/* the '__iso_volatile' functions does not use a memory fence, so these + * definitions are incorrect, comment out as we don't support Windows on + * ARM at present. + #define atomic_load_8(ptr) __iso_volatile_load8((const volatile __int8*)(ptr)) #define atomic_load_16(ptr) __iso_volatile_load16((const volatile __int16*)(ptr)) #define atomic_load_32(ptr) __iso_volatile_load32((const volatile __int32*)(ptr)) @@ -98,7 +102,7 @@ extern "C" { #define atomic_load_ptr atomic_load_32 #define atomic_store_ptr atomic_store_32 #endif - +*/ #else #define atomic_load_8(ptr) (*(char volatile*)(ptr)) @@ -121,35 +125,152 @@ extern "C" { #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) -#define __sync_val_compare_and_swap_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) -#define __sync_val_compare_and_swap_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) -#define __sync_val_compare_and_swap_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) -#define __sync_val_compare_and_swap_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) -#define __sync_val_compare_and_swap_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) +#define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) +#define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) +#define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) +#define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) +#define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) -char interlocked_add_8(char volatile *ptr, char val); -short interlocked_add_16(short volatile *ptr, short val); -long interlocked_add_32(long volatile *ptr, long val); -__int64 interlocked_add_64(__int64 volatile *ptr, __int64 val); +char interlocked_add_fetch_8(char volatile *ptr, char val); +short interlocked_add_fetch_16(short volatile *ptr, short val); +long interlocked_add_fetch_32(long volatile *ptr, long val); +__int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); -#define __sync_add_and_fetch_8(ptr, val) interlocked_add_8((char volatile*)(ptr), (char)(val)) -#define __sync_add_and_fetch_16(ptr, val) interlocked_add_16((short volatile*)(ptr), (short)(val)) -#define __sync_add_and_fetch_32(ptr, val) interlocked_add_32((long volatile*)(ptr), (long)(val)) -#define __sync_add_and_fetch_64(ptr, val) interlocked_add_64((__int64 volatile*)(ptr), (__int64)(val)) +#define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) +#define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) +#define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) +#define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) #ifdef _WIN64 - #define __sync_add_and_fetch_ptr __sync_add_and_fetch_64 + #define atomic_add_fetch_ptr atomic_add_fetch_64 #else - #define __sync_add_and_fetch_ptr __sync_add_and_fetch_32 + #define atomic_add_fetch_ptr atomic_add_fetch_32 #endif -#define __sync_sub_and_fetch_8(ptr, val) __sync_add_and_fetch_8((ptr), -(val)) -#define __sync_sub_and_fetch_16(ptr, val) __sync_add_and_fetch_16((ptr), -(val)) -#define __sync_sub_and_fetch_32(ptr, val) __sync_add_and_fetch_32((ptr), -(val)) -#define __sync_sub_and_fetch_64(ptr, val) __sync_add_and_fetch_64((ptr), -(val)) -#define __sync_sub_and_fetch_ptr(ptr, val) __sync_add_and_fetch_ptr((ptr), -(val)) +#define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) +#define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) +#define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) +#define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) +#ifdef _WIN64 + #define atomic_fetch_add_ptr atomic_fetch_add_64 +#else + #define atomic_fetch_add_ptr atomic_fetch_add_32 +#endif -int32_t __sync_val_load_32(int32_t *ptr); -void __sync_val_restore_32(int32_t *ptr, int32_t newval); +#define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) +#define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) +#define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) +#define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) +#ifdef _WIN64 + #define atomic_sub_fetch_ptr atomic_sub_fetch_64 +#else + #define atomic_sub_fetch_ptr atomic_sub_fetch_32 +#endif + +#define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) +#define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) +#define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) +#define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) +#ifdef _WIN64 + #define atomic_fetch_sub_ptr atomic_fetch_sub_64 +#else + #define atomic_fetch_sub_ptr atomic_fetch_sub_32 +#endif + +char interlocked_and_fetch_8(char volatile* ptr, char val); +short interlocked_and_fetch_16(short volatile* ptr, short val); +long interlocked_and_fetch_32(long volatile* ptr, long val); +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); + +#define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) +#define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) +#define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) +#define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) +#ifdef _WIN64 + #define atomic_and_fetch_ptr atomic_and_fetch_64 +#else + #define atomic_and_fetch_ptr atomic_and_fetch_32 +#endif + +#define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) +#define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) +#define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) + +#ifdef _M_IX86 + __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) +#else + #define atomic_fetch_and_64(ptr, val) _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)) +#endif + +#ifdef _WIN64 + #define atomic_fetch_and_ptr atomic_fetch_and_64 +#else + #define atomic_fetch_and_ptr atomic_fetch_and_32 +#endif + +char interlocked_or_fetch_8(char volatile* ptr, char val); +short interlocked_or_fetch_16(short volatile* ptr, short val); +long interlocked_or_fetch_32(long volatile* ptr, long val); +__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); + +#define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) +#define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) +#define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) +#define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) +#ifdef _WIN64 + #define atomic_or_fetch_ptr atomic_or_fetch_64 +#else + #define atomic_or_fetch_ptr atomic_or_fetch_32 +#endif + +#define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) +#define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) +#define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) + +#ifdef _M_IX86 + __int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) +#else + #define atomic_fetch_or_64(ptr, val) _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)) +#endif + +#ifdef _WIN64 + #define atomic_fetch_or_ptr atomic_fetch_or_64 +#else + #define atomic_fetch_or_ptr atomic_fetch_or_32 +#endif + +char interlocked_xor_fetch_8(char volatile* ptr, char val); +short interlocked_xor_fetch_16(short volatile* ptr, short val); +long interlocked_xor_fetch_32(long volatile* ptr, long val); +__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); + +#define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) +#define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) +#define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) +#define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) +#ifdef _WIN64 + #define atomic_xor_fetch_ptr atomic_xor_fetch_64 +#else + #define atomic_xor_fetch_ptr atomic_xor_fetch_32 +#endif + +#define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) +#define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) +#define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) + +#ifdef _M_IX86 + __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) +#else + #define atomic_fetch_xor_64(ptr, val) _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)) +#endif + +#ifdef _WIN64 + #define atomic_fetch_xor_ptr atomic_fetch_xor_64 +#else + #define atomic_fetch_xor_ptr atomic_fetch_xor_32 +#endif #define SWAP(a, b, c) \ do { \ diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 106cb903b1..6c10ced6a2 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -66,31 +66,143 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle return setsockopt(socketfd, level, optname, optval, optlen); } - -char interlocked_add_8(char volatile* ptr, char val) { +// add +char interlocked_add_fetch_8(char volatile* ptr, char val) { return _InterlockedExchangeAdd8(ptr, val) + val; } -short interlocked_add_16(short volatile* ptr, short val) { +short interlocked_add_fetch_16(short volatile* ptr, short val) { return _InterlockedExchangeAdd16(ptr, val) + val; } -long interlocked_add_32(long volatile* ptr, long val) { +long interlocked_add_fetch_32(long volatile* ptr, long val) { return _InterlockedExchangeAdd(ptr, val) + val; } -__int64 interlocked_add_64(__int64 volatile* ptr, __int64 val) { +__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { return _InterlockedExchangeAdd64(ptr, val) + val; } -int32_t __sync_val_load_32(int32_t *ptr) { - return InterlockedOr(ptr, 0); +// and +char interlocked_and_fetch_8(char volatile* ptr, char val) { + return _InterlockedAnd8(ptr, val) & val; } -void __sync_val_restore_32(int32_t *ptr, int32_t newval) { - InterlockedCompareExchange(ptr, *ptr, newval); +short interlocked_and_fetch_16(short volatile* ptr, short val) { + return _InterlockedAnd16(ptr, val) & val; } +long interlocked_and_fetch_32(long volatile* ptr, long val) { + return _InterlockedAnd(ptr, val) & val; +} + +#ifndef _M_IX86 + +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedAnd64(ptr, val) & val; +} + +#else + +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old & val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old & val, old) != old); + return old; +} + +#endif + +// or +char interlocked_or_fetch_8(char volatile* ptr, char val) { + return _InterlockedOr8(ptr, val) | val; +} + +short interlocked_or_fetch_16(short volatile* ptr, short val) { + return _InterlockedOr16(ptr, val) | val; +} + +long interlocked_or_fetch_32(long volatile* ptr, long val) { + return _InterlockedOr(ptr, val) | val; +} + +#ifndef _M_IX86 + +__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedOr64(ptr, val) & val; +} + +#else + +__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old | val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); + return old; +} + +#endif + +// xor +char interlocked_xor_fetch_8(char volatile* ptr, char val) { + return _InterlockedXor8(ptr, val) ^ val; +} + +short interlocked_xor_fetch_16(short volatile* ptr, short val) { + return _InterlockedXor16(ptr, val) ^ val; +} + +long interlocked_xor_fetch_32(long volatile* ptr, long val) { + return _InterlockedXor(ptr, val) ^ val; +} + +#ifndef _M_IX86 + +__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedXor64(ptr, val) ^ val; +} + +#else + +__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old ^ val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old ^ val, old) != old); + return old; +} + +#endif + void tsPrintOsInfo() {} char *taosCharsetReplace(char *charsetstr) { diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 6f92154e9e..41c628e169 100644 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -164,8 +164,8 @@ char *taosBuildReqHeader(void *param, char type, char *msg) { pHeader->spi = 0; pHeader->tcp = 0; pHeader->encrypt = 0; - pHeader->tranId = __sync_add_and_fetch_32(&pConn->tranId, 1); - if (pHeader->tranId == 0) pHeader->tranId = __sync_add_and_fetch_32(&pConn->tranId, 1); + pHeader->tranId = atomic_add_fetch_32(&pConn->tranId, 1); + if (pHeader->tranId == 0) pHeader->tranId = atomic_add_fetch_32(&pConn->tranId, 1); pHeader->sourceId = pConn->ownId; pHeader->destId = pConn->peerId; @@ -196,8 +196,8 @@ char *taosBuildReqMsgWithSize(void *param, char type, int size) { pHeader->spi = 0; pHeader->tcp = 0; pHeader->encrypt = 0; - pHeader->tranId = __sync_add_and_fetch_32(&pConn->tranId, 1); - if (pHeader->tranId == 0) pHeader->tranId = __sync_add_and_fetch_32(&pConn->tranId, 1); + pHeader->tranId = atomic_add_fetch_32(&pConn->tranId, 1); + if (pHeader->tranId == 0) pHeader->tranId = atomic_add_fetch_32(&pConn->tranId, 1); pHeader->sourceId = pConn->ownId; pHeader->destId = pConn->peerId; diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 4d2ebdfa35..373d9e713b 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -218,15 +218,15 @@ typedef struct { * Only the QInfo.signature == QInfo, this structure can be released safely. */ #define TSDB_QINFO_QUERY_FLAG 0x1 -#define TSDB_QINFO_RESET_SIG(x) ((x)->signature = (uint64_t)(x)) +#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) #define TSDB_QINFO_SET_QUERY_FLAG(x) \ - __sync_val_compare_and_swap(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); + atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); // live lock: wait for query reaching a safe-point, release all resources // belongs to this query #define TSDB_WAIT_TO_SAFE_DROP_QINFO(x) \ { \ - while (__sync_val_compare_and_swap(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ + while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ taosMsleep(1); \ } \ } diff --git a/src/system/detail/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c index a0fdf95fc8..30b3e202ad 100644 --- a/src/system/detail/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -216,8 +216,8 @@ void dnodeResetSystem() { void dnodeCountRequest(SCountInfo *info) { httpGetReqCount(&info->httpReqNum); - info->selectReqNum = __sync_fetch_and_and(&vnodeSelectReqNum, 0); - info->insertReqNum = __sync_fetch_and_and(&vnodeInsertReqNum, 0); + info->selectReqNum = atomic_exchange_32(&vnodeSelectReqNum, 0); + info->insertReqNum = atomic_exchange_32(&vnodeInsertReqNum, 0); } #pragma GCC diagnostic pop \ No newline at end of file diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 43b002992d..11a85086d9 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -922,8 +922,8 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { taosSendMsgToPeer(pConn->thandle, pStart, msgLen); if (rowsToRead == 0) { - int64_t oldSign = __sync_val_compare_and_swap(&pShow->signature, (uint64_t)pShow, 0); - if (oldSign != (uint64_t)pShow) { + uintptr_t oldSign = atomic_val_compare_exchange_ptr(&pShow->signature, pShow, 0); + if (oldSign != (uintptr_t)pShow) { return msgLen; } // pShow->signature = 0; @@ -1093,8 +1093,8 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { } void mgmtEstablishConn(SConnObj *pConn) { - __sync_fetch_and_add(&mgmtShellConns, 1); - __sync_fetch_and_add(&sdbExtConns, 1); + atomic_fetch_add_32(&mgmtShellConns, 1); + atomic_fetch_add_32(&sdbExtConns, 1); pConn->stime = taosGetTimestampMs(); if (strcmp(pConn->pUser->user, "root") == 0 || strcmp(pConn->pUser->user, pConn->pAcct->user) == 0) { @@ -1168,8 +1168,8 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (pConn->pAcct) { mgmtRemoveConnFromAcct(pConn); - __sync_fetch_and_sub(&mgmtShellConns, 1); - __sync_fetch_and_sub(&sdbExtConns, 1); + atomic_fetch_sub_32(&mgmtShellConns, 1); + atomic_fetch_sub_32(&sdbExtConns, 1); } code = 0; @@ -1227,8 +1227,8 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { if (msg == NULL) { if (pConn) { mgmtRemoveConnFromAcct(pConn); - __sync_fetch_and_sub(&mgmtShellConns, 1); - __sync_fetch_and_sub(&sdbExtConns, 1); + atomic_fetch_sub_32(&mgmtShellConns, 1); + atomic_fetch_sub_32(&sdbExtConns, 1); mTrace("connection from %s is closed", pConn->pUser->user); memset(pConn, 0, sizeof(SConnObj)); } diff --git a/src/system/detail/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c index 8b51bc4609..f4bea682e9 100644 --- a/src/system/detail/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -256,7 +256,7 @@ void vnodeUpdateCommitInfo(SMeterObj *pObj, int slot, int pos, uint64_t count) { tslot = (tslot + 1) % pInfo->maxBlocks; } - __sync_fetch_and_add(&pObj->freePoints, pObj->pointsPerBlock * slots); + atomic_fetch_add_32(&pObj->freePoints, pObj->pointsPerBlock * slots); pInfo->commitSlot = slot; pInfo->commitPoint = pos; pObj->commitCount = count; @@ -505,7 +505,7 @@ int vnodeInsertPointToCache(SMeterObj *pObj, char *pData) { pData += pObj->schema[col].bytes; } - __sync_fetch_and_sub(&pObj->freePoints, 1); + atomic_fetch_sub_32(&pObj->freePoints, 1); pCacheBlock->numOfPoints++; pPool->count++; @@ -1114,7 +1114,7 @@ int vnodeSyncRestoreCache(int vnode, int fd) { for (int col = 0; col < pObj->numOfColumns; ++col) if (taosReadMsg(fd, pBlock->offset[col], pObj->schema[col].bytes * points) <= 0) return -1; - __sync_fetch_and_sub(&pObj->freePoints, points); + atomic_fetch_sub_32(&pObj->freePoints, points); blocksReceived++; pointsReceived += points; pObj->lastKey = *((TSKEY *)(pBlock->offset[0] + pObj->schema[0].bytes * (points - 1))); diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index df94c883ac..fdafc01352 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -410,7 +410,7 @@ void vnodeRemoveFile(int vnode, int fileId) { int fd = open(headName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (fd > 0) { vnodeGetHeadFileHeaderInfo(fd, &headInfo); - __sync_fetch_and_add(&(pVnode->vnodeStatistic.totalStorage), -headInfo.totalStorage); + atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), -headInfo.totalStorage); close(fd); } diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index f50b6f4946..ed9c319216 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -497,7 +497,7 @@ int vnodeImportToFile(SImportInfo *pImport) { pInfo->commitPoint = 0; pCacheBlock->numOfPoints = points; if (slot == pInfo->currentSlot) { - __sync_fetch_and_add(&pObj->freePoints, pInfo->commitPoint); + atomic_fetch_add_32(&pObj->freePoints, pInfo->commitPoint); } } else { // if last block is full and committed @@ -625,7 +625,7 @@ int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { } code = 0; - __sync_fetch_and_sub(&pObj->freePoints, rows); + atomic_fetch_sub_32(&pObj->freePoints, rows); dTrace("vid:%d sid:%d id:%s, %d rows data are imported to cache", pObj->vnode, pObj->sid, pObj->meterId, rows); _exit: diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index a595e8f689..c001daf7da 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -643,8 +643,8 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi pData += pObj->bytesPerPoint; points++; } - __sync_fetch_and_add(&(pVnode->vnodeStatistic.pointsWritten), points * (pObj->numOfColumns - 1)); - __sync_fetch_and_add(&(pVnode->vnodeStatistic.totalStorage), points * pObj->bytesPerPoint); + atomic_fetch_add_64(&(pVnode->vnodeStatistic.pointsWritten), points * (pObj->numOfColumns - 1)); + atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), points * pObj->bytesPerPoint); pthread_mutex_lock(&(pVnode->vmutex)); diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 9c0d7883a5..83f883c7b3 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -3889,14 +3889,14 @@ void vnodeDecMeterRefcnt(SQInfo *pQInfo) { SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; if (pSupporter == NULL || pSupporter->numOfMeters == 1) { - __sync_fetch_and_sub(&pQInfo->pObj->numOfQueries, 1); + atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1); dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode, pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries); } else { int32_t num = 0; for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { SMeterObj *pMeter = getMeterObj(pSupporter->pMeterObj, pSupporter->pSidSet->pSids[i]->sid); - __sync_fetch_and_sub(&(pMeter->numOfQueries), 1); + atomic_fetch_sub_32(&(pMeter->numOfQueries), 1); if (pMeter->numOfQueries > 0) { dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pMeter->vnode, pMeter->sid, diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 353b2668ee..d0db4b546f 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -365,7 +365,7 @@ _query_over: vnodeFreeColumnInfo(&pQueryMsg->colList[i]); } - __sync_fetch_and_add(&vnodeSelectReqNum, 1); + atomic_fetch_add_32(&vnodeSelectReqNum, 1); return ret; } @@ -592,6 +592,6 @@ _submit_over: // for import, send the submit response only when return code is not zero if (pSubmit->import == 0 || code != 0) ret = vnodeSendShellSubmitRspMsg(pObj, code, numOfTotalPoints); - __sync_fetch_and_add(&vnodeInsertReqNum, 1); + atomic_fetch_add_32(&vnodeInsertReqNum, 1); return ret; } diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index e00b4de7bc..e4294d72c6 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -351,7 +351,7 @@ void vnodeCalcOpenVnodes() { openVnodes++; } - __sync_val_compare_and_swap(&tsOpenVnodes, tsOpenVnodes, openVnodes); + atomic_store_32(&tsOpenVnodes, openVnodes); } void vnodeUpdateHeadFile(int vnode, int oldTables, int newTables) { diff --git a/src/system/detail/src/vnodeUtil.c b/src/system/detail/src/vnodeUtil.c index 790ad1c4a1..c16f8a1858 100644 --- a/src/system/detail/src/vnodeUtil.c +++ b/src/system/detail/src/vnodeUtil.c @@ -567,7 +567,7 @@ int32_t vnodeIncQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterSidExtInfo** pSid * check if the numOfQueries is 0 or not. */ pMeterObjList[(*numOfInc)++] = pMeter; - __sync_fetch_and_add(&pMeter->numOfQueries, 1); + atomic_fetch_add_32(&pMeter->numOfQueries, 1); // output for meter more than one query executed if (pMeter->numOfQueries > 1) { @@ -591,7 +591,7 @@ void vnodeDecQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterObj** pMeterObjList, SMeterObj* pMeter = pMeterObjList[i]; if (pMeter != NULL) { // here, do not need to lock to perform operations - __sync_fetch_and_sub(&pMeter->numOfQueries, 1); + atomic_fetch_sub_32(&pMeter->numOfQueries, 1); if (pMeter->numOfQueries > 0) { dTrace("qmsg:%p, vid:%d sid:%d id:%s dec query ref, numOfQueries:%d", pQueryMsg, pMeter->vnode, pMeter->sid, @@ -646,7 +646,7 @@ void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj) { } int32_t vnodeSetMeterState(SMeterObj* pMeterObj, int32_t state) { - return __sync_val_compare_and_swap(&pMeterObj->state, TSDB_METER_STATE_READY, state); + return atomic_val_compare_exchange_32(&pMeterObj->state, TSDB_METER_STATE_READY, state); } void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 7804cb0d0e..e92ac5d46d 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -516,7 +516,7 @@ static SDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SDataNode *pNode, char *k pNewNode->addTime = taosGetTimestampMs(); pNewNode->time = pNewNode->addTime + keepTime; - __sync_add_and_fetch_32(&pNewNode->refCount, 1); + atomic_add_fetch_32(&pNewNode->refCount, 1); // the address of this node may be changed, so the prev and next element should update the corresponding pointer taosUpdateInHashTable(pObj, pNewNode); @@ -529,7 +529,7 @@ static SDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SDataNode *pNode, char *k return NULL; } - __sync_add_and_fetch_32(&pNewNode->refCount, 1); + atomic_add_fetch_32(&pNewNode->refCount, 1); assert(hashVal == (*pObj->hashFp)(key, keyLen - 1)); pNewNode->hashVal = hashVal; @@ -558,7 +558,7 @@ static FORCE_INLINE SDataNode *taosAddToCacheImpl(SCacheObj *pObj, char *key, ui return NULL; } - __sync_add_and_fetch_32(&pNode->refCount, 1); + atomic_add_fetch_32(&pNode->refCount, 1); pNode->hashVal = (*pObj->hashFp)(key, keyLen - 1); taosAddNodeToHashTable(pObj, pNode); @@ -616,7 +616,7 @@ static FORCE_INLINE void taosDecRef(SDataNode *pNode) { } if (pNode->refCount > 0) { - __sync_sub_and_fetch_32(&pNode->refCount, 1); + atomic_sub_fetch_32(&pNode->refCount, 1); pTrace("key:%s is released by app.refcnt:%d", pNode->key, pNode->refCount); } else { /* @@ -676,20 +676,20 @@ void *taosGetDataFromCache(void *handle, char *key) { SDataNode *ptNode = taosGetNodeFromHashTable(handle, key, keyLen); if (ptNode != NULL) { - __sync_add_and_fetch_32(&ptNode->refCount, 1); + atomic_add_fetch_32(&ptNode->refCount, 1); } __cache_unlock(pObj); if (ptNode != NULL) { - __sync_add_and_fetch_32(&pObj->statistics.hitCount, 1); + atomic_add_fetch_32(&pObj->statistics.hitCount, 1); pTrace("key:%s is retrieved from cache,refcnt:%d", key, ptNode->refCount); } else { - __sync_add_and_fetch_32(&pObj->statistics.missCount, 1); + atomic_add_fetch_32(&pObj->statistics.missCount, 1); pTrace("key:%s not in cache,retrieved failed", key); } - __sync_add_and_fetch_32(&pObj->statistics.totalAccess, 1); + atomic_add_fetch_32(&pObj->statistics.totalAccess, 1); return (ptNode != NULL) ? ptNode->data : NULL; } diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index c6b56919e9..ebb6b9e0b8 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -60,7 +60,7 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%u-%u"); - snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), __sync_add_and_fetch_32(&tmpFileSerialNum, 1)); + snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1)); } /* diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 02e30be33a..b88cb7dc17 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -381,7 +381,7 @@ void tprintf(const char *const flags, int dflag, const char *const format, ...) } if (taosLogMaxLines > 0) { - __sync_add_and_fetch_32(&taosLogLines, 1); + atomic_add_fetch_32(&taosLogLines, 1); if ((taosLogLines > taosLogMaxLines) && (openInProgress == 0)) taosOpenNewLogFile(); } @@ -458,7 +458,7 @@ void taosPrintLongString(const char *const flags, int dflag, const char *const f taosPushLogBuffer(logHandle, buffer, len); if (taosLogMaxLines > 0) { - __sync_add_and_fetch_32(&taosLogLines, 1); + atomic_add_fetch_32(&taosLogLines, 1); if ((taosLogLines > taosLogMaxLines) && (openInProgress == 0)) taosOpenNewLogFile(); } diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 97e2189153..cf09aaee5e 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -105,15 +105,15 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = __sync_add_and_fetch_ptr(&nextTimerId, 1); + id = atomic_add_fetch_ptr(&nextTimerId, 1); } while (id == 0); return id; } -static void timerAddRef(tmr_obj_t* timer) { __sync_add_and_fetch_8(&timer->refCount, 1); } +static void timerAddRef(tmr_obj_t* timer) { atomic_add_fetch_8(&timer->refCount, 1); } static void timerDecRef(tmr_obj_t* timer) { - if (__sync_sub_and_fetch_8(&timer->refCount, 1) == 0) { + if (atomic_sub_fetch_8(&timer->refCount, 1) == 0) { free(timer); } } @@ -121,7 +121,7 @@ static void timerDecRef(tmr_obj_t* timer) { static void lockTimerList(timer_list_t* list) { int64_t tid = taosGetPthreadId(); int i = 0; - while (__sync_val_compare_and_swap_64(&(list->lockedBy), 0, tid) != 0) { + while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { sched_yield(); } @@ -130,7 +130,7 @@ static void lockTimerList(timer_list_t* list) { static void unlockTimerList(timer_list_t* list) { int64_t tid = taosGetPthreadId(); - if (__sync_val_compare_and_swap_64(&(list->lockedBy), tid, 0) != tid) { + if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { assert(false); tmrError("%d trying to unlock a timer list not locked by current thread.", tid); } @@ -257,7 +257,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { static void processExpiredTimer(void* handle, void* arg) { tmr_obj_t* timer = (tmr_obj_t*)handle; timer->executedBy = taosGetPthreadId(); - uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); + uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] execution start."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); @@ -431,7 +431,7 @@ bool taosTmrStop(tmr_h timerId) { return false; } - uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); + uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); doStopTimer(timer, state); timerDecRef(timer); @@ -456,7 +456,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, if (timer == NULL) { tmrTrace("%s timer[id=%lld] does not exist", ctrl->label, id); } else { - uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); + uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); if (!doStopTimer(timer, state)) { timerDecRef(timer); timer = NULL; From fa0f056f192ea72d74d734d63b372319a64573f1 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 12 Nov 2019 17:21:36 +0800 Subject: [PATCH 28/69] refactor some codes. fix bugs in selectivity+tags/ts query. --- src/client/src/tscFunctionImpl.c | 702 +++++++++++++++++-------- src/client/src/tscSQLParser.c | 77 ++- src/client/src/tscUtil.c | 69 +-- src/system/detail/src/vnodeQueryImpl.c | 13 +- src/system/detail/src/vnodeUtil.c | 2 +- 5 files changed, 587 insertions(+), 276 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 35011a259b..a29f8f1589 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -63,6 +63,14 @@ } \ } while (0); +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ +do {\ +for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ + } \ +} while(0); + void noop(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} typedef struct tValuePair { @@ -104,7 +112,9 @@ typedef struct SFirstLastInfo { } SFirstLastInfo; typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { tMemBucket *pMemBucket; } SPercentileInfo; +typedef struct SPercentileInfo { + tMemBucket *pMemBucket; +} SPercentileInfo; typedef struct STopBotInfo { int32_t num; @@ -118,9 +128,13 @@ typedef struct SLeastsquareInfo { int64_t num; } SLeastsquareInfo; -typedef struct SAPercentileInfo { SHistogramInfo *pHisto; } SAPercentileInfo; +typedef struct SAPercentileInfo { + SHistogramInfo *pHisto; +} SAPercentileInfo; -typedef struct STSCompInfo { STSBuf *pTSBuf; } STSCompInfo; +typedef struct STSCompInfo { + STSBuf *pTSBuf; +} STSCompInfo; int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *bytes, int16_t *intermediateResBytes, int16_t extLength, bool isSuperTable) { @@ -451,21 +465,32 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId } \ }; -#define UPDATE_DATA(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = right; \ - DO_UPDATE_TAG_COLUMNS(ctx, 0); \ - (num) += 1; \ - } \ - } while (0) +#define UPDATE_DATA(ctx, left, right, num, sign, k) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_TAG_COLUMNS(ctx, k); \ + (num) += 1; \ + } \ + } while (0); + +#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ +do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ + (num) += 1; \ + } \ + } while (0); + #define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ for (int32_t i = 0; i < ((ctx)->size); ++i) { \ if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - UPDATE_DATA(ctx, val, (list)[i], num, sign); \ + TSKEY key = (ctx)->ptsList[i]; \ + UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } #define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ @@ -886,16 +911,18 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, index = pCtx->preAggVals.maxIndex; } + TSKEY key = pCtx->ptsList[index]; + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t val = GET_INT64_VAL(tval); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *data = (int8_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *data = (int16_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *data = (int32_t *)pOutput; #if defined(_DEBUG_VIEW) @@ -906,27 +933,27 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, *data = val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - if (__ctx->functionId == TSDB_FUNC_TAG_DUMMY) { - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); - } else if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - *((int64_t *)__ctx->aOutputBuf) = pCtx->ptsList[index]; + if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { + __ctx->tag = (tVariant){.i64Key = key, .nType = TSDB_DATA_TYPE_BIGINT}; } + + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } } } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { int64_t *data = (int64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *data = (double *)pOutput; double val = GET_DOUBLE_VAL(tval); - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *data = (float *)pOutput; double val = GET_DOUBLE_VAL(tval); - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } return; @@ -951,7 +978,9 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[i]); + TSKEY k = pCtx->ptsList[i]; + + DO_UPDATE_TAG_COLUMNS(pCtx, k); } *notNullElems += 1; @@ -1089,12 +1118,12 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t v = GET_INT8_VAL(input); - UPDATE_DATA(pCtx, *(int8_t *)output, v, notNullElems, isMin); + DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); break; }; case TSDB_DATA_TYPE_SMALLINT: { int16_t v = GET_INT16_VAL(input); - UPDATE_DATA(pCtx, *(int16_t *)output, v, notNullElems, isMin); + DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_INT: { @@ -1104,7 +1133,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } notNullElems++; @@ -1113,17 +1142,17 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } case TSDB_DATA_TYPE_FLOAT: { float v = GET_FLOAT_VAL(input); - UPDATE_DATA(pCtx, *(float *)output, v, notNullElems, isMin); + DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_DOUBLE: { double v = GET_DOUBLE_VAL(input); - UPDATE_DATA(pCtx, *(double *)output, v, notNullElems, isMin); + DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_BIGINT: { int64_t v = GET_INT64_VAL(input); - UPDATE_DATA(pCtx, *(int64_t *)output, v, notNullElems, isMin); + DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); break; }; default: @@ -1179,38 +1208,39 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + TSKEY key = pCtx->ptsList[index]; int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *output = (int8_t *)pCtx->aOutputBuf; int8_t i = GET_INT8_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *output = pCtx->aOutputBuf; int16_t i = GET_INT16_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *output = pCtx->aOutputBuf; int32_t i = GET_INT32_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { int64_t *output = pCtx->aOutputBuf; int64_t i = GET_INT64_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *output = pCtx->aOutputBuf; float i = GET_FLOAT_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *output = pCtx->aOutputBuf; double i = GET_DOUBLE_VAL(pData); - UPDATE_DATA(pCtx, *output, i, num, isMin); + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; @@ -1804,6 +1834,11 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { + SQLFunctionCtx* __ctx = pTagInfo->pTagCtxList[i]; + if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { + __ctx->tag = (tVariant) {.nType = TSDB_DATA_TYPE_BIGINT, .i64Key = tsKey}; + } + tVariantDump(&pTagInfo->pTagCtxList[i]->tag, dst->pTags + size, pTagInfo->pTagCtxList[i]->tag.nType); size += pTagInfo->pTagCtxList[i]->outputBytes; } @@ -1825,8 +1860,9 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, tValuePair **pList = pInfo->res; if (pInfo->num < maxLen) { - if (pInfo->num == 0 || ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && - val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || + if (pInfo->num == 0 || + ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && + val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); @@ -4293,173 +4329,427 @@ int32_t funcCompatDefList[28] = { */ 1, 1, 1, -1, 1, 1, 5}; -SQLAggFuncElem aAggs[28] = { - { - // 0, count function does not invoke the finalize function - "count", TSDB_FUNC_COUNT, TSDB_FUNC_COUNT, TSDB_BASE_FUNC_SO, function_setup, count_function, count_function_f, - no_next_step, noop, count_func_merge, count_func_merge, count_load_data_info, - }, - { - // 1 - "sum", TSDB_FUNC_SUM, TSDB_FUNC_SUM, TSDB_BASE_FUNC_SO, function_setup, sum_function, sum_function_f, - no_next_step, function_finalizer, sum_func_merge, sum_func_second_merge, precal_req_load_info, - }, - { - // 2 - "avg", TSDB_FUNC_AVG, TSDB_FUNC_AVG, TSDB_BASE_FUNC_SO, function_setup, avg_function, avg_function_f, - no_next_step, avg_finalizer, avg_func_merge, avg_func_second_merge, precal_req_load_info, - }, - { - // 3 - "min", TSDB_FUNC_MIN, TSDB_FUNC_MIN, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, min_func_setup, - min_function, min_function_f, no_next_step, function_finalizer, min_func_merge, min_func_second_merge, - precal_req_load_info, - }, - { - // 4 - "max", TSDB_FUNC_MAX, TSDB_FUNC_MAX, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, max_func_setup, - max_function, max_function_f, no_next_step, function_finalizer, max_func_merge, max_func_second_merge, - precal_req_load_info, - }, - { - // 5 - "stddev", TSDB_FUNC_STDDEV, TSDB_FUNC_INVALID_ID, TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - function_setup, stddev_function, stddev_function_f, stddev_next_step, stddev_finalizer, noop, noop, - data_req_load_info, - }, - { - // 6 - "percentile", TSDB_FUNC_PERCT, TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, percentile_function_setup, percentile_function, - percentile_function_f, no_next_step, percentile_finalizer, noop, noop, data_req_load_info, - }, - { - // 7 - "apercentile", TSDB_FUNC_APERCT, TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, - apercentile_function_setup, apercentile_function, apercentile_function_f, no_next_step, apercentile_finalizer, - apercentile_func_merge, apercentile_func_second_merge, data_req_load_info, - }, - { - // 8 - "first", TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, - first_function, first_function_f, no_next_step, function_finalizer, noop, noop, first_data_req_info, - }, - { - // 9 - "last", TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, - last_function, last_function_f, no_next_step, function_finalizer, noop, noop, last_data_req_info, - }, - { - // 10 - "last_row", TSDB_FUNC_LAST_ROW, TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, last_row_function, noop, no_next_step, last_row_finalizer, noop, - last_dist_func_second_merge, data_req_load_info, - }, - { - // 11 - "top", TSDB_FUNC_TOP, TSDB_FUNC_TOP, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | - TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, top_function, top_function_f, no_next_step, top_bottom_func_finalizer, - top_func_merge, top_func_second_merge, data_req_load_info, - }, - { - // 12 - "bottom", TSDB_FUNC_BOTTOM, TSDB_FUNC_BOTTOM, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | - TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, bottom_function, bottom_function_f, no_next_step, top_bottom_func_finalizer, - bottom_func_merge, bottom_func_second_merge, data_req_load_info, - }, - { - // 13 - "spread", TSDB_FUNC_SPREAD, TSDB_FUNC_SPREAD, TSDB_BASE_FUNC_SO, spread_function_setup, spread_function, - spread_function_f, no_next_step, spread_function_finalizer, spread_func_merge, spread_func_sec_merge, - count_load_data_info, - }, - { - // 14 - "twa", TSDB_FUNC_TWA, TSDB_FUNC_TWA, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, twa_function_setup, - twa_function, twa_function_f, no_next_step, twa_function_finalizer, twa_func_merge, twa_function_copy, - data_req_load_info, - }, - { - // 15 - "leastsquares", TSDB_FUNC_LEASTSQR, TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, leastsquares_function_setup, - leastsquares_function, leastsquares_function_f, no_next_step, leastsquares_finalizer, noop, noop, - data_req_load_info, - }, - { - // 16 - "ts", TSDB_FUNC_TS, TSDB_FUNC_TS, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, - date_col_output_function, date_col_output_function, no_next_step, noop, copy_function, copy_function, - no_data_info, - }, - { - // 17 - "ts", TSDB_FUNC_TS_DUMMY, TSDB_FUNC_TS_DUMMY, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, noop, - noop, no_next_step, noop, copy_function, copy_function, data_req_load_info, - }, - { - // 18 - "tag", TSDB_FUNC_TAG_DUMMY, TSDB_FUNC_TAG_DUMMY, TSDB_BASE_FUNC_SO, function_setup, tag_function, noop, - no_next_step, noop, copy_function, copy_function, no_data_info, - }, - { - // 19 - "ts", TSDB_FUNC_TS_COMP, TSDB_FUNC_TS_COMP, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, ts_comp_function_setup, - ts_comp_function, ts_comp_function_f, no_next_step, ts_comp_finalize, copy_function, copy_function, - data_req_load_info, - }, - { - // 20 - "tag", TSDB_FUNC_TAG, TSDB_FUNC_TAG, TSDB_BASE_FUNC_SO, function_setup, tag_function, tag_function_f, - no_next_step, noop, copy_function, copy_function, no_data_info, - }, - { - // 21, column project sql function - "colprj", TSDB_FUNC_PRJ, TSDB_FUNC_PRJ, TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, function_setup, - col_project_function, col_project_function_f, no_next_step, noop, copy_function, copy_function, - data_req_load_info, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", TSDB_FUNC_TAGPRJ, TSDB_FUNC_TAGPRJ, TSDB_BASE_FUNC_MO, function_setup, tag_project_function, - tag_project_function_f, no_next_step, noop, copy_function, copy_function, no_data_info, - }, - { - // 23 - "arithmetic", TSDB_FUNC_ARITHM, TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, function_setup, arithmetic_function, - arithmetic_function_f, no_next_step, noop, copy_function, copy_function, data_req_load_info, - }, - { - // 24 - "diff", TSDB_FUNC_DIFF, TSDB_FUNC_INVALID_ID, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, diff_function_setup, - diff_function, diff_function_f, no_next_step, noop, noop, noop, data_req_load_info, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", TSDB_FUNC_FIRST_DST, TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, - first_dist_function, first_dist_function_f, no_next_step, function_finalizer, first_dist_func_merge, - first_dist_func_second_merge, first_dist_data_req_info, - }, - { - // 26 - "last_dist", TSDB_FUNC_LAST_DST, TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, - last_dist_function, last_dist_function_f, no_next_step, function_finalizer, last_dist_func_merge, - last_dist_func_second_merge, last_dist_data_req_info, - }, - { - // 27 - "interp", TSDB_FUNC_INTERP, TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, function_setup, - interp_function, - do_sum_f, // todo filter handle - no_next_step, noop, noop, copy_function, no_data_info, - }}; +SQLAggFuncElem aAggs[28] = {{ + // 0, count function does not invoke the finalize function + "count", + TSDB_FUNC_COUNT, + TSDB_FUNC_COUNT, + TSDB_BASE_FUNC_SO, + function_setup, + count_function, + count_function_f, + no_next_step, + noop, + count_func_merge, + count_func_merge, + count_load_data_info, + }, + { + // 1 + "sum", + TSDB_FUNC_SUM, + TSDB_FUNC_SUM, + TSDB_BASE_FUNC_SO, + function_setup, + sum_function, + sum_function_f, + no_next_step, + function_finalizer, + sum_func_merge, + sum_func_second_merge, + precal_req_load_info, + }, + { + // 2 + "avg", + TSDB_FUNC_AVG, + TSDB_FUNC_AVG, + TSDB_BASE_FUNC_SO, + function_setup, + avg_function, + avg_function_f, + no_next_step, + avg_finalizer, + avg_func_merge, + avg_func_second_merge, + precal_req_load_info, + }, + { + // 3 + "min", + TSDB_FUNC_MIN, + TSDB_FUNC_MIN, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + min_func_setup, + min_function, + min_function_f, + no_next_step, + function_finalizer, + min_func_merge, + min_func_second_merge, + precal_req_load_info, + }, + { + // 4 + "max", + TSDB_FUNC_MAX, + TSDB_FUNC_MAX, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + max_func_setup, + max_function, + max_function_f, + no_next_step, + function_finalizer, + max_func_merge, + max_func_second_merge, + precal_req_load_info, + }, + { + // 5 + "stddev", + TSDB_FUNC_STDDEV, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + function_setup, + stddev_function, + stddev_function_f, + stddev_next_step, + stddev_finalizer, + noop, + noop, + data_req_load_info, + }, + { + // 6 + "percentile", + TSDB_FUNC_PERCT, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + percentile_function_setup, + percentile_function, + percentile_function_f, + no_next_step, + percentile_finalizer, + noop, + noop, + data_req_load_info, + }, + { + // 7 + "apercentile", + TSDB_FUNC_APERCT, + TSDB_FUNC_APERCT, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, + apercentile_function_setup, + apercentile_function, + apercentile_function_f, + no_next_step, + apercentile_finalizer, + apercentile_func_merge, + apercentile_func_second_merge, + data_req_load_info, + }, + { + // 8 + "first", + TSDB_FUNC_FIRST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + first_function, + first_function_f, + no_next_step, + function_finalizer, + noop, + noop, + first_data_req_info, + }, + { + // 9 + "last", + TSDB_FUNC_LAST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + last_function, + last_function_f, + no_next_step, + function_finalizer, + noop, + noop, + last_data_req_info, + }, + { + // 10 + "last_row", + TSDB_FUNC_LAST_ROW, + TSDB_FUNC_LAST_ROW, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_row_function, + noop, + no_next_step, + last_row_finalizer, + noop, + last_dist_func_second_merge, + data_req_load_info, + }, + { + // 11 + "top", + TSDB_FUNC_TOP, + TSDB_FUNC_TOP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + top_function, + top_function_f, + no_next_step, + top_bottom_func_finalizer, + top_func_merge, + top_func_second_merge, + data_req_load_info, + }, + { + // 12 + "bottom", + TSDB_FUNC_BOTTOM, + TSDB_FUNC_BOTTOM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + bottom_function, + bottom_function_f, + no_next_step, + top_bottom_func_finalizer, + bottom_func_merge, + bottom_func_second_merge, + data_req_load_info, + }, + { + // 13 + "spread", + TSDB_FUNC_SPREAD, + TSDB_FUNC_SPREAD, + TSDB_BASE_FUNC_SO, + spread_function_setup, + spread_function, + spread_function_f, + no_next_step, + spread_function_finalizer, + spread_func_merge, + spread_func_sec_merge, + count_load_data_info, + }, + { + // 14 + "twa", + TSDB_FUNC_TWA, + TSDB_FUNC_TWA, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + twa_function_setup, + twa_function, + twa_function_f, + no_next_step, + twa_function_finalizer, + twa_func_merge, + twa_function_copy, + data_req_load_info, + }, + { + // 15 + "leastsquares", + TSDB_FUNC_LEASTSQR, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + leastsquares_function_setup, + leastsquares_function, + leastsquares_function_f, + no_next_step, + leastsquares_finalizer, + noop, + noop, + data_req_load_info, + }, + { + // 16 + "ts", + TSDB_FUNC_TS, + TSDB_FUNC_TS, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + date_col_output_function, + date_col_output_function, + no_next_step, + noop, + copy_function, + copy_function, + no_data_info, + }, + { + // 17 + "ts", + TSDB_FUNC_TS_DUMMY, + TSDB_FUNC_TS_DUMMY, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + noop, + noop, + no_next_step, + noop, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 18 + "tag", + TSDB_FUNC_TAG_DUMMY, + TSDB_FUNC_TAG_DUMMY, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + noop, + no_next_step, + noop, + copy_function, + copy_function, + no_data_info, + }, + { + // 19 + "ts", + TSDB_FUNC_TS_COMP, + TSDB_FUNC_TS_COMP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + ts_comp_function_setup, + ts_comp_function, + ts_comp_function_f, + no_next_step, + ts_comp_finalize, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 20 + "tag", + TSDB_FUNC_TAG, + TSDB_FUNC_TAG, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + tag_function_f, + no_next_step, + noop, + copy_function, + copy_function, + no_data_info, + }, + { + // 21, column project sql function + "colprj", + TSDB_FUNC_PRJ, + TSDB_FUNC_PRJ, + TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + col_project_function, + col_project_function_f, + no_next_step, + noop, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 22, multi-output, tag function has only one result + "tagprj", + TSDB_FUNC_TAGPRJ, + TSDB_FUNC_TAGPRJ, + TSDB_BASE_FUNC_MO, + function_setup, + tag_project_function, + tag_project_function_f, + no_next_step, + noop, + copy_function, + copy_function, + no_data_info, + }, + { + // 23 + "arithmetic", + TSDB_FUNC_ARITHM, + TSDB_FUNC_ARITHM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + arithmetic_function, + arithmetic_function_f, + no_next_step, + noop, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 24 + "diff", + TSDB_FUNC_DIFF, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + diff_function_setup, + diff_function, + diff_function_f, + no_next_step, + noop, + noop, + noop, + data_req_load_info, + }, + // distributed version used in two-stage aggregation processes + { + // 25 + "first_dist", + TSDB_FUNC_FIRST_DST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + first_dist_function, + first_dist_function_f, + no_next_step, + function_finalizer, + first_dist_func_merge, + first_dist_func_second_merge, + first_dist_data_req_info, + }, + { + // 26 + "last_dist", + TSDB_FUNC_LAST_DST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_dist_function, + last_dist_function_f, + no_next_step, + function_finalizer, + last_dist_func_merge, + last_dist_func_second_merge, + last_dist_data_req_info, + }, + { + // 27 + "interp", + TSDB_FUNC_INTERP, + TSDB_FUNC_INTERP, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + interp_function, + do_sum_f, // todo filter handle + no_next_step, + noop, + noop, + copy_function, + no_data_info, + }}; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ae9704effa..1bc0163e5f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -81,7 +81,7 @@ static bool validateIpAddress(char* ip); static bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd); static bool functionCompatibleCheck(SSqlCmd* pCmd); -static void setColumnOffsetValueInResultset(SSqlCmd* pCmd); +static void setColumnOffsetValueInResultset(SSqlCmd* pCmd); static int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList); static int32_t parseIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql); @@ -94,7 +94,7 @@ static int32_t parseFillClause(SSqlCmd* pCmd, SQuerySQL* pQuerySQL); static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, int32_t numOfCols); static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd); static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString); @@ -105,8 +105,8 @@ static int32_t validateLocalConfig(tDCLSQL* pOptions); static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool hasTimestampForPointInterpQuery(SSqlCmd* pCmd); -static void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex); +static bool hasTimestampForPointInterpQuery(SSqlCmd* pCmd); +static void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex); static int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql); static int32_t parseCreateDBOptions(SCreateDBInfo* pCreateDbSql, SSqlCmd* pCmd); @@ -115,13 +115,12 @@ static int32_t getTableIndexByName(SSQLToken* pToken, SSqlCmd* pCmd, SColumnInde static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); -static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex); -static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql); +static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex); +static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql); static int32_t tscQueryOnlyMetricTags(SSqlCmd* pCmd, bool* queryOnMetricTags) { assert(QUERY_IS_STABLE_QUERY(pCmd->type)); - // here colIdx == -1 means the special column tbname that is the name of each table *queryOnMetricTags = true; for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); @@ -1151,7 +1150,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { } // check the invalid sql expresssion: select count(tbname)/count(tag1)/count(tag2) from super_table interval(1d); - for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { setErrMsg(pCmd, msg1); @@ -2791,10 +2790,14 @@ static bool functionCompatibleCheck(SSqlCmd* pCmd) { // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions for (int32_t i = startIdx + 1; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_TS) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + + int16_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) { + continue; + } + + if (functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { continue; } @@ -2809,9 +2812,7 @@ static bool functionCompatibleCheck(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_PRJ || - functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TS || + if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_ARITHM) { continue; } @@ -4986,19 +4987,37 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd) { int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) { bool isProjectionFunction = false; - const char* msg = "column projection is not compatible with interval"; - + const char* msg1 = "column projection is not compatible with interval"; + const char* msg2 = "interval not allowed for tag queries"; + // multi-output set/ todo refactor for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, k); + + // projection query on primary timestamp, the selectivity function needs to be present. + if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + bool hasSelectivity = false; + for(int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { + SSqlExpr* pEx = tscSqlExprGet(pCmd, j); + if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { + hasSelectivity = true; + break; + } + } + + if (hasSelectivity) { + continue; + } + } + if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || - pExpr->functionId == TSDB_FUNC_ARITHM) { + pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } } if (isProjectionFunction) { - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg1); } return isProjectionFunction == true ? TSDB_CODE_INVALID_SQL : TSDB_CODE_SUCCESS; @@ -5164,8 +5183,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { bool queryOnTags = false; - int32_t ret = tscQueryOnlyMetricTags(pCmd, &queryOnTags); - if (ret != TSDB_CODE_SUCCESS) { + if (tscQueryOnlyMetricTags(pCmd, &queryOnTags) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -5382,8 +5400,8 @@ static void doUpdateSqlFunctionForTagPrj(SSqlCmd* pCmd) { } } - int16_t resType = 0; - int16_t resBytes = 0; + int16_t resType = 0; + int16_t resBytes = 0; SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); @@ -5464,7 +5482,7 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { const char* msg2 = "functions not allowed"; bool tagColExists = false; - int16_t numOfTimestamp = 0; // primary timestamp column + int16_t numOfTimestamp = 0; // primary timestamp column int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; @@ -5493,7 +5511,7 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { // When the tag projection function on tag column that is not in the group by clause, aggregation function and // selectivity function exist in select clause is not allowed. - if(numOfAggregation > 0) { + if (numOfAggregation > 0) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } @@ -5598,14 +5616,14 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { const char* msg2 = "interval not allowed in group by normal column"; const char* msg3 = "group by not allowed on projection query"; const char* msg4 = "tags retrieve not compatible with group by"; - const char* msg5 = "retrieve tags not compatible with group by "; + const char* msg5 = "retrieve tags not compatible with group by or interval query"; SSqlCmd* pCmd = &pSql->cmd; SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); // only retrieve tags, group by is not supportted if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { - if (pCmd->groupbyExpr.numOfGroupCols > 0) { + if (pCmd->groupbyExpr.numOfGroupCols > 0 || pCmd->nAggTimeInterval > 0) { setErrMsg(pCmd, msg5); return TSDB_CODE_INVALID_SQL; } else { @@ -5634,7 +5652,7 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { * group by normal columns. * Check if the column projection is identical to the group by column or not */ - if (functId == TSDB_FUNC_PRJ) { + if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool qualified = false; for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { SColIndexEx* pColIndex = &pCmd->groupbyExpr.columnInfo[j]; @@ -5650,7 +5668,8 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { } if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && - functId != TSDB_FUNC_TAGPRJ) { + functId != TSDB_FUNC_TAGPRJ && + (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX)) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 547171cffe..fd23b183b1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -142,7 +142,6 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) { return false; } - void tscGetDBInfoFromMeterId(char* meterId, char* db) { char* st = strstr(meterId, TS_PATH_DELIMITER); if (st != NULL) { @@ -265,7 +264,7 @@ bool tscIsPointInterpQuery(SSqlCmd* pCmd) { } bool tscIsTWAQuery(SSqlCmd* pCmd) { - for(int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); if (pExpr == NULL) { continue; @@ -450,7 +449,8 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock); } -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset) { +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, + uint32_t offset) { uint32_t needed = pDataBlock->numOfParams + 1; if (needed > pDataBlock->numOfAllocedParams) { needed *= 2; @@ -490,13 +490,13 @@ SDataBlockList* tscCreateBlockArrayList() { return pDataBlockArrayList; } -void tscAppendDataBlock(SDataBlockList *pList, STableDataBlocks *pBlocks) { +void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) { if (pList->nSize >= pList->nAlloc) { pList->nAlloc = pList->nAlloc << 1; - pList->pData = realloc(pList->pData, sizeof(void *) * (size_t)pList->nAlloc); + pList->pData = realloc(pList->pData, sizeof(void*) * (size_t)pList->nAlloc); // reset allocated memory - memset(pList->pData + pList->nSize, 0, sizeof(void *) * (pList->nAlloc - pList->nSize)); + memset(pList->pData + pList->nSize, 0, sizeof(void*) * (pList->nAlloc - pList->nSize)); } pList->pData[pList->nSize++] = pBlocks; @@ -553,7 +553,7 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) { } STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name) { - STableDataBlocks *dataBuf = tscCreateDataBlock(size); + STableDataBlocks* dataBuf = tscCreateDataBlock(size); dataBuf->rowSize = rowSize; dataBuf->size = startOffset; @@ -573,7 +573,7 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData } if (dataBuf == NULL) { - dataBuf = tscCreateDataBlockEx((size_t) size, rowSize, startOffset, tableId); + dataBuf = tscCreateDataBlockEx((size_t)size, rowSize, startOffset, tableId); dataBuf = *(STableDataBlocks**)taosAddIntHash(pHashList, id, (char*)&dataBuf); tscAppendDataBlock(pDataBlockList, dataBuf); } @@ -604,7 +604,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi if (tmp != NULL) { dataBuf->pData = tmp; memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); - } else { // failed to allocate memory, free already allocated memory and return error code + } else { // failed to allocate memory, free already allocated memory and return error code tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize); taosCleanUpIntHash(pVnodeDataBlockHashList); @@ -673,7 +673,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { pCmd->allocSize = size; } else { if (pCmd->allocSize < size) { - char* b = realloc(pCmd->payload, size); + char* b = realloc(pCmd->payload, size); if (b == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY; pCmd->payload = b; pCmd->allocSize = size; @@ -869,11 +869,11 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) { static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) { if (size > pExprInfo->numOfAlloc) { - int32_t oldSize = pExprInfo->numOfAlloc; + uint32_t oldSize = pExprInfo->numOfAlloc; - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); + uint32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1U); while (newSize < size) { - newSize = (newSize << 1); + newSize = (newSize << 1U); } if (newSize > TSDB_MAX_COLUMNS) { @@ -1161,7 +1161,7 @@ void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS); for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) { - SColumnBase *pColBase = &(pColumnBaseInfo->pColList[i]); + SColumnBase* pColBase = &(pColumnBaseInfo->pColList[i]); if (pColBase->numOfFilters > 0) { for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { @@ -1179,8 +1179,9 @@ void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { tfree(pColumnBaseInfo->pColList); } - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { _cf_ensureSpace(pColumnBaseInfo, size); } +void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { + _cf_ensureSpace(pColumnBaseInfo, size); +} /* * 1. normal name, not a keyword or number @@ -1228,16 +1229,16 @@ int32_t tscValidateName(SSQLToken* pToken) { int len = tSQLGetToken(pToken->z, &pToken->type); // single token, validate it - if (len == pToken->n){ + if (len == pToken->n) { return validateQuoteToken(pToken); } else { - sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { - return TSDB_CODE_INVALID_SQL; - } + sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); + if (sep == NULL) { + return TSDB_CODE_INVALID_SQL; + } return tscValidateName(pToken); - } + } } else { if (isNumber(pToken)) { return TSDB_CODE_INVALID_SQL; @@ -1616,8 +1617,8 @@ int32_t SStringAlloc(SString* pStr, int32_t size) { #ifdef WINDOWS LPVOID lpMsgBuf; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); + GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR)&lpMsgBuf, 0, NULL); tscTrace("failed to allocate memory, reason:%s", lpMsgBuf); LocalFree(lpMsgBuf); #else @@ -1652,12 +1653,11 @@ int32_t SStringEnsureRemain(SString* pStr, int32_t size) { char* tmp = realloc(pStr->z, newsize); if (tmp == NULL) { - #ifdef WINDOWS LPVOID lpMsgBuf; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); + GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR)&lpMsgBuf, 0, NULL); tscTrace("failed to allocate memory, reason:%s", lpMsgBuf); LocalFree(lpMsgBuf); #else @@ -1728,7 +1728,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex if (pPrevSql != NULL) { pNew->cmd.type = pPrevSql->cmd.type; } else { - pNew->cmd.type |= TSDB_QUERY_TYPE_SUBQUERY; // it must be the subquery + pNew->cmd.type |= TSDB_QUERY_TYPE_SUBQUERY; // it must be the subquery } uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; @@ -1760,7 +1760,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex char key[TSDB_MAX_TAGS_LEN + 1] = {0}; tscGetMetricMetaCacheKey(pCmd, key, pMetermetaInfo->pMeterMeta->uid); - char* name = pMeterMetaInfo->name; + char* name = pMeterMetaInfo->name; SMeterMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { @@ -1768,11 +1768,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex SMetricMeta* pMetricMeta = taosGetDataFromCache(tscCacheHandle, key); pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pMeterMeta, pMetricMeta, pMeterMetaInfo->numOfTags, - pMeterMetaInfo->tagColumnIndex); + pMeterMetaInfo->tagColumnIndex); } else { SMeterMetaInfo* pPrevInfo = tscGetMeterMetaInfo(&pPrevSql->cmd, 0); - pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pPrevInfo->pMeterMeta, pPrevInfo->pMetricMeta, pMeterMetaInfo->numOfTags, - pMeterMetaInfo->tagColumnIndex); + pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pPrevInfo->pMeterMeta, pPrevInfo->pMetricMeta, + pMeterMetaInfo->numOfTags, pMeterMetaInfo->tagColumnIndex); pPrevInfo->pMeterMeta = NULL; pPrevInfo->pMetricMeta = NULL; @@ -1783,13 +1783,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex assert(pFinalInfo->pMetricMeta != NULL); } - tscTrace("%p new subquery %p, vnodeIdx:%d, tableIndex:%d, type:%d", pSql, pNew, vnodeIndex, tableIndex, pNew->cmd.type); + tscTrace("%p new subquery %p, vnodeIdx:%d, tableIndex:%d, type:%d", pSql, pNew, vnodeIndex, tableIndex, + pNew->cmd.type); return pNew; } void tscDoQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; - void* fp = pSql->fp; + void* fp = pSql->fp; if (pCmd->command > TSDB_SQL_LOCAL) { tscProcessLocalCmd(pSql); diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 9c0d7883a5..654f9537e4 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -3122,9 +3122,11 @@ static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || + functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAG_DUMMY) { continue; } + if (functionId != functId && functionId != functIdDst) { return false; } @@ -3137,10 +3139,9 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } -static void rewriteExecOrder(SQuery *pQuery, bool metricQuery) { +static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { // in case of point-interpolation query, use asc order scan - char msg[] = - "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, " + char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, " "new qrange:%lld-%lld"; // descending order query @@ -3614,7 +3615,7 @@ int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMete } setScanLimitationByResultBuffer(pQuery); - rewriteExecOrder(pQuery, false); + changeExecuteScanOrder(pQuery, false); pQInfo->over = 0; pQInfo->pointsRead = 0; @@ -3790,7 +3791,7 @@ int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) pQInfo->pointsRead = 0; pQuery->pointsRead = 0; - rewriteExecOrder(pQuery, true); + changeExecuteScanOrder(pQuery, true); vnodeInitDataBlockInfo(&pSupporter->runtimeEnv.loadBlockInfo); vnodeInitLoadCompBlockInfo(&pSupporter->runtimeEnv.loadCompBlockInfo); diff --git a/src/system/detail/src/vnodeUtil.c b/src/system/detail/src/vnodeUtil.c index 790ad1c4a1..7a2fb5530c 100644 --- a/src/system/detail/src/vnodeUtil.c +++ b/src/system/detail/src/vnodeUtil.c @@ -289,7 +289,7 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* return NULL; } - if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY) { + if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].resBytes; } assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes)); From 907fcfa09cb51fc595a19a001beece8a73bf4906 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 12 Nov 2019 18:05:47 +0800 Subject: [PATCH 29/69] issue #686 --- src/system/detail/src/mgmtUtil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/system/detail/src/mgmtUtil.c b/src/system/detail/src/mgmtUtil.c index be90180f68..3be2e1288d 100644 --- a/src/system/detail/src/mgmtUtil.c +++ b/src/system/detail/src/mgmtUtil.c @@ -92,5 +92,6 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { char dbName[TSDB_DB_NAME_LEN + 1] = {0}; extractDBName(db, dbName); - return (strncasecmp(dbName, monitordb, strlen(dbName)) == 0); + size_t len = strlen(dbName); + return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } From ce863c00f6a229152db51c2893fb0a8b393468a3 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 13 Nov 2019 10:14:35 +0800 Subject: [PATCH 30/69] [TBASE-1112] --- src/client/src/tscServer.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index aa80e505d5..67118b50e2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1332,7 +1332,7 @@ int tscBuildRetrieveMsg(SSqlObj *pSql) { *((uint64_t *)pMsg) = pSql->res.qhandle; pMsg += sizeof(pSql->res.qhandle); - *pMsg = htons(pSql->cmd.type); + *((uint16_t*)pMsg) = htons(pSql->cmd.type); pMsg += sizeof(pSql->cmd.type); msgLen = pMsg - pStart; @@ -3450,7 +3450,12 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { tscSetResultPointer(pCmd, pRes); pRes->row = 0; - if (pRes->numOfRows == 0 && !(tscProjectionQueryOnMetric(pCmd) && pRes->offset > 0)) { + /** + * If the query result is exhausted, the connection will be recycled. + * If current query is to free resource at server side, the connection will be recycle. + */ + if ((pRes->numOfRows == 0 && !(tscProjectionQueryOnMetric(pCmd) && pRes->offset > 0)) || + ((pCmd->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE)) { taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); pSql->thandle = NULL; } else { From 21427af360c9b90f8e57a3ea2e4896f41a598aab Mon Sep 17 00:00:00 2001 From: localvar Date: Wed, 13 Nov 2019 03:24:39 +0000 Subject: [PATCH 31/69] TBASE-306 enhance `taos_options` to support cocurrent calls. note it still a problem if one thread is modifying a global configuration while another thread is reading it. --- src/client/src/tscSystem.c | 91 +++++++++++++++++++++----------------- src/inc/tglobalcfg.h | 2 +- src/util/src/tglobalcfg.c | 13 +++--- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 5e383c9d0b..2842754e40 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -182,51 +182,45 @@ void taos_init_imp() { void taos_init() { pthread_once(&tscinit, taos_init_imp); } -int taos_options(TSDB_OPTION option, const void *arg, ...) { - char * pStr = NULL; - SGlobalConfig *cfg_configDir = tsGetConfigOption("configDir"); - SGlobalConfig *cfg_activetimer = tsGetConfigOption("shellActivityTimer"); - SGlobalConfig *cfg_locale = tsGetConfigOption("locale"); - SGlobalConfig *cfg_charset = tsGetConfigOption("charset"); - SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); - SGlobalConfig *cfg_socket = tsGetConfigOption("sockettype"); +static int taos_options_imp(TSDB_OPTION option, const char *pStr) { + SGlobalConfig *cfg = NULL; switch (option) { case TSDB_OPTION_CONFIGDIR: - pStr = (char *)arg; - if (cfg_configDir && cfg_configDir->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + cfg = tsGetConfigOption("configDir"); + if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strncpy(configDir, pStr, TSDB_FILENAME_LEN); - cfg_configDir->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscPrint("set config file directory:%s", pStr); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg_configDir->option, pStr, - tsCfgStatusStr[cfg_configDir->cfgStatus], (char *)cfg_configDir->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - if (cfg_activetimer && cfg_activetimer->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi((char *)arg); + cfg = tsGetConfigOption("shellActivityTimer"); + if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(pStr); if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg_activetimer->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscPrint("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg_activetimer->option, pStr, - tsCfgStatusStr[cfg_activetimer->cfgStatus], (int32_t *)cfg_activetimer->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, + tsCfgStatusStr[cfg->cfgStatus], (int32_t *)cfg->ptr); } break; case TSDB_OPTION_LOCALE: { // set locale - pStr = (char *)arg; - + cfg = tsGetConfigOption("locale"); size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("Invalid locale:%s, use default", pStr); return -1; } - if (cfg_locale && cfg_charset && cfg_locale->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg && cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { char sep = '.'; if (strlen(tsLocale) == 0) { // locale does not set yet @@ -239,7 +233,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { if (locale != NULL) { tscPrint("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg_locale->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; } else { // set the user-specified localed failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale); @@ -261,7 +255,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { } strncpy(tsCharset, charset, tListLen(tsCharset)); - cfg_charset->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; } else { tscPrint("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); @@ -272,23 +266,22 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { tscPrint("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg_locale->option, pStr, - tsCfgStatusStr[cfg_locale->cfgStatus], (char *)cfg_locale->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } case TSDB_OPTION_CHARSET: { /* set charset will override the value of charset, assigned during system locale changed */ - pStr = (char *)arg; - + cfg = tsGetConfigOption("charset"); size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("failed to set charset:%s", pStr); return -1; } - if (cfg_charset && cfg_charset->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { if (taosValidateEncodec(pStr)) { if (strlen(tsCharset) == 0) { tscPrint("charset is set:%s", pStr); @@ -297,40 +290,41 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { } strncpy(tsCharset, pStr, tListLen(tsCharset)); - cfg_charset->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; } else { tscPrint("charset:%s not valid", pStr); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg_charset->option, pStr, - tsCfgStatusStr[cfg_charset->cfgStatus], (char *)cfg_charset->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } case TSDB_OPTION_TIMEZONE: - pStr = (char *)arg; - if (cfg_timezone && cfg_timezone->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + cfg = tsGetConfigOption("timezone"); + if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strcpy(tsTimezone, pStr); tsSetTimeZone(); - cfg_timezone->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscTrace("timezone set:%s, input:%s by taos_options", tsTimezone, pStr); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg_timezone->option, pStr, - tsCfgStatusStr[cfg_timezone->cfgStatus], (char *)cfg_timezone->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; case TSDB_OPTION_SOCKET_TYPE: - if (cfg_socket && cfg_socket->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { - if (strcasecmp(arg, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(arg, TAOS_SOCKET_TYPE_NAME_TCP) != 0) { + cfg = tsGetConfigOption("sockettype"); + if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_TCP) != 0) { tscError("only 'tcp' or 'udp' allowed for configuring the socket type"); return -1; } - strncpy(tsSocketType, arg, tListLen(tsSocketType)); - cfg_socket->cfgStatus = TSDB_CFG_CSTATUS_OPTION; + strncpy(tsSocketType, pStr, tListLen(tsSocketType)); + cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscPrint("socket type is set:%s", tsSocketType); } break; @@ -342,3 +336,20 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { return 0; } + + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscPrint("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} \ No newline at end of file diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 026d48ba08..09ced23c91 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -247,7 +247,7 @@ typedef struct { extern SGlobalConfig *tsGlobalConfig; extern int tsGlobalConfigNum; extern char * tsCfgStatusStr[]; -SGlobalConfig *tsGetConfigOption(char *option); +SGlobalConfig *tsGetConfigOption(const char *option); #define TSDB_CFG_MAX_NUM 110 #define TSDB_CFG_PRINT_LEN 23 diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 586f329001..bac89db7e2 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -364,7 +364,7 @@ void tsReadLogOption(char *option, char *value) { } } -SGlobalConfig *tsGetConfigOption(char *option) { +SGlobalConfig *tsGetConfigOption(const char *option) { tsInitGlobalConfig(); for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalConfig *cfg = tsGlobalConfig + i; @@ -374,7 +374,7 @@ SGlobalConfig *tsGetConfigOption(char *option) { return NULL; } -void tsReadConfigOption(char *option, char *value) { +void tsReadConfigOption(const char *option, char *value) { for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalConfig *cfg = tsGlobalConfig + i; if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; @@ -423,9 +423,7 @@ void tsInitConfigOption(SGlobalConfig *cfg, char *name, void *ptr, int8_t valTyp cfg->cfgStatus = TSDB_CFG_CSTATUS_NONE; } -void tsInitGlobalConfig() { - if (tsGlobalConfig != NULL) return; - +static void doInitGlobalConfig() { tsGlobalConfig = (SGlobalConfig *) malloc(sizeof(SGlobalConfig) * TSDB_CFG_MAX_NUM); memset(tsGlobalConfig, 0, sizeof(SGlobalConfig) * TSDB_CFG_MAX_NUM); @@ -783,6 +781,11 @@ void tsInitGlobalConfig() { tsGlobalConfigNum = (int)(cfg - tsGlobalConfig); } +static pthread_once_t initGlobalConfig = PTHREAD_ONCE_INIT; +void tsInitGlobalConfig() { + pthread_once(&initGlobalConfig, doInitGlobalConfig); +} + void tsReadGlobalLogConfig() { tsInitGlobalConfig(); From cf1ae56ebb11ad51f152bc79adf3d71f2a8c5911 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 13 Nov 2019 11:28:38 +0800 Subject: [PATCH 32/69] [TBASE-1112], and refactor some code. --- src/client/src/tscServer.c | 9 +++++---- src/inc/taosmsg.h | 2 +- src/kit/shell/src/shellEngine.c | 2 +- src/system/detail/src/mgmtShell.c | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 67118b50e2..6148aed91b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2460,10 +2460,10 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); - *((uint64_t *)pMsg) = pSql->res.qhandle; + *((uint64_t *) pMsg) = pSql->res.qhandle; pMsg += sizeof(pSql->res.qhandle); - *pMsg = htons(pCmd->type); + *((uint16_t*) pMsg) = htons(pCmd->type); pMsg += sizeof(pCmd->type); msgLen = pMsg - pStart; @@ -3451,11 +3451,12 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { pRes->row = 0; /** - * If the query result is exhausted, the connection will be recycled. - * If current query is to free resource at server side, the connection will be recycle. + * If the query result is exhausted, or current query is to free resource at server side, + * the connection will be recycled. */ if ((pRes->numOfRows == 0 && !(tscProjectionQueryOnMetric(pCmd) && pRes->offset > 0)) || ((pCmd->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE)) { + tscTrace("%p no result or free resource, recycle connection", pSql); taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); pSql->thandle = NULL; } else { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1d5e1bb5a5..4f037a2f07 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -673,7 +673,7 @@ typedef struct { typedef struct { uint64_t qhandle; - int16_t free; + uint16_t free; } SRetrieveMeterMsg; typedef struct { diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 12c844d00d..0e6519bec0 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -846,7 +846,7 @@ void shellGetGrantInfo(void *con) { TAOS_FIELD *fields = taos_fetch_fields(result); TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { - fprintf(stderr, "\nGrant information is empty.\n"); + fprintf(stderr, "\nFailed to grant information from server. Abort.\n"); exit(0); } diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 43b002992d..2661ec0d37 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -873,7 +873,7 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, TSDB_CODE_MEMORY_CORRUPTED); return -1; } else { - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) == 0) { + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsToRead = pShow->numOfRows - pShow->numOfReads; } @@ -905,7 +905,7 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { pMsg = pRsp->data; // if free flag is set, client wants to clean the resources - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) == 0) + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) rowsRead = (*mgmtRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, pConn); if (rowsRead < 0) { From ddfa301e140db8cd77e97ce50a9a51e7db06adea Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 13 Nov 2019 18:08:50 +0800 Subject: [PATCH 33/69] refactor some codes --- src/client/inc/tscUtil.h | 8 ++++---- src/client/src/tscFunctionImpl.c | 6 ------ src/client/src/tscSQLParser.c | 5 +++-- src/client/src/tscServer.c | 11 +++++++++-- src/os/linux/inc/os.h | 2 ++ 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0474697c3f..a0c16f062d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -30,10 +30,10 @@ extern "C" { #include "tsdb.h" #include "tscSecondaryMerge.h" -#define UTIL_METER_IS_METRIC(cmd) (((cmd)->pMeterMeta != NULL) && ((cmd)->pMeterMeta->meterType == TSDB_METER_METRIC)) -#define UTIL_METER_IS_NOMRAL_METER(cmd) (!(UTIL_METER_IS_METRIC(cmd))) -#define UTIL_METER_IS_CREATE_FROM_METRIC(cmd) \ - (((cmd)->pMeterMeta != NULL) && ((cmd)->pMeterMeta->meterType == TSDB_METER_MTABLE)) +#define UTIL_METER_IS_METRIC(metaInfo) (((metaInfo)->pMeterMeta != NULL) && ((metaInfo)->pMeterMeta->meterType == TSDB_METER_METRIC)) +#define UTIL_METER_IS_NOMRAL_METER(metaInfo) (!(UTIL_METER_IS_METRIC(metaInfo))) +#define UTIL_METER_IS_CREATE_FROM_METRIC(metaInfo) \ + (((metaInfo)->pMeterMeta != NULL) && ((metaInfo)->pMeterMeta->meterType == TSDB_METER_MTABLE)) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 8e1fdf998a..eebb765889 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3779,9 +3779,6 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, assert(numOfRow <= INT16_MAX); - int64_t lastKey = 0; - int64_t lastVal = TSDB_DATA_BIGINT_NULL; - for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(&data[i], TSDB_DATA_TYPE_BIGINT)) { (*numOfNull) += 1; @@ -3873,9 +3870,6 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do assert(numOfRow <= INT16_MAX); - int64_t lastKey = 0; - double lastVal = TSDB_DATA_DOUBLE_NULL; - for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(&data[i], TSDB_DATA_TYPE_DOUBLE)) { (*numOfNull) += 1; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 1bc0163e5f..9ee8ea7630 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5668,8 +5668,7 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { } if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && - functId != TSDB_FUNC_TAGPRJ && - (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX)) { + functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_PRJ) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } @@ -5697,6 +5696,8 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } + + return TSDB_CODE_SUCCESS; } else { return checkUpdateTagPrjFunctions(pCmd); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6148aed91b..e5fc8fbb4e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -723,9 +723,16 @@ int tscProcessSql(SSqlObj *pSql) { #else pSql->maxRetry = 2; #endif + + // the pMeterMetaInfo cannot be NULL + if (pMeterMetaInfo == NULL) { + pSql->res.code = TSDB_CODE_OTHERS; + return pSql->res.code; + } + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { pSql->index = pMeterMetaInfo->pMeterMeta->index; - } else { // it must be the parent SSqlObj for metric query + } else { // it must be the parent SSqlObj for super table query if ((pSql->cmd.type & TSDB_QUERY_TYPE_SUBQUERY) != 0) { int32_t idx = pSql->cmd.vnodeIdx; SVnodeSidList *pSidList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); @@ -3770,7 +3777,7 @@ void tscInitMsgs() { tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromMgmt; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromVnode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_TAGS] = tscProcessTagRetrieveRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index ee65965a5b..5b44836ebe 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -25,6 +25,7 @@ extern "C" { #include #include +#include #include #include #include @@ -61,6 +62,7 @@ extern "C" { #include #include #include +#include #define taosCloseSocket(x) \ { \ From 32c5c18c3fc68e5c8056234615c5bad47658709f Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 13 Nov 2019 19:08:00 +0800 Subject: [PATCH 34/69] [TBASE-1116] --- src/modules/http/src/httpSystem.c | 2 +- src/modules/monitor/src/monitorSystem.c | 3 +++ src/system/detail/src/dnodeSystem.c | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c index 67be206842..df49251f13 100644 --- a/src/modules/http/src/httpSystem.c +++ b/src/modules/http/src/httpSystem.c @@ -77,7 +77,7 @@ int httpStartSystem() { if (httpServer == NULL) { httpError("http server is null"); - return -1; + httpInitSystem(); } if (httpServer->pContextPool == NULL) { diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index 78cee40f96..4d6577c8f3 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -95,6 +95,9 @@ int monitorInitSystem() { } int monitorStartSystem() { + if (monitor == NULL) { + monitorInitSystem(); + } taosTmrReset(monitorInitConn, 10, NULL, tscTmr, &monitor->initTimer); return 0; } diff --git a/src/system/detail/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c index 30b3e202ad..1b5d9418b4 100644 --- a/src/system/detail/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -33,8 +33,8 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Woverflow" -SModule tsModule[TSDB_MOD_MAX]; -uint32_t tsModuleStatus; +SModule tsModule[TSDB_MOD_MAX] = {0}; +uint32_t tsModuleStatus = 0; pthread_mutex_t dmutex; extern int vnodeSelectReqNum; extern int vnodeInsertReqNum; From 88abf75491f78992ab521aef077ebd84cc1c5f61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Nov 2019 11:51:48 +0000 Subject: [PATCH 35/69] Bump jackson-databind in /tests/comparisonTest/opentsdb/opentsdbtest Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.9.10 to 2.9.10.1. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Signed-off-by: dependabot[bot] --- tests/comparisonTest/opentsdb/opentsdbtest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index 9271d478a6..6f5fa6e132 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -118,7 +118,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10 + 2.9.10.1 From 891c12cdee2edc58ec6a06d4760551f3d3132100 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 14 Nov 2019 16:26:46 +0800 Subject: [PATCH 36/69] [TBASE-686]add test case --- src/client/src/tscFunctionImpl.c | 18 ++-- src/client/src/tscSQLParser.c | 120 +++++++++++-------------- src/os/linux/inc/os.h | 1 + src/system/detail/src/vnodeQueryImpl.c | 2 +- 4 files changed, 66 insertions(+), 75 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index eebb765889..227095fc03 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1482,7 +1482,9 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + + TSKEY ts = pCtx->ptsList[index]; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; @@ -1575,7 +1577,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } } @@ -1623,7 +1625,9 @@ static void last_function(SQLFunctionCtx *pCtx) { } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + + TSKEY ts = pCtx->ptsList[i]; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; @@ -1648,7 +1652,9 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + + TSKEY ts = pCtx->ptsList[index]; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -1745,7 +1751,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } } @@ -1800,7 +1806,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { pInfo1->ts = pCtx->param[0].i64Key; pInfo1->hasResult = DATA_SET_FLAG; - DO_UPDATE_TAG_COLUMNS(pCtx, 0); + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } SET_VAL(pCtx, pCtx->size, 1); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9ee8ea7630..c66d524fee 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2806,59 +2806,6 @@ static bool functionCompatibleCheck(SSqlCmd* pCmd) { } } - // additional check for select aggfuntion(column), column1 from table_name group by(column1); - if ((pCmd->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) == TSDB_QUERY_TYPE_PROJECTION_QUERY) { - bool isAggFunc = false; - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - - if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS || - functionId == TSDB_FUNC_ARITHM) { - continue; - } - - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { - isAggFunc = true; - break; - } - } - - // TODO change the type, the type is not correct - if (isAggFunc) { - pCmd->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); - - // agg function mixed up with project query without group by exists - if (pCmd->groupbyExpr.numOfGroupCols == 0) { - return false; - } - - // get the project column - int32_t numOfPrjColumn = 0; - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->functionId == TSDB_FUNC_PRJ) { - numOfPrjColumn += 1; - - bool qualifiedCol = false; - for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { - if (pExpr->colInfo.colId == pCmd->groupbyExpr.columnInfo[j].colId) { - qualifiedCol = true; - - pExpr->param[0].i64Key = 1; // limit the output to be 1 for each state value - pExpr->numOfParams = 1; - break; - } - } - - if (!qualifiedCol) { - setErrMsg(pCmd, msg1); - return false; - } - } - } - } - } - return true; } @@ -5416,6 +5363,27 @@ static void doUpdateSqlFunctionForTagPrj(SSqlCmd* pCmd) { } } +static void doUpdateSqlFunctionForColPrj(SSqlCmd* pCmd) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_PRJ) { + + bool qualifiedCol = false; + for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { + if (pExpr->colInfo.colId == pCmd->groupbyExpr.columnInfo[j].colId) { + qualifiedCol = true; + + pExpr->param[0].i64Key = 1; // limit the output to be 1 for each state value + pExpr->numOfParams = 1; + break; + } + } + + assert(qualifiedCol); + } + } +} + static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { if (columnId == pGroupbyExpr->columnInfo[j].colId && pGroupbyExpr->columnInfo[j].flag == TSDB_COL_TAG) { @@ -5480,7 +5448,8 @@ static void updateTagPrjFunction(SSqlCmd* pCmd) { static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { const char* msg1 = "only one selectivity function allowed in presence of tags function"; const char* msg2 = "functions not allowed"; - + const char* msg3 = "aggregation function should not be mixed up with projection"; + bool tagColExists = false; int16_t numOfTimestamp = 0; // primary timestamp column int16_t numOfSelectivity = 0; @@ -5494,21 +5463,21 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { break; } } - - if (tagColExists) { // check if the selectivity function exists - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { - continue; - } - - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - numOfSelectivity++; - } else { - numOfAggregation++; - } + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { + continue; } - + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } else { + numOfAggregation++; + } + } + + if (tagColExists) { // check if the selectivity function exists // When the tag projection function on tag column that is not in the group by clause, aggregation function and // selectivity function exist in select clause is not allowed. if (numOfAggregation > 0) { @@ -5521,6 +5490,7 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { */ if (numOfSelectivity == 1) { doUpdateSqlFunctionForTagPrj(pCmd); + doUpdateSqlFunctionForColPrj(pCmd); } else if (numOfSelectivity > 1) { /* * If more than one selectivity functions exist, all the selectivity functions must be last_row. @@ -5539,6 +5509,20 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { } doUpdateSqlFunctionForTagPrj(pCmd); + doUpdateSqlFunctionForColPrj(pCmd); + } + } else { + if ((pCmd->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) == TSDB_QUERY_TYPE_PROJECTION_QUERY) { + if (numOfAggregation > 0 && pCmd->groupbyExpr.numOfGroupCols == 0) { + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + + if (numOfAggregation > 0 || numOfSelectivity > 0) { + // clear the projection type flag + pCmd->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); + doUpdateSqlFunctionForColPrj(pCmd); + } } } diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 8055777622..6ecb64eda5 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -28,6 +28,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 9967094a26..f7f456ec53 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -1840,7 +1840,7 @@ static void setCtxTagColumnInfo(SQuery* pQuery, SQueryRuntimeEnv* pRuntimeEnv) { // ts may be the required primary timestamp column continue; } else { - assert(0); + // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ } } From 69cdd4cd30c2e40f704de434375dc290d43b11d3 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 14 Nov 2019 16:34:13 +0800 Subject: [PATCH 37/69] [tbase-686] --- src/kit/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 0e6519bec0..34f51e3b3f 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -846,7 +846,7 @@ void shellGetGrantInfo(void *con) { TAOS_FIELD *fields = taos_fetch_fields(result); TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { - fprintf(stderr, "\nFailed to grant information from server. Abort.\n"); + fprintf(stderr, "\nFailed to get grant information from server. Abort.\n"); exit(0); } From 0bf6ef471a894f5e6ec63916e49c2367a49ecd1e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 Nov 2019 09:45:04 +0800 Subject: [PATCH 38/69] Fix #719 --- src/system/detail/src/vnodeImport.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index ed9c319216..cd81a9b0b8 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -363,19 +363,19 @@ int vnodeImportToFile(SImportInfo *pImport) { SData *cdata[TSDB_MAX_COLUMNS]; char *buffer1 = - malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); + malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES + sizeof(TSCKSUM)) * pObj->numOfColumns); cdata[0] = (SData *)buffer1; SData *data[TSDB_MAX_COLUMNS]; char *buffer2 = - malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); + malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES + sizeof(TSCKSUM)) * pObj->numOfColumns); data[0] = (SData *)buffer2; for (col = 1; col < pObj->numOfColumns; ++col) { cdata[col] = (SData *)(((char *)cdata[col - 1]) + sizeof(SData) + EXTRA_BYTES + - pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes); + pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes + sizeof(TSCKSUM)); data[col] = (SData *)(((char *)data[col - 1]) + sizeof(SData) + EXTRA_BYTES + - pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes); + pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes + sizeof(TSCKSUM)); } int rowsBefore = 0; From 331315527119d500d625238d843081caa51610d9 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 15 Nov 2019 11:00:37 +0800 Subject: [PATCH 39/69] [TBASE-814] --- src/client/src/tscServer.c | 56 ++++++++++++++++-------------- src/inc/taosmsg.h | 6 ++-- src/rpc/src/tstring.c | 10 +++--- src/system/detail/src/dnodeMgmt.c | 20 +++++------ src/system/detail/src/vnodeMeter.c | 2 +- src/system/detail/src/vnodeShell.c | 20 +++++------ src/system/detail/src/vnodeUtil.c | 2 +- 7 files changed, 62 insertions(+), 54 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3dbf6596f1..e980e55446 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -134,6 +134,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { tscProcessSql(pObj->pHb); } +//TODO HANDLE error from mgmt void tscGetConnToMgmt(SSqlObj *pSql, uint8_t *pCode) { STscObj *pTscObj = pSql->pTscObj; #ifdef CLUSTER @@ -163,10 +164,11 @@ void tscGetConnToMgmt(SSqlObj *pSql, uint8_t *pCode) { connInit.spi = 1; connInit.encrypt = 0; connInit.secret = pSql->pTscObj->pass; + #ifdef CLUSTER connInit.peerIp = tscMgmtIpList.ipstr[pSql->index]; #else - connInit.peerIp = tsServerIpStr; + connInit.peerIp = tsServerIpStr; #endif thandle = taosOpenRpcConn(&connInit, pCode); } @@ -278,6 +280,11 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { break; } + + // the pSql->res.code is the previous error code. + if (pSql->thandle == NULL && pSql->retry >= pSql->maxRetry) { + *pCode = pSql->res.code; + } } int tscSendMsgToServer(SSqlObj *pSql) { @@ -389,10 +396,8 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { // todo taos_stop_query() in async model /* * in case of - * 1. query cancelled(pRes->code != TSDB_CODE_QUERY_CANCELLED), do NOT re-issue the - * request to server. - * 2. retrieve, do NOT re-issue the retrieve request since the qhandle may - * have been released by server + * 1. query cancelled(pRes->code != TSDB_CODE_QUERY_CANCELLED), do NOT re-issue the request to server. + * 2. retrieve, do NOT re-issue the retrieve request since the qhandle may have been released by server */ if (pCmd->command != TSDB_SQL_FETCH && pCmd->command != TSDB_SQL_RETRIEVE && pCmd->command != TSDB_SQL_KILL_QUERY && pRes->code != TSDB_CODE_QUERY_CANCELLED) { @@ -419,7 +424,9 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { } } else { #ifdef CLUSTER - if (pMsg->content[0] == TSDB_CODE_REDIRECT) { + uint16_t rspCode = pMsg->content[0]; + + if (rspCode == TSDB_CODE_REDIRECT) { tscTrace("%p it shall be redirected!", pSql); taosAddConnIntoCache(tscConnCache, thandle, pSql->ip, pSql->vnode, pObj->user); pSql->thandle = NULL; @@ -433,28 +440,23 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { code = tscSendMsgToServer(pSql); if (code == 0) return pSql; msg = NULL; - } else if (pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION || pMsg->content[0] == TSDB_CODE_NETWORK_UNAVAIL || - pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { + } else if (rspCode == TSDB_CODE_NOT_ACTIVE_TABLE || rspCode == TSDB_CODE_INVALID_TABLE_ID || + rspCode == TSDB_CODE_INVALID_VNODE_ID || rspCode == TSDB_CODE_NOT_ACTIVE_VNODE || + rspCode == TSDB_CODE_NETWORK_UNAVAIL) { #else - if (pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION || pMsg->content[0] == TSDB_CODE_NETWORK_UNAVAIL || - pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { + if (rspCode == TSDB_CODE_NOT_ACTIVE_TABLE || rspCode == TSDB_CODE_INVALID_TABLE_ID || + rspCode == TSDB_CODE_INVALID_VNODE_ID || rspCode == TSDB_CODE_NOT_ACTIVE_VNODE || + rspCode == TSDB_CODE_NETWORK_UNAVAIL) { #endif pSql->thandle = NULL; taosAddConnIntoCache(tscConnCache, thandle, pSql->ip, pSql->vnode, pObj->user); - - if (pMeterMetaInfo != NULL && UTIL_METER_IS_METRIC(pMeterMetaInfo) && - pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION) { + + if ((pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) && + (rspCode == TSDB_CODE_INVALID_TABLE_ID || rspCode == TSDB_CODE_INVALID_VNODE_ID)) { /* - * for metric query, in case of any meter missing during query, sub-query of metric query will failed, - * causing metric query failed, and return TSDB_CODE_METRICMETA_EXPIRED code to app - */ - tscTrace("%p invalid meters id cause metric query failed, code:%d", pSql, pMsg->content[0]); - code = TSDB_CODE_METRICMETA_EXPIRED; - } else if ((pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) && - pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { - /* - * session id is invalid(e.g., less than 0 or larger than maximum session per - * vnode) in submit/query msg, no retry + * In case of the insert/select operations, the invalid table(vnode) id means + * the submit/query msg is invalid, renew meter meta will not help to fix this problem, + * so return the invalid_query_msg to client directly. */ code = TSDB_CODE_INVALID_QUERY_MSG; } else if (pCmd->command == TSDB_SQL_CONNECT) { @@ -462,9 +464,11 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { } else if (pCmd->command == TSDB_SQL_HB) { code = TSDB_CODE_NOT_READY; } else { - tscTrace("%p it shall renew meter meta, code:%d", pSql, pMsg->content[0]); + tscTrace("%p it shall renew meter meta, code:%d", pSql, rspCode); + pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; - + pSql->res.code = (uint8_t) rspCode; // keep the previous error code + code = tscRenewMeterMeta(pSql, pMeterMetaInfo->name); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return pSql; @@ -476,7 +480,7 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { msg = NULL; } else { // for other error set and return to invoker - code = pMsg->content[0]; + code = rspCode; } } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 4f037a2f07..71e4c75634 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -41,7 +41,7 @@ extern "C" { #define TSDB_CODE_ACTION_NOT_ONLINE 18 #define TSDB_CODE_ACTION_SEND_FAILD 19 #define TSDB_CODE_NOT_ACTIVE_SESSION 20 -#define TSDB_CODE_INSERT_FAILED 21 +#define TSDB_CODE_INVALID_VNODE_ID 21 #define TSDB_CODE_APP_ERROR 22 #define TSDB_CODE_INVALID_IE 23 #define TSDB_CODE_INVALID_VALUE 24 @@ -74,7 +74,7 @@ extern "C" { #define TSDB_CODE_OTHERS 51 #define TSDB_CODE_NO_REMOVE_MASTER 52 #define TSDB_CODE_WRONG_SCHEMA 53 -#define TSDB_CODE_NO_RESULT 54 +#define TSDB_CODE_NOT_ACTIVE_VNODE 54 #define TSDB_CODE_TOO_MANY_USERS 55 #define TSDB_CODE_TOO_MANY_DATABSES 56 #define TSDB_CODE_TOO_MANY_TABLES 57 @@ -134,6 +134,8 @@ extern "C" { #define TSDB_CODE_NOT_SUPER_TABLE 111 // #define TSDB_CODE_DUPLICATE_TAGS 112 // tags value for join not unique #define TSDB_CODE_INVALID_SUBMIT_MSG 113 +#define TSDB_CODE_NOT_ACTIVE_TABLE 114 +#define TSDB_CODE_INVALID_TABLE_ID 115 // message type #define TSDB_MSG_TYPE_REG 1 diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index c7afe54d9a..9978ba9f5d 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -145,7 +145,7 @@ char *tsError[] = {"success", "not online", "send failed", "not active session", // 20 - "insert failed", + "invalid vnode id", "App error", "invalid IE", "invalid value", @@ -178,7 +178,7 @@ char *tsError[] = {"success", "others", "can't remove dnode which is master", "wrong schema", - "no results", + "vnode not active(not created yet or dropped already)", "num of users execeed maxUsers", //55 "num of databases execeed maxDbs", "num of tables execeed maxTables", @@ -233,9 +233,11 @@ char *tsError[] = {"success", "invalid query message", "timestamp disordered in cache block", "timestamp disordered in file block", - "invalid commit log", //110 - "server no disk space", + "invalid commit log", + "server no disk space", //110 "only super table has metric meta info", "tags value not unique for join", "invalid submit message", + "not active table(not created yet or deleted already)", //114 + "invalid table id", }; diff --git a/src/system/detail/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c index 3a6c9e50a8..d3f84639a6 100644 --- a/src/system/detail/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -105,14 +105,14 @@ int vnodeProcessCreateMeterRequest(char *pMsg, int msgLen, SMgmtObj *pObj) { if (vid >= TSDB_MAX_VNODES || vid < 0) { dError("vid:%d, vnode is out of range", vid); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_VNODE_ID; goto _over; } pVnode = vnodeList + vid; if (pVnode->cfg.maxSessions <= 0) { dError("vid:%d, not activated", vid); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_VNODE; goto _over; } @@ -141,27 +141,27 @@ int vnodeProcessAlterStreamRequest(char *pMsg, int msgLen, SMgmtObj *pObj) { if (vid >= TSDB_MAX_VNODES || vid < 0) { dError("vid:%d, vnode is out of range", vid); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_VNODE_ID; goto _over; } pVnode = vnodeList + vid; if (pVnode->cfg.maxSessions <= 0 || pVnode->pCachePool == NULL) { dError("vid:%d is not activated yet", pAlter->vnode); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_NOT_ACTIVE_VNODE; goto _over; } if (pAlter->sid >= pVnode->cfg.maxSessions || pAlter->sid < 0) { dError("vid:%d sid:%d uid:%ld, sid is out of range", pAlter->vnode, pAlter->sid, pAlter->uid); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_TABLE_ID; goto _over; } SMeterObj *pMeterObj = vnodeList[vid].meterList[sid]; if (pMeterObj == NULL || sid != pMeterObj->sid || vid != pMeterObj->vnode) { - dError("vid:%d sid:%d, no active session", vid, sid); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + dError("vid:%d sid:%d, no active table", vid, sid); + code = TSDB_CODE_NOT_ACTIVE_TABLE; goto _over; } @@ -195,7 +195,7 @@ int vnodeProcessCreateMeterMsg(char *pMsg, int msgLen) { if (pCreate->vnode >= TSDB_MAX_VNODES || pCreate->vnode < 0) { dError("vid:%d is out of range", pCreate->vnode); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_VNODE_ID; goto _create_over; } @@ -203,13 +203,13 @@ int vnodeProcessCreateMeterMsg(char *pMsg, int msgLen) { if (pVnode->pCachePool == NULL) { dError("vid:%d is not activated yet", pCreate->vnode); vnodeSendVpeerCfgMsg(pCreate->vnode); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_VNODE; goto _create_over; } if (pCreate->sid >= pVnode->cfg.maxSessions || pCreate->sid < 0) { dError("vid:%d sid:%d id:%s, sid is out of range", pCreate->vnode, pCreate->sid, pCreate->meterId); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_TABLE_ID; goto _create_over; } diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index c001daf7da..e3cde9a84d 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -618,7 +618,7 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi dWarn("vid:%d sid:%d id:%s, meter is dropped, abort insert, state:%d", pObj->vnode, pObj->sid, pObj->meterId, pObj->state); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_TABLE; break; } diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 1b4b43f5a2..5973b01d71 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -269,7 +269,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pQueryMsg->vnode >= TSDB_MAX_VNODES || pQueryMsg->vnode < 0) { dTrace("qmsg:%p,vid:%d is out of range", pQueryMsg, pQueryMsg->vnode); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_TABLE_ID; goto _query_over; } @@ -278,7 +278,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pVnode->cfg.maxSessions == 0) { dError("qmsg:%p,vid:%d is not activated yet", pQueryMsg, pQueryMsg->vnode); vnodeSendVpeerCfgMsg(pQueryMsg->vnode); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_TABLE; goto _query_over; } @@ -295,7 +295,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pVnode->meterList == NULL) { dError("qmsg:%p,vid:%d has been closed", pQueryMsg, pQueryMsg->vnode); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_VNODE; goto _query_over; } @@ -305,7 +305,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { dTrace("qmsg:%p sid:%d is out of range, valid range:[%d,%d]", pQueryMsg, pSids[i]->sid, 0, pVnode->cfg.maxSessions); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_TABLE_ID; goto _query_over; } } @@ -488,7 +488,7 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pSubmit->vnode >= TSDB_MAX_VNODES || pSubmit->vnode < 0) { dTrace("vnode:%d is out of range", pSubmit->vnode); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_VNODE_ID; goto _submit_over; } @@ -496,7 +496,7 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pVnode->cfg.maxSessions == 0 || pVnode->meterList == NULL) { dError("vid:%d is not activated for submit", pSubmit->vnode); vnodeSendVpeerCfgMsg(pSubmit->vnode); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_VNODE; goto _submit_over; } @@ -529,7 +529,7 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (pBlocks->sid >= pVnode->cfg.maxSessions || pBlocks->sid <= 0) { dTrace("sid:%d is out of range", pBlocks->sid); - code = TSDB_CODE_INVALID_SESSION_ID; + code = TSDB_CODE_INVALID_TABLE_ID; goto _submit_over; } @@ -538,9 +538,9 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { SMeterObj *pMeterObj = vnodeList[vnode].meterList[sid]; if (pMeterObj == NULL) { - dError("vid:%d sid:%d, no active session", vnode, sid); + dError("vid:%d sid:%d, no active table", vnode, sid); vnodeSendMeterCfgMsg(vnode, sid); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_TABLE; goto _submit_over; } @@ -579,7 +579,7 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DELETING)) { dTrace("vid:%d sid:%d id:%s, it is removed, state:%d", pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pMeterObj->state); - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_TABLE; break; } else {// waiting for 300ms by default and try again dTrace("vid:%d sid:%d id:%s, try submit again since in state:%d", pMeterObj->vnode, pMeterObj->sid, diff --git a/src/system/detail/src/vnodeUtil.c b/src/system/detail/src/vnodeUtil.c index b097b4706c..6691b2245d 100644 --- a/src/system/detail/src/vnodeUtil.c +++ b/src/system/detail/src/vnodeUtil.c @@ -553,7 +553,7 @@ int32_t vnodeIncQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterSidExtInfo** pSid if (pMeter == NULL || (pMeter->state > TSDB_METER_STATE_INSERT)) { if (pMeter == NULL || vnodeIsMeterState(pMeter, TSDB_METER_STATE_DELETING)) { - code = TSDB_CODE_NOT_ACTIVE_SESSION; + code = TSDB_CODE_NOT_ACTIVE_TABLE; dError("qmsg:%p, vid:%d sid:%d, not there or will be dropped", pQueryMsg, pQueryMsg->vnode, pSids[i]->sid); vnodeSendMeterCfgMsg(pQueryMsg->vnode, pSids[i]->sid); } else {//update or import From 7a4bbc5c23bcb244953e39001722e711b759a7fe Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 15 Nov 2019 11:11:23 +0800 Subject: [PATCH 40/69] [tbase-1147] --- src/client/src/tscServer.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e980e55446..cea17400cd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -320,11 +320,19 @@ int tscSendMsgToServer(SSqlObj *pSql) { char *pStart = taosBuildReqHeader(pSql->thandle, pSql->cmd.msgType, buf); if (pStart) { + /* + * this SQL object may be released by other thread due to the completion of this query even before the log + * is dumped to log file. So the signature needs to be kept in a local variable. + */ + uint64_t signature = (uint64_t) pSql->signature; if (tscUpdateVnodeMsg[pSql->cmd.command]) (*tscUpdateVnodeMsg[pSql->cmd.command])(pSql, buf); + int ret = taosSendMsgToPeerH(pSql->thandle, pStart, pSql->cmd.payloadLen, pSql); - - if (ret >= 0) code = 0; - tscTrace("%p send msg ret:%d code:%d sig:%p", pSql, ret, code, pSql->signature); + if (ret >= 0) { + code = 0; + } + + tscTrace("%p send msg ret:%d code:%d sig:%p", pSql, ret, code, signature); } } From 683baa289999ddff3e5e3f004de849a2aa08f735 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 15 Nov 2019 11:59:36 +0800 Subject: [PATCH 41/69] [TBASE1020] --- src/client/src/tscSql.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 10eac0bb63..6ec8f425ca 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -647,11 +647,8 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { pCmd->limit.limit = pSql->cmd.globalLimit - pRes->numOfTotal; pCmd->limit.offset = pRes->offset; -#ifdef CLUSTER - if ((++pSql->cmd.vnodeIdx) <= pMeterMetaInfo->pMetricMeta->numOfVnodes) { -#else + if ((++pSql->cmd.vnodeIdx) < pMeterMetaInfo->pMetricMeta->numOfVnodes) { -#endif pSql->cmd.command = TSDB_SQL_SELECT; assert(pSql->fp == NULL); tscProcessSql(pSql); From 5a09ffebb9bd446cf1f15eda6d8fc0b12afe0556 Mon Sep 17 00:00:00 2001 From: malong Date: Fri, 15 Nov 2019 15:06:01 +0800 Subject: [PATCH 42/69] =?UTF-8?q?=E6=8C=89=E6=97=B6=E9=97=B4=E5=91=A8?= =?UTF-8?q?=E6=9C=9F=E8=81=9A=E5=90=88=E6=9F=A5=E8=AF=A2=20=E4=B8=AD=20WHE?= =?UTF-8?q?RE=E5=90=8E=E9=9D=A2=E7=9A=84name=20=E6=94=B9=E4=B8=BA=20locati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- documentation/tdenginedocs-cn/super-table/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tdenginedocs-cn/super-table/index.html b/documentation/tdenginedocs-cn/super-table/index.html index 5ee9587b2f..3d2681990a 100644 --- a/documentation/tdenginedocs-cn/super-table/index.html +++ b/documentation/tdenginedocs-cn/super-table/index.html @@ -105,6 +105,6 @@ GROUP BY location, type

查询仅位于北京以外地区的温度传感器最近24小时(24h)采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)和最低温度min(degree)，将采集结果按照10分钟为周期进行聚合，并将结果按所处地域(location)和传感器类型(type)再次进行聚合。

SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)
 FROM thermometer
-WHERE name<>'beijing' and ts>=now-1d
+WHERE location<>'beijing' and ts>=now-1d
 INTERVAL(10M)
 GROUP BY location, type
回去 From a52ea6e614c465e2b72fdb39e887f50dc5a9f3c9 Mon Sep 17 00:00:00 2001 From: malong Date: Fri, 15 Nov 2019 15:22:39 +0800 Subject: [PATCH 43/69] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=B1=BBSQL=E8=AF=AD?= =?UTF-8?q?=E8=A8=80=E7=94=A8=E6=8F=92=E5=85=A5=E6=88=96=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20=E6=94=B9=E4=B8=BA=EF=BC=9A=20=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=B1=BBSQL=E8=AF=AD=E8=A8=80=E6=8F=92=E5=85=A5?= =?UTF-8?q?=E6=88=96=E6=9F=A5=E8=AF=A2=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- documentation/tdenginedocs-cn/getting-started/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tdenginedocs-cn/getting-started/index.html b/documentation/tdenginedocs-cn/getting-started/index.html index d7e8fe311c..65ef667d00 100644 --- a/documentation/tdenginedocs-cn/getting-started/index.html +++ b/documentation/tdenginedocs-cn/getting-started/index.html @@ -63,7 +63,7 @@ Query OK, 2 row(s) in set (0.001700s)

主要功能

TDengine的核心功能是时序数据库。除此之外，为减少研发的复杂度、系统维护的难度，TDengine还提供缓存、消息队列、订阅、流式计算等功能。更详细的功能如下：

    -
  • 使用类SQL语言用插入或查询数据
    • +
  • 使用类SQL语言插入或查询数据
  • 支持C/C++, Java(JDBC), Python, Go, RESTful, and Node.JS 开发接口
  • 可通过Python/R/Matlab or TDengine shell做Ad Hoc查询分析
  • 通过定时连续查询支持基于滑动窗口的流式计算
    • From e42fff8bb770698d02c56b9af91b070b0f6e1329 Mon Sep 17 00:00:00 2001 From: malong Date: Fri, 15 Nov 2019 15:38:55 +0800 Subject: [PATCH 44/69] =?UTF-8?q?=E6=AD=A4=E6=97=B6=E5=8F=AF=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E8=87=AA=E5=8A=A8=E5=BB=BA=E8=A1=A8=E8=AF=AD=E6=B3=95?= =?UTF-8?q?=E6=9D=A5=E5=AE=9E=E7=8E=B0=E5=86=99=E5=85=A5=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=97=B6=E9=87=8C=E7=94=A8=E8=B6=85=E7=BA=A7=E8=A1=A8=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E7=9A=84=E8=A1=A8=E7=BB=93=E6=9E=84=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=9A=84=E5=AD=90?= =?UTF-8?q?=E8=A1=A8=20=E5=A4=9A=E4=BA=86=E4=B8=AA=E2=80=98=E9=87=8C?= =?UTF-8?q?=E2=80=99=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- documentation/tdenginedocs-cn/super-table/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tdenginedocs-cn/super-table/index.html b/documentation/tdenginedocs-cn/super-table/index.html index 3d2681990a..679f4c307b 100644 --- a/documentation/tdenginedocs-cn/super-table/index.html +++ b/documentation/tdenginedocs-cn/super-table/index.html @@ -40,7 +40,7 @@ tags (location binary(20), type int)

    统计属于某个STable并满足查询条件的子表的数量

写数据时自动建子表

-

在某些特殊场景中，用户在写数据时并不确定某个设备的表是否存在，此时可使用自动建表语法来实现写入数据时里用超级表定义的表结构自动创建不存在的子表，若该表已存在则不会建立新表。注意：自动建表语句只能自动建立子表而不能建立超级表，这就要求超级表已经被事先定义好。自动建表语法跟insert/import语法非常相似，唯一区别是语句中增加了超级表和标签信息。具体语法如下：

+

在某些特殊场景中，用户在写数据时并不确定某个设备的表是否存在，此时可使用自动建表语法来实现写入数据时用超级表定义的表结构自动创建不存在的子表，若该表已存在则不会建立新表。注意：自动建表语句只能自动建立子表而不能建立超级表，这就要求超级表已经被事先定义好。自动建表语法跟insert/import语法非常相似，唯一区别是语句中增加了超级表和标签信息。具体语法如下：

INSERT INTO <tb_name> USING <stb_name> TAGS (<tag1_value>, ...) VALUES (field_value, ...) (field_value, ...) ...;

向表tb_name中插入一条或多条记录，如果tb_name这张表不存在，则会用超级表stb_name定义的表结构以及用户指定的标签值(即tag1_value…)来创建名为tb_name新表，并将用户指定的值写入表中。如果tb_name已经存在，则建表过程会被忽略，系统也不会检查tb_name的标签是否与用户指定的标签值一致，也即不会更新已存在表的标签。

INSERT INTO <tb1_name> USING <stb1_name> TAGS (<tag1_value1>, ...) VALUES (<field1_value1>, ...) (<field1_value2>, ...) ... <tb_name2> USING <stb_name2> TAGS(<tag1_value2>, ...) VALUES (<field1_value1>, ...) ...;
From 48c470e21c81c4c1c651b31d8823e15aac6573bc Mon Sep 17 00:00:00 2001 From: malong Date: Fri, 15 Nov 2019 15:47:34 +0800 Subject: [PATCH 45/69] =?UTF-8?q?=E2=80=9C=E8=B6=85=E7=BA=A7=E8=A1=A8?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E2=80=9D=E7=9A=84=E8=AF=B4=E6=98=8E=E2=80=9C?= =?UTF-8?q?2.TAGS=E5=88=97=E7=9A=84=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E6=98=AFtimestamp=E5=92=8Cnchar=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=EF=BC=9B=E6=8F=8F=E8=BF=B0=E9=94=99=E8=AF=AF=EF=BC=8C?= =?UTF-8?q?=E5=AE=9E=E9=99=85tags=E6=98=AF=E6=94=AF=E6=8C=81nchar=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- documentation/tdenginedocs-cn/super-table/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tdenginedocs-cn/super-table/index.html b/documentation/tdenginedocs-cn/super-table/index.html index 679f4c307b..828a69bb0c 100644 --- a/documentation/tdenginedocs-cn/super-table/index.html +++ b/documentation/tdenginedocs-cn/super-table/index.html @@ -24,7 +24,7 @@ tags (location binary(20), type int)

说明：

  1. TAGS列总长度不能超过512 bytes；
    2. -
  2. TAGS列的数据类型不能是timestamp和nchar类型；
    3. +
  3. TAGS列的数据类型不能是timestamp类型；
  4. TAGS列名不能与其他列名相同;
  5. TAGS列名不能为预留关键字.

  • 显示已创建的超级表

    From 29a98bbe6108c5ee4cb6c879dac8b113a15db592 Mon Sep 17 00:00:00 2001 From: fangpanpan Date: Fri, 15 Nov 2019 16:03:04 +0800 Subject: [PATCH 46/69] [tbase-814] --- src/client/src/tscServer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cea17400cd..75e4c9606f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -431,8 +431,9 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { } } } else { -#ifdef CLUSTER uint16_t rspCode = pMsg->content[0]; +#ifdef CLUSTER + if (rspCode == TSDB_CODE_REDIRECT) { tscTrace("%p it shall be redirected!", pSql); From ad3c52a806b207de058955e65ec4277acbd7e1c8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 15 Nov 2019 16:29:18 +0800 Subject: [PATCH 47/69] [tbase-1147] --- src/client/src/tscServer.c | 11 +++++++---- src/system/detail/src/mgmtDnodeInt.c | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cea17400cd..8a6a7e30ab 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -224,7 +224,7 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { (pSql->index) = (pSql->index + 1) % TSDB_VNODES_SUPPORT; continue; } - *pCode = 0; + *pCode = TSDB_CODE_SUCCESS; void *thandle = taosGetConnFromCache(tscConnCache, pVPeersDesc[pSql->index].ip, pVPeersDesc[pSql->index].vnode, pTscObj->user); @@ -250,7 +250,7 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { pSql->thandle = thandle; pSql->ip = pVPeersDesc[pSql->index].ip; pSql->vnode = pVPeersDesc[pSql->index].vnode; - tscTrace("%p vnode:%d ip:0x%x index:%d is picked up, pConn:%p", pSql, pVPeersDesc[pSql->index].vnode, + tscTrace("%p vnode:%d ip:%p index:%d is picked up, pConn:%p", pSql, pVPeersDesc[pSql->index].vnode, pVPeersDesc[pSql->index].ip, pSql->index, pSql->thandle); #else *pCode = 0; @@ -283,7 +283,11 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { // the pSql->res.code is the previous error code. if (pSql->thandle == NULL && pSql->retry >= pSql->maxRetry) { - *pCode = pSql->res.code; + if (pSql->res.code != TSDB_CODE_SUCCESS) { + *pCode = pSql->res.code; + } + + tscError("%p reach the max retry:%d, code:%d", pSql, pSql->retry, *pCode); } } @@ -400,7 +404,6 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { // for single node situation, do NOT try next index #endif pSql->thandle = NULL; - // todo taos_stop_query() in async model /* * in case of diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index 2b7fe3cf44..e215022869 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -103,7 +103,7 @@ int mgmtProcessVpeerCfgMsg(char *cont, int contLen, SDnodeObj *pObj) { mTrace("dnode:%s, vnode:%d, vgroup:%d, send create meter msg, code:%d", taosIpStr(pObj->privateIp), vnode, pVgroup->vgId, *pMsg); } else { mTrace("dnode:%s, vnode:%d, no vgroup info, vgroup:%d", taosIpStr(pObj->privateIp), vnode, pObj->vload[vnode].vgId); - *pMsg = TSDB_CODE_INVALID_VALUE; + *pMsg = TSDB_CODE_NOT_ACTIVE_VNODE; pMsg++; *(int32_t *)pMsg = htonl(vnode); pMsg += sizeof(int32_t); From 95955cb474d57c1233546036d6b0507ed81c02b0 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 15 Nov 2019 19:01:01 +0800 Subject: [PATCH 48/69] [TBASE-1155] --- src/os/darwin/src/tdarwin.c | 12 ++++++++---- src/os/linux/src/tsystem.c | 19 ------------------- src/os/windows/src/twindows.c | 4 ---- src/system/detail/src/vnodeImport.c | 3 ++- src/util/src/tutil.c | 19 +++++++++++++++++++ 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/os/darwin/src/tdarwin.c b/src/os/darwin/src/tdarwin.c index eec252d660..71e8a56466 100644 --- a/src/os/darwin/src/tdarwin.c +++ b/src/os/darwin/src/tdarwin.c @@ -243,10 +243,6 @@ int taosInitTimer(void (*callback)(int), int ms) { return setitimer(ITIMER_REAL, &tv, NULL); } -char *taosCharsetReplace(char *charsetstr) { - return charsetstr; -} - void taosGetSystemTimezone() { // get and set default timezone SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); @@ -416,3 +412,11 @@ int tsem_post(dispatch_semaphore_t *sem) { int tsem_destroy(dispatch_semaphore_t *sem) { return 0; } + +int32_t __sync_val_load_32(int32_t *ptr) { + return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); +} + +void __sync_val_restore_32(int32_t *ptr, int32_t newval) { + __atomic_store_n(ptr, newval, __ATOMIC_RELEASE); +} \ No newline at end of file diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index fc2d9860d7..ccf9266f9b 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -210,25 +210,6 @@ void taosGetSystemTimezone() { pPrint("timezone not configured, set to system default:%s", tsTimezone); } -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - {"utf8", "UTF-8"}, {"936", "CP936"}, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} - /* * POSIX format locale string: * (Language Strings)_(Country/Region Strings).(code_page) diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 6c10ced6a2..9089f90d6a 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -205,10 +205,6 @@ __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { void tsPrintOsInfo() {} -char *taosCharsetReplace(char *charsetstr) { - return charsetstr; -} - void taosGetSystemTimezone() { // get and set default timezone SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index ed9c319216..4e1c3e33c6 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -546,6 +546,7 @@ int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { return code; } + assert(rows); dTrace("vid:%d sid:%d id:%s, %d rows data will be imported to cache, firstKey:%ld lastKey:%ld", pObj->vnode, pObj->sid, pObj->meterId, rows, firstKey, lastKey); @@ -781,7 +782,7 @@ int vnodeImportStartToCache(SImportInfo *pImport, char *payload, int rows) { pImport->importedRows = rows; code = vnodeImportToCache(pImport, payload, rows); } else { - dTrace("vid:%d sid:%d id:%s, data is already imported to cache", pObj->vnode, pObj->sid, pObj->meterId); + dTrace("vid:%d sid:%d id:%s, data is already imported to cache, firstKey:%lld", pObj->vnode, pObj->sid, pObj->meterId, pImport->firstKey); } return code; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 0d1ea9ed50..af75a3ce0f 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -517,3 +517,22 @@ FORCE_INLINE double taos_align_get_double(char* pBuf) { *(int64_t*)(&dv) = *(int64_t*)pBuf; return dv; } + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + { "utf8", "UTF-8" }, { "936", "CP936" }, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} From 460bc57a3071be81386844cbc37ab73903762de6 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 15 Nov 2019 22:28:03 +0800 Subject: [PATCH 49/69] =?UTF-8?q?In=20some=20cases,=20"mgmt=20node=20is=20?= =?UTF-8?q?gone=E2=80=9C=20error=20will=20occur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/system/detail/src/mgmtDnodeInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index 2b7fe3cf44..e215022869 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -103,7 +103,7 @@ int mgmtProcessVpeerCfgMsg(char *cont, int contLen, SDnodeObj *pObj) { mTrace("dnode:%s, vnode:%d, vgroup:%d, send create meter msg, code:%d", taosIpStr(pObj->privateIp), vnode, pVgroup->vgId, *pMsg); } else { mTrace("dnode:%s, vnode:%d, no vgroup info, vgroup:%d", taosIpStr(pObj->privateIp), vnode, pObj->vload[vnode].vgId); - *pMsg = TSDB_CODE_INVALID_VALUE; + *pMsg = TSDB_CODE_NOT_ACTIVE_VNODE; pMsg++; *(int32_t *)pMsg = htonl(vnode); pMsg += sizeof(int32_t); From de2ef67b066deef5049e4b727a036ae87c98e762 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 16 Nov 2019 10:35:12 +0800 Subject: [PATCH 50/69] [TBASE-1165] --- src/inc/taoserror.h | 142 ++++++++++++++++++++++++++++++++++++++++++++ src/inc/taosmsg.h | 116 +----------------------------------- 2 files changed, 143 insertions(+), 115 deletions(-) create mode 100644 src/inc/taoserror.h diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h new file mode 100644 index 0000000000..fbb7bf86da --- /dev/null +++ b/src/inc/taoserror.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TAOSERROR_H +#define TDENGINE_TAOSERROR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_CODE_SUCCESS 0 +#define TSDB_CODE_ACTION_IN_PROGRESS 1 + +#define TSDB_CODE_LAST_SESSION_NOT_FINISHED 5 +#define TSDB_CODE_INVALID_SESSION_ID 6 +#define TSDB_CODE_INVALID_TRAN_ID 7 +#define TSDB_CODE_INVALID_MSG_TYPE 8 +#define TSDB_CODE_ALREADY_PROCESSED 9 +#define TSDB_CODE_AUTH_FAILURE 10 +#define TSDB_CODE_WRONG_MSG_SIZE 11 +#define TSDB_CODE_UNEXPECTED_RESPONSE 12 +#define TSDB_CODE_INVALID_RESPONSE_TYPE 13 +#define TSDB_CODE_NO_RESOURCE 14 +#define TSDB_CODE_INVALID_TIME_STAMP 15 +#define TSDB_CODE_MISMATCHED_METER_ID 16 +#define TSDB_CODE_ACTION_TRANS_NOT_FINISHED 17 +#define TSDB_CODE_ACTION_NOT_ONLINE 18 +#define TSDB_CODE_ACTION_SEND_FAILD 19 +#define TSDB_CODE_NOT_ACTIVE_SESSION 20 +#define TSDB_CODE_INVALID_VNODE_ID 21 +#define TSDB_CODE_APP_ERROR 22 +#define TSDB_CODE_INVALID_IE 23 +#define TSDB_CODE_INVALID_VALUE 24 +#define TSDB_CODE_REDIRECT 25 +#define TSDB_CODE_ALREADY_THERE 26 +#define TSDB_CODE_INVALID_METER_ID 27 +#define TSDB_CODE_INVALID_SQL 28 +#define TSDB_CODE_NETWORK_UNAVAIL 29 +#define TSDB_CODE_INVALID_MSG_LEN 30 +#define TSDB_CODE_INVALID_DB 31 +#define TSDB_CODE_INVALID_TABLE 32 +#define TSDB_CODE_DB_ALREADY_EXIST 33 +#define TSDB_CODE_TABLE_ALREADY_EXIST 34 +#define TSDB_CODE_INVALID_USER 35 +#define TSDB_CODE_INVALID_ACCT 36 +#define TSDB_CODE_INVALID_PASS 37 +#define TSDB_CODE_DB_NOT_SELECTED 38 +#define TSDB_CODE_MEMORY_CORRUPTED 39 +#define TSDB_CODE_USER_ALREADY_EXIST 40 +#define TSDB_CODE_NO_RIGHTS 41 +#define TSDB_CODE_DISCONNECTED 42 +#define TSDB_CODE_NO_MASTER 43 +#define TSDB_CODE_NOT_CONFIGURED 44 +#define TSDB_CODE_INVALID_OPTION 45 +#define TSDB_CODE_NODE_OFFLINE 46 +#define TSDB_CODE_SYNC_REQUIRED 47 +#define TSDB_CODE_NO_ENOUGH_DNODES 48 +#define TSDB_CODE_UNSYNCED 49 +#define TSDB_CODE_TOO_SLOW 50 +#define TSDB_CODE_OTHERS 51 +#define TSDB_CODE_NO_REMOVE_MASTER 52 +#define TSDB_CODE_WRONG_SCHEMA 53 +#define TSDB_CODE_NOT_ACTIVE_VNODE 54 +#define TSDB_CODE_TOO_MANY_USERS 55 +#define TSDB_CODE_TOO_MANY_DATABSES 56 +#define TSDB_CODE_TOO_MANY_TABLES 57 +#define TSDB_CODE_TOO_MANY_DNODES 58 +#define TSDB_CODE_TOO_MANY_ACCTS 59 +#define TSDB_CODE_ACCT_ALREADY_EXIST 60 +#define TSDB_CODE_DNODE_ALREADY_EXIST 61 +#define TSDB_CODE_SDB_ERROR 62 +#define TSDB_CODE_METRICMETA_EXPIRED 63 // local cached metric-meta expired causes error in metric query +#define TSDB_CODE_NOT_READY 64 // peer is not ready to process data +#define TSDB_CODE_MAX_SESSIONS 65 // too many sessions +#define TSDB_CODE_MAX_CONNECTIONS 66 // too many connections +#define TSDB_CODE_SESSION_ALREADY_EXIST 67 +#define TSDB_CODE_NO_QSUMMARY 68 +#define TSDB_CODE_SERV_OUT_OF_MEMORY 69 +#define TSDB_CODE_INVALID_QHANDLE 70 +#define TSDB_CODE_RELATED_TABLES_EXIST 71 +#define TSDB_CODE_MONITOR_DB_FORBEIDDEN 72 +#define TSDB_CODE_VG_COMMITLOG_INIT_FAILED 73 +#define TSDB_CODE_VG_INIT_FAILED 74 +#define TSDB_CODE_DATA_ALREADY_IMPORTED 75 +#define TSDB_CODE_OPS_NOT_SUPPORT 76 +#define TSDB_CODE_INVALID_QUERY_ID 77 +#define TSDB_CODE_INVALID_STREAM_ID 78 +#define TSDB_CODE_INVALID_CONNECTION 79 +#define TSDB_CODE_ACTION_NOT_BALANCED 80 +#define TSDB_CODE_CLI_OUT_OF_MEMORY 81 +#define TSDB_CODE_DATA_OVERFLOW 82 +#define TSDB_CODE_QUERY_CANCELLED 83 +#define TSDB_CODE_GRANT_TIMESERIES_LIMITED 84 +#define TSDB_CODE_GRANT_EXPIRED 85 +#define TSDB_CODE_CLI_NO_DISKSPACE 86 +#define TSDB_CODE_FILE_CORRUPTED 87 +#define TSDB_CODE_INVALID_CLIENT_VERSION 88 +#define TSDB_CODE_INVALID_ACCT_PARAMETER 89 +#define TSDB_CODE_NOT_ENOUGH_TIME_SERIES 90 +#define TSDB_CODE_NO_WRITE_ACCESS 91 +#define TSDB_CODE_NO_READ_ACCESS 92 +#define TSDB_CODE_GRANT_DB_LIMITED 93 +#define TSDB_CODE_GRANT_USER_LIMITED 94 +#define TSDB_CODE_GRANT_CONN_LIMITED 95 +#define TSDB_CODE_GRANT_STREAM_LIMITED 96 +#define TSDB_CODE_GRANT_SPEED_LIMITED 97 +#define TSDB_CODE_GRANT_STORAGE_LIMITED 98 +#define TSDB_CODE_GRANT_QUERYTIME_LIMITED 99 +#define TSDB_CODE_GRANT_ACCT_LIMITED 100 +#define TSDB_CODE_GRANT_DNODE_LIMITED 101 +#define TSDB_CODE_GRANT_CPU_LIMITED 102 +#define TSDB_CODE_SESSION_NOT_READY 103 // table NOT in ready state +#define TSDB_CODE_BATCH_SIZE_TOO_BIG 104 +#define TSDB_CODE_TIMESTAMP_OUT_OF_RANGE 105 +#define TSDB_CODE_INVALID_QUERY_MSG 106 // failed to validate the sql expression msg by vnode +#define TSDB_CODE_CACHE_BLOCK_TS_DISORDERED 107 // time stamp in cache block is disordered +#define TSDB_CODE_FILE_BLOCK_TS_DISORDERED 108 // time stamp in file block is disordered +#define TSDB_CODE_INVALID_COMMIT_LOG 109 // commit log init failed +#define TSDB_CODE_SERVER_NO_SPACE 110 +#define TSDB_CODE_NOT_SUPER_TABLE 111 // operation only available for super table +#define TSDB_CODE_DUPLICATE_TAGS 112 // tags value for join not unique +#define TSDB_CODE_INVALID_SUBMIT_MSG 113 +#define TSDB_CODE_NOT_ACTIVE_TABLE 114 +#define TSDB_CODE_INVALID_TABLE_ID 115 + +#ifdef __cplusplus +} +#endif + +#endif //TDENGINE_TAOSERROR_H diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 71e4c75634..e75404ad98 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -21,121 +21,7 @@ extern "C" { #endif #include "tsdb.h" - -#define TSDB_CODE_SUCCESS 0 -#define TSDB_CODE_ACTION_IN_PROGRESS 1 - -#define TSDB_CODE_LAST_SESSION_NOT_FINISHED 5 -#define TSDB_CODE_INVALID_SESSION_ID 6 -#define TSDB_CODE_INVALID_TRAN_ID 7 -#define TSDB_CODE_INVALID_MSG_TYPE 8 -#define TSDB_CODE_ALREADY_PROCESSED 9 -#define TSDB_CODE_AUTH_FAILURE 10 -#define TSDB_CODE_WRONG_MSG_SIZE 11 -#define TSDB_CODE_UNEXPECTED_RESPONSE 12 -#define TSDB_CODE_INVALID_RESPONSE_TYPE 13 -#define TSDB_CODE_NO_RESOURCE 14 -#define TSDB_CODE_INVALID_TIME_STAMP 15 -#define TSDB_CODE_MISMATCHED_METER_ID 16 -#define TSDB_CODE_ACTION_TRANS_NOT_FINISHED 17 -#define TSDB_CODE_ACTION_NOT_ONLINE 18 -#define TSDB_CODE_ACTION_SEND_FAILD 19 -#define TSDB_CODE_NOT_ACTIVE_SESSION 20 -#define TSDB_CODE_INVALID_VNODE_ID 21 -#define TSDB_CODE_APP_ERROR 22 -#define TSDB_CODE_INVALID_IE 23 -#define TSDB_CODE_INVALID_VALUE 24 -#define TSDB_CODE_REDIRECT 25 -#define TSDB_CODE_ALREADY_THERE 26 -#define TSDB_CODE_INVALID_METER_ID 27 -#define TSDB_CODE_INVALID_SQL 28 -#define TSDB_CODE_NETWORK_UNAVAIL 29 -#define TSDB_CODE_INVALID_MSG_LEN 30 -#define TSDB_CODE_INVALID_DB 31 -#define TSDB_CODE_INVALID_TABLE 32 -#define TSDB_CODE_DB_ALREADY_EXIST 33 -#define TSDB_CODE_TABLE_ALREADY_EXIST 34 -#define TSDB_CODE_INVALID_USER 35 -#define TSDB_CODE_INVALID_ACCT 36 -#define TSDB_CODE_INVALID_PASS 37 -#define TSDB_CODE_DB_NOT_SELECTED 38 -#define TSDB_CODE_MEMORY_CORRUPTED 39 -#define TSDB_CODE_USER_ALREADY_EXIST 40 -#define TSDB_CODE_NO_RIGHTS 41 -#define TSDB_CODE_DISCONNECTED 42 -#define TSDB_CODE_NO_MASTER 43 -#define TSDB_CODE_NOT_CONFIGURED 44 -#define TSDB_CODE_INVALID_OPTION 45 -#define TSDB_CODE_NODE_OFFLINE 46 -#define TSDB_CODE_SYNC_REQUIRED 47 -#define TSDB_CODE_NO_ENOUGH_DNODES 48 -#define TSDB_CODE_UNSYNCED 49 -#define TSDB_CODE_TOO_SLOW 50 -#define TSDB_CODE_OTHERS 51 -#define TSDB_CODE_NO_REMOVE_MASTER 52 -#define TSDB_CODE_WRONG_SCHEMA 53 -#define TSDB_CODE_NOT_ACTIVE_VNODE 54 -#define TSDB_CODE_TOO_MANY_USERS 55 -#define TSDB_CODE_TOO_MANY_DATABSES 56 -#define TSDB_CODE_TOO_MANY_TABLES 57 -#define TSDB_CODE_TOO_MANY_DNODES 58 -#define TSDB_CODE_TOO_MANY_ACCTS 59 -#define TSDB_CODE_ACCT_ALREADY_EXIST 60 -#define TSDB_CODE_DNODE_ALREADY_EXIST 61 -#define TSDB_CODE_SDB_ERROR 62 -#define TSDB_CODE_METRICMETA_EXPIRED 63 // local cached metric-meta expired causes error in metric query -#define TSDB_CODE_NOT_READY 64 // peer is not ready to process data -#define TSDB_CODE_MAX_SESSIONS 65 // too many sessions -#define TSDB_CODE_MAX_CONNECTIONS 66 // too many connections -#define TSDB_CODE_SESSION_ALREADY_EXIST 67 -#define TSDB_CODE_NO_QSUMMARY 68 -#define TSDB_CODE_SERV_OUT_OF_MEMORY 69 -#define TSDB_CODE_INVALID_QHANDLE 70 -#define TSDB_CODE_RELATED_TABLES_EXIST 71 -#define TSDB_CODE_MONITOR_DB_FORBEIDDEN 72 -#define TSDB_CODE_VG_COMMITLOG_INIT_FAILED 73 -#define TSDB_CODE_VG_INIT_FAILED 74 -#define TSDB_CODE_DATA_ALREADY_IMPORTED 75 -#define TSDB_CODE_OPS_NOT_SUPPORT 76 -#define TSDB_CODE_INVALID_QUERY_ID 77 -#define TSDB_CODE_INVALID_STREAM_ID 78 -#define TSDB_CODE_INVALID_CONNECTION 79 -#define TSDB_CODE_ACTION_NOT_BALANCED 80 -#define TSDB_CODE_CLI_OUT_OF_MEMORY 81 -#define TSDB_CODE_DATA_OVERFLOW 82 -#define TSDB_CODE_QUERY_CANCELLED 83 -#define TSDB_CODE_GRANT_TIMESERIES_LIMITED 84 -#define TSDB_CODE_GRANT_EXPIRED 85 -#define TSDB_CODE_CLI_NO_DISKSPACE 86 -#define TSDB_CODE_FILE_CORRUPTED 87 -#define TSDB_CODE_INVALID_CLIENT_VERSION 88 -#define TSDB_CODE_INVALID_ACCT_PARAMETER 89 -#define TSDB_CODE_NOT_ENOUGH_TIME_SERIES 90 -#define TSDB_CODE_NO_WRITE_ACCESS 91 -#define TSDB_CODE_NO_READ_ACCESS 92 -#define TSDB_CODE_GRANT_DB_LIMITED 93 -#define TSDB_CODE_GRANT_USER_LIMITED 94 -#define TSDB_CODE_GRANT_CONN_LIMITED 95 -#define TSDB_CODE_GRANT_STREAM_LIMITED 96 -#define TSDB_CODE_GRANT_SPEED_LIMITED 97 -#define TSDB_CODE_GRANT_STORAGE_LIMITED 98 -#define TSDB_CODE_GRANT_QUERYTIME_LIMITED 99 -#define TSDB_CODE_GRANT_ACCT_LIMITED 100 -#define TSDB_CODE_GRANT_DNODE_LIMITED 101 -#define TSDB_CODE_GRANT_CPU_LIMITED 102 -#define TSDB_CODE_SESSION_NOT_READY 103 // table NOT in ready state -#define TSDB_CODE_BATCH_SIZE_TOO_BIG 104 -#define TSDB_CODE_TIMESTAMP_OUT_OF_RANGE 105 -#define TSDB_CODE_INVALID_QUERY_MSG 106 // failed to validate the sql expression msg by vnode -#define TSDB_CODE_CACHE_BLOCK_TS_DISORDERED 107 // time stamp in cache block is disordered -#define TSDB_CODE_FILE_BLOCK_TS_DISORDERED 108 // time stamp in file block is disordered -#define TSDB_CODE_INVALID_COMMIT_LOG 109 // commit log init failed -#define TSDB_CODE_SERVER_NO_SPACE 110 -#define TSDB_CODE_NOT_SUPER_TABLE 111 // -#define TSDB_CODE_DUPLICATE_TAGS 112 // tags value for join not unique -#define TSDB_CODE_INVALID_SUBMIT_MSG 113 -#define TSDB_CODE_NOT_ACTIVE_TABLE 114 -#define TSDB_CODE_INVALID_TABLE_ID 115 +#include "taoserror.h" // message type #define TSDB_MSG_TYPE_REG 1 From 4e7e75b7ad34e142953cfeca307c7397226d730f Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 16 Nov 2019 10:39:47 +0800 Subject: [PATCH 51/69] 4] --- src/client/src/tscServer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8a6a7e30ab..025b7968ac 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -281,9 +281,9 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { break; } - // the pSql->res.code is the previous error code. + // the pSql->res.code is the previous error(status) code. if (pSql->thandle == NULL && pSql->retry >= pSql->maxRetry) { - if (pSql->res.code != TSDB_CODE_SUCCESS) { + if (pSql->res.code != TSDB_CODE_SUCCESS && pSql->res.code != TSDB_CODE_ACTION_IN_PROGRESS) { *pCode = pSql->res.code; } From 9c7fb6b00faf26250b94d1f32f8bc60a71feee59 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 16 Nov 2019 11:29:19 +0800 Subject: [PATCH 52/69] [TBASE-1161] --- src/inc/sdb.h | 6 +++--- src/inc/trpc.h | 3 +-- src/util/src/tglobalcfg.c | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/inc/sdb.h b/src/inc/sdb.h index 4a1969b700..f645039441 100644 --- a/src/inc/sdb.h +++ b/src/inc/sdb.h @@ -23,10 +23,10 @@ extern "C" { #include "taosmsg.h" #include "tsdb.h" -extern short sdbPeerPort; -extern short sdbSyncPort; +extern short tsMgmtMgmtPort; +extern short tsMgmtSyncPort; extern int sdbMaxNodes; -extern int sdbHbTimer; // seconds +extern int tsMgmtPeerHBTimer; // seconds extern char sdbZone[]; extern char sdbMasterIp[]; extern char sdbPrivateIp[]; diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 3ae6e9ea97..ef86e672d1 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -25,7 +25,6 @@ extern "C" { #define TAOS_CONN_UDPS 0 #define TAOS_CONN_UDPC 1 -#define TAOS_CONN_UDP 1 #define TAOS_CONN_TCPS 2 #define TAOS_CONN_TCPC 3 #define TAOS_CONN_HTTPS 4 @@ -39,7 +38,7 @@ extern "C" { #define TAOS_ID_REALLOCATE 2 #define TAOS_CONN_SOCKET_TYPE_S() ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPS:TAOS_CONN_TCPS) -#define TAOS_CONN_SOCKET_TYPE_C() ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDP:TAOS_CONN_TCPC) +#define TAOS_CONN_SOCKET_TYPE_C() ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPC:TAOS_CONN_TCPC) #define taosSendMsgToPeer(x, y, z) taosSendMsgToPeerH(x, y, z, NULL) #define taosOpenRpcChann(x, y, z) taosOpenRpcChannWithQ(x,y,z,NULL) diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bac89db7e2..407761cff0 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -62,8 +62,8 @@ short tsMgmtShellPort = 6030; // udp[6030-6034] tcp[6030] short tsVnodeShellPort = 6035; // udp[6035-6039] tcp[6035] short tsMgmtVnodePort = 6040; // udp[6040-6044] tcp[6040] short tsVnodeVnodePort = 6045; // tcp[6045] -short tsMgmtMgmtPort = 6050; // sdbPeerPort only udp, numOfVnodes fixed to 1, range udp[6050] -short tsMgmtSyncPort = 6050; // sdbSyncPort only tcp, range tcp[6050] +short tsMgmtMgmtPort = 6050; // udp, numOfVnodes fixed to 1, range udp[6050] +short tsMgmtSyncPort = 6050; // tcp, range tcp[6050] int tsStatusInterval = 1; // second int tsShellActivityTimer = 3; // second From edbf603ee44326fbf586c0185a5e53b5bd4b2dea Mon Sep 17 00:00:00 2001 From: lihui Date: Sat, 16 Nov 2019 13:35:46 +0800 Subject: [PATCH 53/69] [change return error result] --- src/client/src/tscParseInsert.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index c8ce728de1..abebea64df 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -191,10 +191,10 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *((int8_t *)payload) = TSDB_DATA_TINYINT_NULL; } else { numType = tscToInteger(pToken, &iv, &endptr); - if (errno == ERANGE || iv > INT8_MAX || iv <= INT8_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } else if (TK_ILLEGAL == numType) { + if (TK_ILLEGAL == numType) { INVALID_SQL_RET_MSG(msg, "data is illegal"); + } else if (errno == ERANGE || iv > INT8_MAX || iv <= INT8_MIN) { + INVALID_SQL_RET_MSG(msg, "data is overflow"); } *((int8_t *)payload) = (int8_t)iv; @@ -210,10 +210,10 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; } else { numType = tscToInteger(pToken, &iv, &endptr); - if (errno == ERANGE || iv > INT16_MAX || iv <= INT16_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } else if (TK_ILLEGAL == numType) { + if (TK_ILLEGAL == numType) { INVALID_SQL_RET_MSG(msg, "data is illegal"); + } else if (errno == ERANGE || iv > INT16_MAX || iv <= INT16_MIN) { + INVALID_SQL_RET_MSG(msg, "data is overflow"); } *((int16_t *)payload) = (int16_t)iv; @@ -228,10 +228,10 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *((int32_t *)payload) = TSDB_DATA_INT_NULL; } else { numType = tscToInteger(pToken, &iv, &endptr); - if (errno == ERANGE || iv > INT32_MAX || iv <= INT32_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } else if (TK_ILLEGAL == numType) { + if (TK_ILLEGAL == numType) { INVALID_SQL_RET_MSG(msg, "data is illegal"); + } else if (errno == ERANGE || iv > INT32_MAX || iv <= INT32_MIN) { + INVALID_SQL_RET_MSG(msg, "data is overflow"); } *((int32_t *)payload) = (int32_t)iv; @@ -247,10 +247,10 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; } else { numType = tscToInteger(pToken, &iv, &endptr); - if (errno == ERANGE || iv > INT64_MAX || iv <= INT64_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } else if (TK_ILLEGAL == numType) { + if (TK_ILLEGAL == numType) { INVALID_SQL_RET_MSG(msg, "data is illegal"); + } else if (errno == ERANGE || iv > INT64_MAX || iv <= INT64_MIN) { + INVALID_SQL_RET_MSG(msg, "data is overflow"); } *((int64_t *)payload) = iv; From 1efed1e01b13291e64cb5f1727d468fcf9a5b8b0 Mon Sep 17 00:00:00 2001 From: lihui Date: Sat, 16 Nov 2019 13:37:05 +0800 Subject: [PATCH 54/69] [add taoserror.h] --- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ packaging/tools/install.sh | 3 ++- packaging/tools/make_install.sh | 5 +++-- packaging/tools/post.sh | 3 ++- packaging/tools/preun.sh | 1 + packaging/tools/remove.sh | 1 + 7 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index d54953113a..e4c795f1b8 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -49,6 +49,7 @@ cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples cp -r ${top_dir}/src/connector/grafana ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 6e9e5e6f87..9ede4102e7 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -62,6 +62,7 @@ cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include cp -r %{_compiledir}/../src/connector/grafana %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector @@ -138,6 +139,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${log_link_dir} || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index acd6c6ccce..4b653dec9c 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -98,9 +98,10 @@ function install_lib() { } function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function install_config() { diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 4660ba5b2a..c43f918e8d 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -109,9 +109,10 @@ function install_lib() { function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h || : - ${csudo} cp -f ${source_dir}/src/inc/taos.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function install_config() { diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 9e18783657..11f3ab9a9b 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -61,8 +61,9 @@ function kill_taosd() { } function install_include() { - ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h|| : ${csudo} ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h } function install_lib() { diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 4566db1b5a..a08f4e304e 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -94,6 +94,7 @@ ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : +${csudo} rm -f ${inc_link_dir}/taoserror.h || : ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${log_link_dir} || : diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 4ada19762c..6c2c6d5ea8 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -61,6 +61,7 @@ function clean_lib() { function clean_header() { # Remove link ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : } function clean_config() { From 55ed99d78e413a43070a0ad1e5c9812146eb77a9 Mon Sep 17 00:00:00 2001 From: lihui Date: Sat, 16 Nov 2019 13:39:51 +0800 Subject: [PATCH 55/69] [add taoserror.h] --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 1a24ae8a1f..967b14cee9 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -22,7 +22,7 @@ install_dir="${release_dir}/taos-${version}-${package_name}-$(echo ${build_time} bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh" versioninfo=$(${script_dir}/get_version.sh ${code_dir}/util/src/version.c) lib_files="${build_dir}/lib/libtaos.so.${versioninfo}" -header_files="${code_dir}/inc/taos.h" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" cfg_files="${top_dir}/packaging/cfg/*.cfg" install_files="${script_dir}/install.sh ${script_dir}/install_client.sh" From 6e7b3cf17a7a53860d4cb67afa9ef0c422e1a628 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Nov 2019 14:36:54 +0800 Subject: [PATCH 56/69] add an assert for debug purpose --- src/system/detail/src/vnodeFile.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index fdafc01352..132ee3f7de 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -216,6 +216,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (numOfFiles >= pVnode->numOfFiles) { // create empty header files backward filesAdded = numOfFiles - pVnode->numOfFiles + 1; + assert(filesAdded <= pVnode->maxFiles + 2); for (int i = 0; i < filesAdded; ++i) { fileId = pVnode->fileId - pVnode->numOfFiles - i; if (vnodeCreateEmptyCompFile(vnode, fileId) < 0) From c302d338e7d5af23e16f678b26e549b872bf7f0c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 16 Nov 2019 15:30:28 +0800 Subject: [PATCH 57/69] refactor code, reorganize the header files --- src/kit/shell/src/shellCommand.c | 8 -------- src/kit/shell/src/shellEngine.c | 10 ---------- src/kit/shell/src/shellLinux.c | 20 +------------------- src/kit/shell/src/shellMain.c | 7 ------- src/modules/http/src/httpServer.c | 18 +----------------- src/os/linux/inc/os.h | 8 ++++++++ src/rpc/src/thaship.c | 8 -------- src/rpc/src/tmsghdr.c | 5 +---- src/rpc/src/trpc.c | 10 +--------- src/rpc/src/ttcpclient.c | 9 --------- src/rpc/src/ttcpserver.c | 10 ---------- src/rpc/src/tudp.c | 9 --------- src/sdb/src/hashstr.c | 8 +------- src/sdb/src/sdbEngine.c | 14 +------------- src/system/detail/inc/mgmt.h | 7 +------ src/system/detail/inc/mgmtBalance.h | 4 +--- src/system/detail/inc/mgmtSystem.h | 2 -- src/system/detail/inc/vnode.h | 7 +------ src/system/detail/inc/vnodePeer.h | 2 +- src/system/detail/inc/vnodeQueryImpl.h | 3 +-- src/system/detail/inc/vnodeRead.h | 2 +- src/system/detail/inc/vnodeShell.h | 2 +- src/system/detail/inc/vnodeStore.h | 2 -- src/system/detail/inc/vnodeSystem.h | 2 -- src/system/detail/src/dnodeMgmt.c | 5 ++--- src/system/detail/src/dnodeService.c | 16 ++-------------- src/system/detail/src/dnodeSystem.c | 6 +----- src/system/detail/src/mgmtAcct.c | 2 +- src/system/detail/src/mgmtConn.c | 4 +++- src/system/detail/src/mgmtDnode.c | 4 +--- src/system/detail/src/mgmtDnodeInt.c | 3 +-- src/system/detail/src/mgmtMeter.c | 5 +---- src/system/detail/src/mgmtProfile.c | 3 ++- src/system/detail/src/mgmtShell.c | 2 +- src/system/detail/src/mgmtSupertableQuery.c | 5 +---- src/system/detail/src/mgmtSystem.c | 11 +---------- src/system/detail/src/mgmtUser.c | 2 +- src/system/detail/src/mgmtUtil.c | 5 +---- src/system/detail/src/mgmtVgroup.c | 2 +- src/system/detail/src/vnodeCache.c | 4 +--- src/system/detail/src/vnodeCommit.c | 10 +--------- src/system/detail/src/vnodeFile.c | 10 +--------- src/system/detail/src/vnodeFileUtil.c | 6 +----- src/system/detail/src/vnodeFilterFunc.c | 6 +----- src/system/detail/src/vnodeImport.c | 5 +---- src/system/detail/src/vnodeMeter.c | 5 +---- src/system/detail/src/vnodeQueryProcess.c | 5 +---- src/system/detail/src/vnodeRead.c | 7 +------ src/system/detail/src/vnodeShell.c | 6 ++---- src/system/detail/src/vnodeStore.c | 6 +----- src/system/detail/src/vnodeSystem.c | 10 +--------- src/system/detail/src/vnodeTagMgmt.c | 4 +--- src/system/detail/src/vnodeUtil.c | 5 +---- src/util/src/ihash.c | 7 ------- src/util/src/tcache.c | 10 +--------- src/util/src/tcompression.c | 6 ------ src/util/src/textbuffer.c | 11 ----------- src/util/src/thash.c | 7 ------- src/util/src/thistogram.c | 8 +------- src/util/src/tidpool.c | 5 +---- src/util/src/tlog.c | 16 ---------------- src/util/src/tmem.c | 8 +------- src/util/src/tmodule.c | 6 ------ src/util/src/tsched.c | 8 -------- src/util/src/tsocket.c | 11 ----------- src/util/src/ttimer.c | 6 ------ src/util/src/ttypes.c | 9 --------- src/util/src/tutil.c | 8 -------- 68 files changed, 60 insertions(+), 407 deletions(-) diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c index 9bf182ab0e..46aa04c1d7 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/src/kit/shell/src/shellCommand.c @@ -13,16 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #define __USE_XOPEN -#include - #include "os.h" #include "shell.h" #include "shellCommand.h" diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 34f51e3b3f..e2897da698 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -16,16 +16,6 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "shell.h" #include "shellCommand.h" diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 145025cbe1..0ece4efbb2 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -13,27 +13,9 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #define __USE_XOPEN -#include +#include "os.h" #include "shell.h" #include "shellCommand.h" diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 63c9eac0db..82333020f1 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" #include "shell.h" #include "tsclient.h" diff --git a/src/modules/http/src/httpServer.c b/src/modules/http/src/httpServer.c index 49ff3562bf..ae8f9eeaad 100644 --- a/src/modules/http/src/httpServer.c +++ b/src/modules/http/src/httpServer.c @@ -13,23 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "tlog.h" diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 6ecb64eda5..dc3dc1cfdf 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -23,6 +23,7 @@ extern "C" { #include #include +#include #include #include #include @@ -31,6 +32,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -41,9 +43,12 @@ extern "C" { #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -55,7 +60,9 @@ extern "C" { #include #include #include +#include #include +#include #include #include #include @@ -65,6 +72,7 @@ extern "C" { #include #include + #define taosCloseSocket(x) \ { \ if (VALIDFD(x)) { \ diff --git a/src/rpc/src/thaship.c b/src/rpc/src/thaship.c index 4cc6feeea3..2c46e53258 100644 --- a/src/rpc/src/thaship.c +++ b/src/rpc/src/thaship.c @@ -13,14 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tlog.h" #include "tmempool.h" diff --git a/src/rpc/src/tmsghdr.c b/src/rpc/src/tmsghdr.c index 960d1fc5f6..a46f182b1d 100644 --- a/src/rpc/src/tmsghdr.c +++ b/src/rpc/src/tmsghdr.c @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include +#include "os.h" void taosFreeMsgHdr(void *hdr) { struct msghdr *msgHdr = (struct msghdr *)hdr; diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 41c628e169..de21598dd9 100644 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -13,16 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" + #include "shash.h" #include "taosmsg.h" #include "tidpool.h" diff --git a/src/rpc/src/ttcpclient.c b/src/rpc/src/ttcpclient.c index e12f1e1728..8e6f91a661 100644 --- a/src/rpc/src/ttcpclient.c +++ b/src/rpc/src/ttcpclient.c @@ -13,15 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tlog.h" diff --git a/src/rpc/src/ttcpserver.c b/src/rpc/src/ttcpserver.c index b6b0e07230..04078abb64 100644 --- a/src/rpc/src/ttcpserver.c +++ b/src/rpc/src/ttcpserver.c @@ -13,16 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tlog.h" diff --git a/src/rpc/src/tudp.c b/src/rpc/src/tudp.c index 1940c89e85..7a4961f90c 100644 --- a/src/rpc/src/tudp.c +++ b/src/rpc/src/tudp.c @@ -13,15 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "thash.h" diff --git a/src/sdb/src/hashstr.c b/src/sdb/src/hashstr.c index 1a9a7fefb4..4b9aa33928 100644 --- a/src/sdb/src/hashstr.c +++ b/src/sdb/src/hashstr.c @@ -13,13 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tsdb.h" #define MAX_STR_LEN 40 diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c index 59e3d7e039..326e9a4df2 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/sdb/src/sdbEngine.c @@ -13,19 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "sdb.h" #include "sdbint.h" diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index 1d1f325ec2..702eb00875 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include -#include +#include "os.h" #include "sdb.h" #include "tglobalcfg.h" @@ -34,7 +30,6 @@ extern "C" { #include "tmempool.h" #include "trpc.h" #include "tsdb.h" -#include "tsdb.h" #include "tskiplist.h" #include "tsocket.h" #include "ttime.h" diff --git a/src/system/detail/inc/mgmtBalance.h b/src/system/detail/inc/mgmtBalance.h index 4157458a7f..7d478416ac 100644 --- a/src/system/detail/inc/mgmtBalance.h +++ b/src/system/detail/inc/mgmtBalance.h @@ -20,9 +20,7 @@ extern "C" { #endif -#include -#include -#include +#include "os.h" #include "dnodeSystem.h" #include "mgmt.h" diff --git a/src/system/detail/inc/mgmtSystem.h b/src/system/detail/inc/mgmtSystem.h index 1262d7e834..44bb0331e1 100644 --- a/src/system/detail/inc/mgmtSystem.h +++ b/src/system/detail/inc/mgmtSystem.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include - int mgmtInitRedirect(); void mgmtCleanUpRedirect(); diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 373d9e713b..35b5c010ab 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include -#include +#include "os.h" #include "tglobalcfg.h" #include "tidpool.h" @@ -33,7 +29,6 @@ extern "C" { #include "trpc.h" #include "tsclient.h" #include "tsdb.h" -#include "tsdb.h" #include "tsocket.h" #include "ttime.h" #include "ttimer.h" diff --git a/src/system/detail/inc/vnodePeer.h b/src/system/detail/inc/vnodePeer.h index d44143e619..4f17e66a70 100644 --- a/src/system/detail/inc/vnodePeer.h +++ b/src/system/detail/inc/vnodePeer.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_VNODEPEER_H #define TDENGINE_VNODEPEER_H -#include +#include "os.h" #ifdef __cplusplus extern "C" { diff --git a/src/system/detail/inc/vnodeQueryImpl.h b/src/system/detail/inc/vnodeQueryImpl.h index c00af3b8e9..810105d638 100644 --- a/src/system/detail/inc/vnodeQueryImpl.h +++ b/src/system/detail/inc/vnodeQueryImpl.h @@ -20,8 +20,7 @@ extern "C" { #endif -#include -#include +#include "os.h" #include "ihash.h" diff --git a/src/system/detail/inc/vnodeRead.h b/src/system/detail/inc/vnodeRead.h index 20f49354f2..1de9f97e82 100644 --- a/src/system/detail/inc/vnodeRead.h +++ b/src/system/detail/inc/vnodeRead.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include +#include "os.h" #include "tinterpolation.h" #include "vnodeTagMgmt.h" diff --git a/src/system/detail/inc/vnodeShell.h b/src/system/detail/inc/vnodeShell.h index d0194a1765..e450983dd7 100644 --- a/src/system/detail/inc/vnodeShell.h +++ b/src/system/detail/inc/vnodeShell.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include +#include "os.h" typedef struct { int sid; diff --git a/src/system/detail/inc/vnodeStore.h b/src/system/detail/inc/vnodeStore.h index d4eedd4ce0..638bcb54bb 100644 --- a/src/system/detail/inc/vnodeStore.h +++ b/src/system/detail/inc/vnodeStore.h @@ -16,8 +16,6 @@ #ifndef TDENGINE_VNODESTORE_H #define TDENGINE_VNODESTORE_H -#include - #ifdef __cplusplus extern "C" { #endif diff --git a/src/system/detail/inc/vnodeSystem.h b/src/system/detail/inc/vnodeSystem.h index e69b0b9f1e..e436288fc1 100644 --- a/src/system/detail/inc/vnodeSystem.h +++ b/src/system/detail/inc/vnodeSystem.h @@ -16,8 +16,6 @@ #ifndef TDENGINE_VNODESYSTEM_H #define TDENGINE_VNODESYSTEM_H -#include - #ifdef __cplusplus extern "C" { #endif diff --git a/src/system/detail/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c index d3f84639a6..ac651f05f2 100644 --- a/src/system/detail/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -14,9 +14,8 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include + +#include "os.h" #include "dnodeSystem.h" #include "taosmsg.h" diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index 86d3b4a795..a14ec1fda6 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -14,20 +14,8 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include + +#include "os.h" #include "dnodeSystem.h" #include "tglobalcfg.h" diff --git a/src/system/detail/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c index 1b5d9418b4..a6392cee4d 100644 --- a/src/system/detail/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -14,11 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include +#include "os.h" #include "mgmt.h" #include "vnode.h" diff --git a/src/system/detail/src/mgmtAcct.c b/src/system/detail/src/mgmtAcct.c index dac67518e9..e5f52b17f9 100644 --- a/src/system/detail/src/mgmtAcct.c +++ b/src/system/detail/src/mgmtAcct.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +#include "os.h" #include "mgmt.h" #include "tschemautil.h" diff --git a/src/system/detail/src/mgmtConn.c b/src/system/detail/src/mgmtConn.c index b3fb24de6b..40385d2fc4 100644 --- a/src/system/detail/src/mgmtConn.c +++ b/src/system/detail/src/mgmtConn.c @@ -14,8 +14,10 @@ */ #define _DEFAULT_SOURCE + +#include "os.h" + #include "mgmt.h" -#include #include "taosmsg.h" #include "tschemautil.h" diff --git a/src/system/detail/src/mgmtDnode.c b/src/system/detail/src/mgmtDnode.c index 36e3a41595..3a4936041c 100644 --- a/src/system/detail/src/mgmtDnode.c +++ b/src/system/detail/src/mgmtDnode.c @@ -15,9 +15,7 @@ #define _DEFAULT_SOURCE -#include -#include -#include +#include "os.h" #include "dnodeSystem.h" #include "mgmt.h" diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index e215022869..af86757d51 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -14,8 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include +#include "os.h" #include "dnodeSystem.h" #include "mgmt.h" diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 2db920fdd7..d4ffabe4b7 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "mgmt.h" #include "mgmtUtil.h" diff --git a/src/system/detail/src/mgmtProfile.c b/src/system/detail/src/mgmtProfile.c index e641739e31..bd5540e5ee 100644 --- a/src/system/detail/src/mgmtProfile.c +++ b/src/system/detail/src/mgmtProfile.c @@ -14,8 +14,9 @@ */ #define _DEFAULT_SOURCE +#include "os.h" + #include "mgmt.h" -#include #include "mgmtProfile.h" #include "taosmsg.h" #include "tschemautil.h" diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index f74ee9f51f..efd863a7ef 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +#include "os.h" #include "dnodeSystem.h" #include "mgmt.h" diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index 31f3e29425..f5e0509c24 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "mgmt.h" #include "mgmtUtil.h" diff --git a/src/system/detail/src/mgmtSystem.c b/src/system/detail/src/mgmtSystem.c index bb05c35e9b..375e100a83 100644 --- a/src/system/detail/src/mgmtSystem.c +++ b/src/system/detail/src/mgmtSystem.c @@ -14,16 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "dnodeSystem.h" #include "mgmt.h" diff --git a/src/system/detail/src/mgmtUser.c b/src/system/detail/src/mgmtUser.c index 83e619841b..1acceb1ade 100644 --- a/src/system/detail/src/mgmtUser.c +++ b/src/system/detail/src/mgmtUser.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +#include "os.h" #include "mgmt.h" #include "tschemautil.h" diff --git a/src/system/detail/src/mgmtUtil.c b/src/system/detail/src/mgmtUtil.c index 3be2e1288d..d31198be64 100644 --- a/src/system/detail/src/mgmtUtil.c +++ b/src/system/detail/src/mgmtUtil.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "mgmt.h" #include "mgmtUtil.h" diff --git a/src/system/detail/src/mgmtVgroup.c b/src/system/detail/src/mgmtVgroup.c index 5858be54ca..3b1da25a0a 100644 --- a/src/system/detail/src/mgmtVgroup.c +++ b/src/system/detail/src/mgmtVgroup.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +#include "os.h" #include "mgmt.h" #include "tschemautil.h" diff --git a/src/system/detail/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c index f4bea682e9..94ac006060 100644 --- a/src/system/detail/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "vnode.h" diff --git a/src/system/detail/src/vnodeCommit.c b/src/system/detail/src/vnodeCommit.c index af14f2be4a..73c891f7bb 100644 --- a/src/system/detail/src/vnodeCommit.c +++ b/src/system/detail/src/vnodeCommit.c @@ -14,15 +14,7 @@ */ #define _GNU_SOURCE /* See feature_test_macros(7) */ -#include - -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tsdb.h" #include "vnode.h" diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index fdafc01352..5292f5bac7 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -14,15 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tscompression.h" #include "tutil.h" diff --git a/src/system/detail/src/vnodeFileUtil.c b/src/system/detail/src/vnodeFileUtil.c index f8de6c4a42..b40e7cfd41 100644 --- a/src/system/detail/src/vnodeFileUtil.c +++ b/src/system/detail/src/vnodeFileUtil.c @@ -14,11 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include +#include "os.h" #include "vnode.h" diff --git a/src/system/detail/src/vnodeFilterFunc.c b/src/system/detail/src/vnodeFilterFunc.c index bd829460d9..08fe78c188 100644 --- a/src/system/detail/src/vnodeFilterFunc.c +++ b/src/system/detail/src/vnodeFilterFunc.c @@ -14,11 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "tsqlfunction.h" diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index de795f820b..260d531200 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "trpc.h" #include "ttimer.h" diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index e3cde9a84d..14cdf9eb78 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "trpc.h" #include "tschemautil.h" diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index 73d97f5e6e..7378963301 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "textbuffer.h" diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index f35567e720..0f9565a3b5 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -14,12 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include +#include "os.h" #include "ihash.h" #include "taosmsg.h" diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 5973b01d71..47b7b470bb 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -14,10 +14,8 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" + #include "taosmsg.h" #include "vnode.h" #include "vnodeShell.h" diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index e4294d72c6..d299d7fbf8 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -14,11 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include +#include "os.h" #include "dnodeSystem.h" #include "trpc.h" diff --git a/src/system/detail/src/vnodeSystem.c b/src/system/detail/src/vnodeSystem.c index 2f350db3fa..631e258b25 100644 --- a/src/system/detail/src/vnodeSystem.c +++ b/src/system/detail/src/vnodeSystem.c @@ -14,15 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tsdb.h" #include "tsocket.h" diff --git a/src/system/detail/src/vnodeTagMgmt.c b/src/system/detail/src/vnodeTagMgmt.c index 9b76e14854..adf4e544bb 100644 --- a/src/system/detail/src/vnodeTagMgmt.c +++ b/src/system/detail/src/vnodeTagMgmt.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include +#include "os.h" #include "tsdb.h" #include "tlog.h" diff --git a/src/system/detail/src/vnodeUtil.c b/src/system/detail/src/vnodeUtil.c index 6691b2245d..b8c463f50b 100644 --- a/src/system/detail/src/vnodeUtil.c +++ b/src/system/detail/src/vnodeUtil.c @@ -14,10 +14,7 @@ */ #define _DEFAULT_SOURCE -#include -#include -#include -#include +#include "os.h" #include "tast.h" #include "tscUtil.h" diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index 606bddd79b..8c492b03f8 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" typedef struct _str_node_t { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index e92ac5d46d..8a2f1347df 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -13,15 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - -#include -#include -#include +#include "os.h" #include "tcache.h" #include "tlog.h" diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index cd69f30c3f..1ac42377fb 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -46,12 +46,6 @@ * of the XORed value with informations. If not, record the first corresponding bytes. * */ -#include -#include -#include -#include -#include -#include #include "os.h" #include "lz4.h" diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index ebb6b9e0b8..8944dd4391 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -13,17 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - -#include -#include -#include - -#include - #include "os.h" #include "taos.h" #include "taosmsg.h" diff --git a/src/util/src/thash.c b/src/util/src/thash.c index 3b9cec0df5..e3c6fe26b4 100644 --- a/src/util/src/thash.c +++ b/src/util/src/thash.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include #include "os.h" #include "tmempool.h" diff --git a/src/util/src/thistogram.c b/src/util/src/thistogram.c index 6cd02cc72e..5fef9077ea 100644 --- a/src/util/src/thistogram.c +++ b/src/util/src/thistogram.c @@ -13,13 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "thistogram.h" diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index a9cc78a026..c50c38aa3c 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include +#include "os.h" #include "tlog.h" typedef struct { diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index b88cb7dc17..1a7f672e00 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -13,22 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tlog.h" #include "tutil.h" diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index c2bb836635..462da884b9 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -13,13 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - +#include "os.h" #include "tlog.h" extern int32_t taosGetTimestampSec(); diff --git a/src/util/src/tmodule.c b/src/util/src/tmodule.c index fabdc1ed4f..54669a20be 100644 --- a/src/util/src/tmodule.c +++ b/src/util/src/tmodule.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #include "os.h" #include "tmodule.h" #include "tutil.h" diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index ee16b92b65..bd49c670f6 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -13,14 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tlog.h" #include "tsched.h" diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index be352eb7ad..8de70f8a01 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -13,17 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - -#include -#include -#include -#include -#include - #include "os.h" #include "tglobalcfg.h" #include "tlog.h" diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index cf09aaee5e..9dad1e4e1e 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include #include "os.h" #include "tlog.h" #include "tsched.h" diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index 3b36b1b31c..185a63c0eb 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -13,15 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taos.h" #include "tsdb.h" diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index af75a3ce0f..8413294bcc 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -13,14 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #ifdef USE_LIBICONV From 8c55d4860a4432ed149572aa6f8c5958c24cf90a Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 16 Nov 2019 16:55:41 +0800 Subject: [PATCH 58/69] [TBASE-1164] --- src/os/darwin/src/tdarwin.c | 1 - src/os/linux/src/tlinux.c | 1 - src/rpc/src/ttcpserver.c | 3 ++- src/util/src/tglobalcfg.c | 2 +- src/util/src/tsocket.c | 4 ---- 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/os/darwin/src/tdarwin.c b/src/os/darwin/src/tdarwin.c index 71e8a56466..de37c76edd 100644 --- a/src/os/darwin/src/tdarwin.c +++ b/src/os/darwin/src/tdarwin.c @@ -204,7 +204,6 @@ int taosOpenUDServerSocket(char *ip, short port) { char name[128]; pTrace("open ud socket:%s", name); - // if (tsAllowLocalhost) ip = "0.0.0.0"; sprintf(name, "%s.%d", ip, port); bzero((char *)&serverAdd, sizeof(serverAdd)); diff --git a/src/os/linux/src/tlinux.c b/src/os/linux/src/tlinux.c index f78250cb52..6a7225b476 100644 --- a/src/os/linux/src/tlinux.c +++ b/src/os/linux/src/tlinux.c @@ -197,7 +197,6 @@ int taosOpenUDServerSocket(char *ip, short port) { char name[128]; pTrace("open ud socket:%s", name); - // if (tsAllowLocalhost) ip = "0.0.0.0"; sprintf(name, "%s.%d", ip, port); bzero((char *)&serverAdd, sizeof(serverAdd)); diff --git a/src/rpc/src/ttcpserver.c b/src/rpc/src/ttcpserver.c index b6b0e07230..abd4740854 100644 --- a/src/rpc/src/ttcpserver.c +++ b/src/rpc/src/ttcpserver.c @@ -195,8 +195,9 @@ static void taosProcessTcpData(void *param) { void *buffer = malloc(1024); int headLen = taosReadMsg(pFdObj->fd, buffer, sizeof(STaosHeader)); + if (headLen != sizeof(STaosHeader)) { - tError("%s read error, headLen:%d", pThreadObj->label, headLen); + tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno); taosCleanUpFdObj(pFdObj); tfree(buffer); continue; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 407761cff0..ab26eda348 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -77,7 +77,7 @@ float tsRatioOfQueryThreads = 0.5; char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; char tsInternalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsServerIpStr[TSDB_IPv4ADDR_LEN] = "0.0.0.0"; +char tsServerIpStr[TSDB_IPv4ADDR_LEN] = "127.0.0.1"; short tsNumOfVnodesPerCore = 8; short tsNumOfTotalVnodes = 0; short tsCheckHeaderFile = 0; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index be352eb7ad..da57cd148a 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -280,7 +280,6 @@ int taosOpenUdpSocket(char *ip, short port) { int bufSize = 8192000; pTrace("open udp socket:%s:%d", ip, port); - // if (tsAllowLocalhost) ip = "0.0.0.0"; memset((char *)&localAddr, 0, sizeof(localAddr)); localAddr.sin_family = AF_INET; @@ -345,7 +344,6 @@ int taosOpenTcpClientSocket(char *destIp, short destPort, char *clientIp) { int ret; pTrace("open tcp client socket:%s:%d", destIp, destPort); - // if (tsAllowLocalhost) destIp = "0.0.0.0"; sockFd = (int)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -441,7 +439,6 @@ int taosOpenTcpServerSocket(char *ip, short port) { int reuse; pTrace("open tcp server socket:%s:%d", ip, port); - // if (tsAllowLocalhost) ip = "0.0.0.0"; bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; @@ -483,7 +480,6 @@ int taosOpenRawSocket(char *ip) { struct sockaddr_in rawAdd; pTrace("open udp raw socket:%s", ip); - // if (tsAllowLocalhost) ip = "0.0.0.0"; fd = (int)socket(AF_INET, SOCK_RAW, IPPROTO_UDP); if (fd < 0) { From f2cbef3fe1c6cdc06217e7199f2ae26118a8c2dc Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 18 Nov 2019 11:35:34 +0800 Subject: [PATCH 59/69] [TBASE-1123] --- src/client/src/tscSystem.c | 2 + src/os/linux/inc/os.h | 3 + src/os/linux/src/tsystem.c | 126 +++++++++++++++++++++++++++- src/system/detail/src/dnodeSystem.c | 2 + 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 2842754e40..2c5ad2939a 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -83,6 +83,8 @@ void taos_init_imp() { tscTrace("Local IP address is:%s", tsLocalIp); } + taosSetCoreDump(); + #ifdef CLUSTER tscMgmtIpList.numOfIps = 2; strcpy(tscMgmtIpList.ipstr[0], tsMasterIp); diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 6ecb64eda5..64074104ca 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -214,6 +214,9 @@ bool taosSkipSocketCheck(); int64_t str2int64(char *str); +void taosSetCoreDump(); + + #define BUILDIN_CLZL(val) __builtin_clzl(val) #define BUILDIN_CLZ(val) __builtin_clz(val) #define BUILDIN_CTZL(val) __builtin_ctzl(val) diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index ccf9266f9b..0aeebbe586 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -25,6 +25,14 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include "tglobalcfg.h" #include "tlog.h" @@ -576,4 +584,120 @@ void taosKillSystem() { // SIGINT pPrint("taosd will shut down soon"); kill(tsProcId, 2); -} \ No newline at end of file +} + + +int _sysctl(struct __sysctl_args *args ); +void taosSetCoreDump() { + // 1. set ulimit -c unlimited + struct rlimit rlim; + struct rlimit rlim_new; + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { + pPrint("the old unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max); + rlim_new.rlim_cur = RLIM_INFINITY; + rlim_new.rlim_max = RLIM_INFINITY; + if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { + pPrint("set unlimited fail, error: %s", strerror(errno)); + rlim_new.rlim_cur = rlim.rlim_max; + rlim_new.rlim_max = rlim.rlim_max; + (void)setrlimit(RLIMIT_CORE, &rlim_new); + } + } + + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { + pPrint("the new unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max); + } + + // 2. set the path for saving core file + struct __sysctl_args args; + int old_usespid; + size_t old_len; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + args.newval = &new_usespid; + args.newlen = new_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + pPrint("_sysctl(kern_core_uses_pid) fail!"); + } + + pPrint("The old core_uses_pid[%d]: %d", old_len, old_usespid); + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + pPrint("_sysctl(kern_core_uses_pid) fail!"); + } + + pPrint("The new core_uses_pid[%d]: %d", old_len, old_usespid); + +#if 0 + // 3. set the path for saving core file + int status; + char coredump_dir[32] = "/var/log/taosdump"; + if (opendir(coredump_dir) == NULL) { + status = mkdir(coredump_dir, S_IRWXU | S_IRWXG | S_IRWXO); + if (status) { + pPrint("mkdir fail, error: %s\n", strerror(errno)); + } + } + + // 4. set kernel.core_pattern + struct __sysctl_args args; + char old_corefile[128]; + size_t old_len; + char new_corefile[128] = "/var/log/taosdump/core-%e-%p"; + size_t new_len = sizeof(new_corefile); + + int name[] = {CTL_KERN, KERN_CORE_PATTERN}; + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = old_corefile; + args.oldlenp = &old_len; + args.newval = new_corefile; + args.newlen = new_len; + + old_len = sizeof(old_corefile); + + if (syscall(SYS__sysctl, &args) == -1) { + pPrint("_sysctl(kern_core_pattern) fail!"); + } + + pPrint("The old kern_core_pattern: %*s\n", old_len, old_corefile); + + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = old_corefile; + args.oldlenp = &old_len; + + old_len = sizeof(old_corefile); + + if (syscall(SYS__sysctl, &args) == -1) { + pPrint("_sysctl(kern_core_pattern) fail!"); + } + + pPrint("The new kern_core_pattern: %*s\n", old_len, old_corefile); +#endif + +} + diff --git a/src/system/detail/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c index 1b5d9418b4..888504cce7 100644 --- a/src/system/detail/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -143,6 +143,8 @@ int dnodeInitSystem() { tsPrintGlobalConfig(); dPrint("Server IP address is:%s", tsInternalIp); + taosSetCoreDump(); + signal(SIGPIPE, SIG_IGN); dnodeInitModules(); From 3fe46509246503ace92d235fddce8a15a60c51b2 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 18 Nov 2019 13:52:46 +0800 Subject: [PATCH 60/69] [TBASE-1123] --- src/os/linux/src/tsystem.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 0aeebbe586..fa41e8e6d0 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -610,8 +610,8 @@ void taosSetCoreDump() { // 2. set the path for saving core file struct __sysctl_args args; - int old_usespid; - size_t old_len; + int old_usespid = 0; + size_t old_len = 0; int new_usespid = 1; size_t new_len = sizeof(new_usespid); @@ -628,11 +628,14 @@ void taosSetCoreDump() { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - pPrint("_sysctl(kern_core_uses_pid) fail!"); + pPrint("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } pPrint("The old core_uses_pid[%d]: %d", old_len, old_usespid); - + + + old_usespid = 0; + old_len = 0; memset(&args, 0, sizeof(struct __sysctl_args)); args.name = name; args.nlen = sizeof(name)/sizeof(name[0]); @@ -642,7 +645,7 @@ void taosSetCoreDump() { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - pPrint("_sysctl(kern_core_uses_pid) fail!"); + pPrint("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } pPrint("The new core_uses_pid[%d]: %d", old_len, old_usespid); @@ -678,7 +681,7 @@ void taosSetCoreDump() { old_len = sizeof(old_corefile); if (syscall(SYS__sysctl, &args) == -1) { - pPrint("_sysctl(kern_core_pattern) fail!"); + pPrint("_sysctl(kern_core_pattern) set fail: %s", strerror(errno)); } pPrint("The old kern_core_pattern: %*s\n", old_len, old_corefile); @@ -693,7 +696,7 @@ void taosSetCoreDump() { old_len = sizeof(old_corefile); if (syscall(SYS__sysctl, &args) == -1) { - pPrint("_sysctl(kern_core_pattern) fail!"); + pPrint("_sysctl(kern_core_pattern) get fail: %s", strerror(errno)); } pPrint("The new kern_core_pattern: %*s\n", old_len, old_corefile); From d458545cce87bd97178ae34d5cb96f65ab831c7d Mon Sep 17 00:00:00 2001 From: localvar Date: Mon, 18 Nov 2019 06:29:11 +0000 Subject: [PATCH 61/69] fix TBASE-1176 --- src/util/src/ttimer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 9dad1e4e1e..a1a356b13b 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -549,7 +549,9 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab void taosTmrCleanUp(void* handle) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; - assert(ctrl != NULL && ctrl->label[0] != 0); + if (ctrl == NULL || ctrl->label[0] == 0) { + return; + } tmrTrace("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; From 8527587b4998ad368e43b4c1ea38b24a1e385b70 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 18 Nov 2019 17:44:36 +0800 Subject: [PATCH 62/69] [TBASE-1186] --- src/inc/taoserror.h | 1 + src/inc/tsdb.h | 34 ++++++++--- src/inc/tstatus.h | 9 +-- src/rpc/src/trpc.c | 4 ++ src/rpc/src/tstring.c | 1 + src/system/detail/inc/mgmtBalance.h | 9 --- src/system/detail/inc/vnode.h | 2 +- src/system/detail/src/dnodeMgmt.c | 24 ++++++-- src/system/detail/src/mgmtDb.c | 11 ++-- src/system/detail/src/mgmtDnode.c | 17 +++--- src/system/detail/src/vnodeCache.c | 2 +- src/system/detail/src/vnodeFile.c | 2 +- src/system/detail/src/vnodeShell.c | 20 +++++-- src/system/detail/src/vnodeStore.c | 78 +++++++++++++++++-------- src/system/detail/src/vnodeStream.c | 2 +- src/system/lite/src/mgmtDnode.spec.c | 2 +- src/system/lite/src/mgmtDnodeInt.spec.c | 4 +- src/system/lite/src/vnodePeer.spec.c | 5 +- src/util/src/tstatus.c | 52 +++++++++++++++-- 19 files changed, 199 insertions(+), 80 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index fbb7bf86da..9afc3de7a8 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -134,6 +134,7 @@ extern "C" { #define TSDB_CODE_INVALID_SUBMIT_MSG 113 #define TSDB_CODE_NOT_ACTIVE_TABLE 114 #define TSDB_CODE_INVALID_TABLE_ID 115 +#define TSDB_CODE_INVALID_VNODE_STATUS 116 #ifdef __cplusplus } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 4c43171283..771046d4d1 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -44,22 +44,40 @@ extern "C" { #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" -enum _status { - TSDB_STATUS_OFFLINE, - TSDB_STATUS_CREATING, - TSDB_STATUS_UNSYNCED, - TSDB_STATUS_SLAVE, - TSDB_STATUS_MASTER, - TSDB_STATUS_READY, + enum _vnode_status { + TSDB_VNODE_STATUS_OFFLINE, + TSDB_VNODE_STATUS_CREATING, + TSDB_VNODE_STATUS_UNSYNCED, + TSDB_VNODE_STATUS_SLAVE, + TSDB_VNODE_STATUS_MASTER, + TSDB_VNODE_STATUS_CLOSING, + TSDB_VNODE_STATUS_DELETING, }; -enum _syncstatus { +enum _vnode_sync_status { STDB_SSTATUS_INIT, TSDB_SSTATUS_SYNCING, TSDB_SSTATUS_SYNC_CACHE, TSDB_SSTATUS_SYNC_FILE, }; +enum _dnode_status { + TSDB_DNODE_STATUS_OFFLINE, + TSDB_DNODE_STATUS_READY +}; + +enum _dnode_balance_status { + LB_DNODE_STATE_BALANCED, + LB_DNODE_STATE_BALANCING, + LB_DNODE_STATE_OFFLINE_REMOVING, + LB_DNODE_STATE_SHELL_REMOVING +}; + +enum _vgroup_status { + LB_VGROUP_STATE_READY, + LB_VGROUP_STATE_UPDATE +}; + #define TSDB_DATA_TYPE_BOOL 1 // 1 bytes #define TSDB_DATA_TYPE_TINYINT 2 // 1 byte #define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes diff --git a/src/inc/tstatus.h b/src/inc/tstatus.h index 34bc7c6f27..34319e40b3 100644 --- a/src/inc/tstatus.h +++ b/src/inc/tstatus.h @@ -20,10 +20,11 @@ extern "C" { #endif -extern char *sdbDnodeStatusStr[]; -extern char *sdbDnodeBalanceStateStr[]; -extern char *sdbVnodeDropStateStr[]; -extern char *sdbVnodeSyncStatusStr[]; +const char* taosGetVnodeStatusStr(int vnodeStatus); +const char* taosGetDnodeStatusStr(int dnodeStatus); +const char* taosGetDnodeBalanceStateStr(int dnodeBalanceStatus); +const char* taosGetVnodeSyncStatusStr(int vnodeSyncStatus); +const char* taosGetVnodeDropStatusStr(int dropping); #ifdef __cplusplus } diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index de21598dd9..5403caac9d 100644 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -354,6 +354,8 @@ int taosOpenRpcChannWithQ(void *handle, int cid, int sessions, void *qhandle) { STaosRpc * pServer = (STaosRpc *)handle; SRpcChann *pChann; + tTrace("cid:%d, handle:%p open rpc chann", cid, handle); + if (pServer == NULL) return -1; if (cid >= pServer->numOfChanns || cid < 0) { tError("%s: cid:%d, chann is out of range, max:%d", pServer->label, cid, pServer->numOfChanns); @@ -402,6 +404,8 @@ void taosCloseRpcChann(void *handle, int cid) { STaosRpc * pServer = (STaosRpc *)handle; SRpcChann *pChann; + tTrace("cid:%d, handle:%p close rpc chann", cid, handle); + if (pServer == NULL) return; if (cid >= pServer->numOfChanns || cid < 0) { tError("%s cid:%d, chann is out of range, max:%d", pServer->label, cid, pServer->numOfChanns); diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index 9978ba9f5d..d11f75885f 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -240,4 +240,5 @@ char *tsError[] = {"success", "invalid submit message", "not active table(not created yet or deleted already)", //114 "invalid table id", + "invalid vnode status", //116 }; diff --git a/src/system/detail/inc/mgmtBalance.h b/src/system/detail/inc/mgmtBalance.h index 7d478416ac..67bfd55db2 100644 --- a/src/system/detail/inc/mgmtBalance.h +++ b/src/system/detail/inc/mgmtBalance.h @@ -28,15 +28,6 @@ extern "C" { #include "tstatus.h" #include "ttime.h" -enum { - LB_DNODE_STATE_BALANCED, - LB_DNODE_STATE_BALANCING, - LB_DNODE_STATE_OFFLINE_REMOVING, - LB_DNODE_STATE_SHELL_REMOVING -}; - -enum { LB_VGROUP_STATE_READY, LB_VGROUP_STATE_UPDATE }; - void mgmtCreateDnodeOrderList(); void mgmtReleaseDnodeOrderList(); diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 35b5c010ab..c944c06d31 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -92,7 +92,7 @@ typedef struct { SVPeerDesc vpeers[TSDB_VNODES_SUPPORT]; SVnodePeer * peerInfo[TSDB_VNODES_SUPPORT]; char selfIndex; - char status; + char vnodeStatus; char accessState; // Vnode access state, Readable/Writable char syncStatus; char commitInProcess; diff --git a/src/system/detail/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c index ac651f05f2..a1668b89cf 100644 --- a/src/system/detail/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -26,6 +26,7 @@ #include "vnodeMgmt.h" #include "vnodeSystem.h" #include "vnodeUtil.h" +#include "tstatus.h" SMgmtObj mgmtObj; extern uint64_t tsCreatedTime; @@ -330,7 +331,7 @@ int vnodeProcessVPeerCfg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { return -1; } - if (vnodeList[vnode].status == TSDB_STATUS_CREATING) { + if (vnodeList[vnode].vnodeStatus == TSDB_VNODE_STATUS_CREATING) { dTrace("vid:%d, vnode is still under creating", vnode); return 0; } @@ -359,13 +360,23 @@ int vnodeProcessVPeerCfg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { } if (vnodeList[vnode].cfg.maxSessions == 0) { + dTrace("vid:%d, vnode is empty", vnode); if (pCfg->maxSessions > 0) { - return vnodeCreateVnode(vnode, pCfg, pMsg->vpeerDesc); + if (vnodeList[vnode].vnodeStatus == TSDB_VNODE_STATUS_OFFLINE) { + dTrace("vid:%d, status:%s, start to create vnode", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + return vnodeCreateVnode(vnode, pCfg, pMsg->vpeerDesc); + } else { + dTrace("vid:%d, status:%s, cannot preform create vnode operation", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + return TSDB_CODE_INVALID_VNODE_STATUS; + } } } else { + dTrace("vid:%d, vnode is not empty", vnode); if (pCfg->maxSessions > 0) { + dTrace("vid:%d, status:%s, start to update vnode", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + /* if (pCfg->maxSessions != vnodeList[vnode].cfg.maxSessions) { - vnodeCleanUpOneVnode(vnode); + vnodeCleanUpOneVnode(vnode); } vnodeConfigVPeers(vnode, pCfg->replications, pMsg->vpeerDesc); @@ -376,7 +387,10 @@ int vnodeProcessVPeerCfg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { vnodeList[vnode].cfg.maxSessions = pCfg->maxSessions; vnodeOpenVnode(vnode); } + */ + return 0; } else { + dTrace("vid:%d, status:%s, start to delete vnode", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); vnodeRemoveVnode(vnode); } } @@ -434,11 +448,11 @@ int vnodeProcessFreeVnodeRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj) { pFree->vnode = htons(pFree->vnode); if (pFree->vnode < 0 || pFree->vnode >= TSDB_MAX_VNODES) { - dWarn("vid:%d out of range", pFree->vnode); + dWarn("vid:%d, out of range", pFree->vnode); return -1; } - dTrace("vid:%d receive free vnode message", pFree->vnode); + dTrace("vid:%d, receive free vnode message", pFree->vnode); int32_t code = vnodeRemoveVnode(pFree->vnode); assert(code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS); diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index e87266aa4b..16a028d2de 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -141,11 +141,10 @@ int mgmtCheckDbParams(SCreateDbMsg *pCreate) { if (pCreate->cacheNumOfBlocks.fraction < 0) pCreate->cacheNumOfBlocks.fraction = tsAverageCacheBlocks; // //-1 for balance -#ifdef CLUSTER - if (pCreate->replications > TSDB_VNODES_SUPPORT - 1) pCreate->replications = TSDB_VNODES_SUPPORT - 1; -#else - pCreate->replications = 1; -#endif + if (pCreate->replications <= 0 || pCreate->replications > TSDB_REPLICA_MAX_NUM) { + mTrace("invalid db option replications: %d", pCreate->replications); + return TSDB_CODE_INVALID_OPTION; + } if (pCreate->commitLog < 0 || pCreate->commitLog > 1) { mTrace("invalid db option commitLog: %d", pCreate->commitLog); @@ -316,7 +315,7 @@ bool mgmtCheckDropDbFinished(SDbObj *pDb) { SDnodeObj *pDnode = mgmtGetDnode(pVnodeGid->ip); if (pDnode == NULL) continue; - if (pDnode->status == TSDB_STATUS_OFFLINE) continue; + if (pDnode->status == TSDB_DNODE_STATUS_OFFLINE) continue; SVnodeLoad *pVload = &pDnode->vload[pVnodeGid->vnode]; if (pVload->dropStatus == TSDB_VN_STATUS_DROPPING) { diff --git a/src/system/detail/src/mgmtDnode.c b/src/system/detail/src/mgmtDnode.c index 3a4936041c..d8cfbbe917 100644 --- a/src/system/detail/src/mgmtDnode.c +++ b/src/system/detail/src/mgmtDnode.c @@ -21,6 +21,7 @@ #include "mgmt.h" #include "tschemautil.h" #include "tstatus.h" +#include "tstatus.h" bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int moduleType); int mgmtGetDnodesNum(); @@ -43,9 +44,9 @@ void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) { pDnode->openVnodes = 0; #ifdef CLUSTER - pDnode->status = TSDB_STATUS_OFFLINE; + pDnode->status = TSDB_DNODE_STATUS_OFFLINE; #else - pDnode->status = TSDB_STATUS_READY; + pDnode->status = TSDB_DNODE_STATUS_READY; #endif } @@ -57,9 +58,9 @@ void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode) { if (pVload->vgId != 0) { mTrace("dnode:%s, calc free vnodes, exist vnode:%d, vgroup:%d, state:%d %s, dropstate:%d %s, syncstatus:%d %s", taosIpStr(pDnode->privateIp), i, pVload->vgId, - pVload->status, sdbDnodeStatusStr[pVload->status], - pVload->dropStatus, sdbVnodeDropStateStr[pVload->dropStatus], - pVload->syncStatus, sdbVnodeSyncStatusStr[pVload->syncStatus]); + pVload->status, taosGetDnodeStatusStr(pVload->status), + pVload->dropStatus, taosGetVnodeDropStatusStr(pVload->dropStatus), + pVload->syncStatus, taosGetVnodeSyncStatusStr(pVload->syncStatus)); totalVnodes++; } } @@ -196,11 +197,11 @@ int mgmtRetrieveDnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbDnodeStatusStr[pDnode->status]); + strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbDnodeBalanceStateStr[pDnode->lbState]); + strcpy(pWrite, taosGetDnodeBalanceStateStr(pDnode->lbState)); cols++; tinet_ntoa(ipstr, pDnode->publicIp); @@ -292,7 +293,7 @@ int mgmtRetrieveModules(SShowObj *pShow, char *data, int rows, SConnObj *pConn) cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbDnodeStatusStr[pDnode->status]); + strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); cols++; numOfRows++; diff --git a/src/system/detail/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c index 94ac006060..0b464ca32b 100644 --- a/src/system/detail/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -296,7 +296,7 @@ pthread_t vnodeCreateCommitThread(SVnodeObj *pVnode) { taosTmrStopA(&pVnode->commitTimer); - if (pVnode->status == TSDB_STATUS_UNSYNCED) { + if (pVnode->vnodeStatus == TSDB_VNODE_STATUS_UNSYNCED) { taosTmrReset(vnodeProcessCommitTimer, pVnode->cfg.commitTime * 1000, pVnode, vnodeTmrCtrl, &pVnode->commitTimer); dTrace("vid:%d, it is in unsyc state, commit later", pVnode->vnode); return pVnode->commitThread; diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index a2ddca7a28..c8c7d2debe 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -1290,7 +1290,7 @@ int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pCompBlock, SData *data[] pCompBlock->len += wlen; } - dTrace("vid: %d vnode compStorage size is: %ld", pObj->vnode, pVnode->vnodeStatistic.compStorage); + dTrace("vid:%d, vnode compStorage size is: %ld", pObj->vnode, pVnode->vnodeStatistic.compStorage); pCompBlock->algorithm = pCfg->compression; pCompBlock->numOfPoints = points; diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 47b7b470bb..f6c70811ca 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -27,6 +27,7 @@ #include "vnode.h" #include "vnodeRead.h" #include "vnodeUtil.h" +#include "vnodeStore.h" #pragma GCC diagnostic ignored "-Wint-conversion" extern int tsMaxQueues; @@ -154,6 +155,11 @@ int vnodeInitShell() { } int vnodeOpenShellVnode(int vnode) { + if (shellList[vnode] != NULL) { + dError("vid:%d, shell is already opened", vnode); + return -1; + } + const int32_t MIN_NUM_OF_SESSIONS = 300; SVnodeCfg *pCfg = &vnodeList[vnode].cfg; @@ -162,23 +168,29 @@ int vnodeOpenShellVnode(int vnode) { size_t size = sessions * sizeof(SShellObj); shellList[vnode] = (SShellObj *)calloc(1, size); if (shellList[vnode] == NULL) { - dError("vid:%d failed to allocate shellObj, size:%d", vnode, size); + dError("vid:%d, sessions:%d, failed to allocate shellObj, size:%d", vnode, pCfg->maxSessions, size); return -1; } if(taosOpenRpcChannWithQ(pShellServer, vnode, sessions, rpcQhandle[(vnode+1)%tsMaxQueues]) != TSDB_CODE_SUCCESS) { + dError("vid:%d, sessions:%d, failed to open shell", vnode, pCfg->maxSessions); return -1; } + dTrace("vid:%d, sessions:%d, shell is opened", vnode, pCfg->maxSessions); return TSDB_CODE_SUCCESS; } static void vnodeDelayedFreeResource(void *param, void *tmrId) { int32_t vnode = *(int32_t*) param; - taosCloseRpcChann(pShellServer, vnode); // close connection - tfree (shellList[vnode]); //free SShellObj + dTrace("vid:%d, start to free resources", vnode); + taosCloseRpcChann(pShellServer, vnode); // close connection + tfree(shellList[vnode]); //free SShellObj tfree(param); + + memset(vnodeList + vnode, 0, sizeof(SVnodeObj)); + vnodeCalcOpenVnodes(); } void vnodeCloseShellVnode(int vnode) { @@ -197,7 +209,7 @@ void vnodeCloseShellVnode(int vnode) { * 2. Free connection may cause *(SRpcConn*)pObj->thandle to be invalid to access. */ dTrace("vid:%d, free resources in 500ms", vnode); - taosTmrStart(vnodeDelayedFreeResource, 500, v, vnodeTmrCtrl); + taosTmrStart(vnodeDelayedFreeResource, 5000, v, vnodeTmrCtrl); } void vnodeCleanUpShell() { diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index d299d7fbf8..8c958c21d7 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -22,6 +22,7 @@ #include "vnode.h" #include "vnodeStore.h" #include "vnodeUtil.h" +#include "tstatus.h" #pragma GCC diagnostic push #pragma GCC diagnostic warning "-Woverflow" @@ -30,12 +31,14 @@ int tsMaxVnode = -1; int tsOpenVnodes = 0; SVnodeObj *vnodeList = NULL; -int vnodeInitStoreVnode(int vnode) { +static int vnodeInitStoreVnode(int vnode) { SVnodeObj *pVnode = vnodeList + vnode; pVnode->vnode = vnode; vnodeOpenMetersVnode(vnode); - if (pVnode->cfg.maxSessions == 0) return 0; + if (pVnode->cfg.maxSessions <= 0) { + return TSDB_CODE_SUCCESS; + } pVnode->firstKey = taosGetTimestamp(pVnode->cfg.precision); @@ -45,9 +48,10 @@ int vnodeInitStoreVnode(int vnode) { return -1; } - if (vnodeInitFile(vnode) < 0) return -1; - - // vnodeOpenMeterMgmtStoreVnode(vnode); + if (vnodeInitFile(vnode) < 0) { + dError("vid:%d, files init failed.", pVnode->vnode); + return -1; + } if (vnodeInitCommit(vnode) < 0) { dError("vid:%d, commit init failed.", pVnode->vnode); @@ -70,10 +74,17 @@ int vnodeOpenVnode(int vnode) { pVnode->accessState = TSDB_VN_ALL_ACCCESS; // vnode is empty - if (pVnode->cfg.maxSessions == 0) return 0; + if (pVnode->cfg.maxSessions <= 0) { + return TSDB_CODE_SUCCESS; + } + if (!(pVnode->vnodeStatus == TSDB_VNODE_STATUS_OFFLINE || pVnode->vnodeStatus == TSDB_VNODE_STATUS_CREATING)) { + dError("vid:%d, status:%s, cannot enter open operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + return TSDB_CODE_INVALID_VNODE_STATUS; + } + + dTrace("vid:%d, status:%s, start to open", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); pthread_mutex_lock(&dmutex); - // vnodeOpenMeterMgmtVnode(vnode); // not enough memory, abort if ((code = vnodeOpenShellVnode(vnode)) != TSDB_CODE_SUCCESS) { @@ -93,14 +104,13 @@ int vnodeOpenVnode(int vnode) { vnodeOpenStreams(pVnode, NULL); #endif - dTrace("vid:%d, vnode is opened, openVnodes:%d", vnode, tsOpenVnodes); + dTrace("vid:%d, vnode is opened, openVnodes:%d, status:%s", vnode, tsOpenVnodes, taosGetVnodeStatusStr(pVnode->vnodeStatus)); - return 0; + return TSDB_CODE_SUCCESS; } static int32_t vnodeMarkAllMetersDropped(SVnodeObj* pVnode) { if (pVnode->meterList == NULL) { - assert(pVnode->cfg.maxSessions == 0); return TSDB_CODE_SUCCESS; } @@ -119,7 +129,7 @@ static int32_t vnodeMarkAllMetersDropped(SVnodeObj* pVnode) { return ready? TSDB_CODE_SUCCESS:TSDB_CODE_ACTION_IN_PROGRESS; } -int vnodeCloseVnode(int vnode) { +static int vnodeCloseVnode(int vnode) { if (vnodeList == NULL) return TSDB_CODE_SUCCESS; SVnodeObj* pVnode = &vnodeList[vnode]; @@ -130,12 +140,23 @@ int vnodeCloseVnode(int vnode) { return TSDB_CODE_SUCCESS; } + if (pVnode->vnodeStatus == TSDB_VNODE_STATUS_DELETING) { + dTrace("vid:%d, status:%s, another performed delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + return TSDB_CODE_SUCCESS; + } else { + dTrace("vid:%d, status:%s, enter close operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + pVnode->vnodeStatus = TSDB_VNODE_STATUS_CLOSING; + } + // set the meter is dropped flag if (vnodeMarkAllMetersDropped(pVnode) != TSDB_CODE_SUCCESS) { pthread_mutex_unlock(&dmutex); return TSDB_CODE_ACTION_IN_PROGRESS; } + dTrace("vid:%d, status:%s, enter delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + pVnode->vnodeStatus = TSDB_VNODE_STATUS_DELETING; + vnodeCloseStream(vnodeList + vnode); vnodeCancelCommit(vnodeList + vnode); vnodeClosePeerVnode(vnode); @@ -149,9 +170,6 @@ int vnodeCloseVnode(int vnode) { if (tsMaxVnode == vnode) tsMaxVnode = vnode - 1; tfree(vnodeList[vnode].meterIndex); - memset(vnodeList + vnode, 0, sizeof(SVnodeObj)); - - vnodeCalcOpenVnodes(); pthread_mutex_unlock(&dmutex); return TSDB_CODE_SUCCESS; @@ -160,7 +178,12 @@ int vnodeCloseVnode(int vnode) { int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { char fileName[128]; - vnodeList[vnode].status = TSDB_STATUS_CREATING; + if (vnodeList[vnode].vnodeStatus != TSDB_VNODE_STATUS_OFFLINE) { + dError("vid:%d, status:%s, cannot enter create operation", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + return TSDB_CODE_INVALID_VNODE_STATUS; + } + + vnodeList[vnode].vnodeStatus = TSDB_VNODE_STATUS_CREATING; sprintf(fileName, "%s/vnode%d", tsDirectory, vnode); mkdir(fileName, 0755); @@ -177,14 +200,14 @@ int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { return TSDB_CODE_VG_INIT_FAILED; } - if (vnodeInitStoreVnode(vnode) != 0) { + if (vnodeInitStoreVnode(vnode) < 0) { return TSDB_CODE_VG_COMMITLOG_INIT_FAILED; } return vnodeOpenVnode(vnode); } -void vnodeRemoveDataFiles(int vnode) { +static void vnodeRemoveDataFiles(int vnode) { char vnodeDir[TSDB_FILENAME_LEN]; char dfilePath[TSDB_FILENAME_LEN]; char linkFile[TSDB_FILENAME_LEN]; @@ -234,12 +257,21 @@ int vnodeRemoveVnode(int vnode) { if (vnodeList == NULL) return TSDB_CODE_SUCCESS; if (vnodeList[vnode].cfg.maxSessions > 0) { - int32_t ret = vnodeCloseVnode(vnode); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + SVnodeObj* pVnode = &vnodeList[vnode]; + if (pVnode->vnodeStatus == TSDB_VNODE_STATUS_CREATING + || pVnode->vnodeStatus == TSDB_VNODE_STATUS_OFFLINE + || pVnode->vnodeStatus == TSDB_VNODE_STATUS_DELETING) { + dError("vid:%d, status:%s, cannot enter close/delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + return TSDB_CODE_ACTION_IN_PROGRESS; + } else { + int32_t ret = vnodeCloseVnode(vnode); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + vnodeRemoveDataFiles(vnode); } - vnodeRemoveDataFiles(vnode); } else { dTrace("vid:%d, max sessions:%d, this vnode already dropped!!!", vnode, vnodeList[vnode].cfg.maxSessions); vnodeList[vnode].cfg.maxSessions = 0; //reset value @@ -293,7 +325,7 @@ void vnodeCleanUpOneVnode(int vnode) { again = 1; if (vnodeList[vnode].pCachePool) { - vnodeList[vnode].status = TSDB_STATUS_OFFLINE; + vnodeList[vnode].vnodeStatus = TSDB_VNODE_STATUS_OFFLINE; vnodeClosePeerVnode(vnode); } @@ -322,7 +354,7 @@ void vnodeCleanUpVnodes() { for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) { if (vnodeList[vnode].pCachePool) { - vnodeList[vnode].status = TSDB_STATUS_OFFLINE; + vnodeList[vnode].vnodeStatus = TSDB_VNODE_STATUS_OFFLINE; vnodeClosePeerVnode(vnode); } } diff --git a/src/system/detail/src/vnodeStream.c b/src/system/detail/src/vnodeStream.c index 0667ee77bd..562b7eb73c 100644 --- a/src/system/detail/src/vnodeStream.c +++ b/src/system/detail/src/vnodeStream.c @@ -171,7 +171,7 @@ void vnodeCloseStream(SVnodeObj *pVnode) { void vnodeUpdateStreamRole(SVnodeObj *pVnode) { /* SMeterObj *pObj; */ - int newRole = (pVnode->status == TSDB_STATUS_MASTER) ? 1 : 0; + int newRole = (pVnode->vnodeStatus == TSDB_VNODE_STATUS_MASTER) ? 1 : 0; if (newRole != pVnode->streamRole) { dTrace("vid:%d, stream role is changed to:%d", pVnode->vnode, newRole); pVnode->streamRole = newRole; diff --git a/src/system/lite/src/mgmtDnode.spec.c b/src/system/lite/src/mgmtDnode.spec.c index c34ac58c00..3c7c7f06ba 100644 --- a/src/system/lite/src/mgmtDnode.spec.c +++ b/src/system/lite/src/mgmtDnode.spec.c @@ -30,7 +30,7 @@ int mgmtInitDnodes() { dnodeObj.createdTime = (int64_t)tsRebootTime * 1000; dnodeObj.lastReboot = tsRebootTime; dnodeObj.numOfCores = (uint16_t)tsNumOfCores; - dnodeObj.status = TSDB_STATUS_READY; + dnodeObj.status = TSDB_DNODE_STATUS_READY; dnodeObj.alternativeRole = TSDB_DNODE_ROLE_ANY; dnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes; dnodeObj.thandle = (void*)(1); //hack way diff --git a/src/system/lite/src/mgmtDnodeInt.spec.c b/src/system/lite/src/mgmtDnodeInt.spec.c index acde36e7b8..10628d0fb0 100644 --- a/src/system/lite/src/mgmtDnodeInt.spec.c +++ b/src/system/lite/src/mgmtDnodeInt.spec.c @@ -82,7 +82,7 @@ void mgmtCleanUpDnodeInt() {} void mgmtProcessDnodeStatus(void *handle, void *tmrId) { SDnodeObj *pObj = &dnodeObj; pObj->openVnodes = tsOpenVnodes; - pObj->status = TSDB_STATUS_READY; + pObj->status = TSDB_DNODE_STATUS_READY; float memoryUsedMB = 0; taosGetSysMemory(&memoryUsedMB); @@ -97,7 +97,7 @@ void mgmtProcessDnodeStatus(void *handle, void *tmrId) { if (vnodeList[vnode].cfg.maxSessions <= 0) { pVload->dropStatus = TSDB_VN_STATUS_READY; pVload->status = TSDB_VN_STATUS_READY; - mPrint("vid:%d, drop finished", pObj->privateIp, vnode); + mPrint("dnode:%s, vid:%d, drop finished", taosIpStr(pObj->privateIp), vnode); taosTmrStart(mgmtMonitorDbDrop, 10000, NULL, mgmtTmr); } } diff --git a/src/system/lite/src/vnodePeer.spec.c b/src/system/lite/src/vnodePeer.spec.c index 1ceb8465c3..d33e4ead6e 100644 --- a/src/system/lite/src/vnodePeer.spec.c +++ b/src/system/lite/src/vnodePeer.spec.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vnode.h" +#include "tstatus.h" int vnodeInitPeer(int numOfThreads) { return 0; } @@ -30,8 +31,8 @@ void vnodeBroadcastStatusToUnsyncedPeer(SVnodeObj *pVnode) {} int vnodeOpenPeerVnode(int vnode) { SVnodeObj *pVnode = vnodeList + vnode; - pVnode->status = (pVnode->cfg.replications > 1) ? TSDB_STATUS_UNSYNCED : TSDB_STATUS_MASTER; - dTrace("vid:%d, vnode status:%d numOfPeers:%d", vnode, pVnode->status, pVnode->cfg.replications-1); + pVnode->vnodeStatus = (pVnode->cfg.replications > 1) ? TSDB_VNODE_STATUS_UNSYNCED : TSDB_VNODE_STATUS_MASTER; + dTrace("vid:%d, status:%s numOfPeers:%d", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus), pVnode->cfg.replications - 1); vnodeUpdateStreamRole(pVnode); return 0; } diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index 570ba547eb..1e21285960 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -13,10 +13,54 @@ * along with this program. If not, see . */ -char* sdbDnodeStatusStr[] = {"offline", "creating", "unsynced", "slave", "master", "ready"}; +#include "taosmsg.h" +#include "tsdb.h" -char* sdbDnodeBalanceStateStr[] = {"balanced", "balancing", "offline removing", "shell removing"}; +const char* taosGetVnodeStatusStr(int vnodeStatus) { + switch (vnodeStatus) { + case TSDB_VNODE_STATUS_OFFLINE:return "offline"; + case TSDB_VNODE_STATUS_CREATING: return "creating"; + case TSDB_VNODE_STATUS_UNSYNCED: return "unsynced"; + case TSDB_VNODE_STATUS_SLAVE: return "slave"; + case TSDB_VNODE_STATUS_MASTER: return "master"; + case TSDB_VNODE_STATUS_CLOSING: return "closing"; + case TSDB_VNODE_STATUS_DELETING: return "deleting"; + default: return "undefined"; + } +} -char* sdbVnodeSyncStatusStr[] = {"init", "syncing", "sync_cache", "sync_file"}; +const char* taosGetDnodeStatusStr(int dnodeStatus) { + switch (dnodeStatus) { + case TSDB_DNODE_STATUS_OFFLINE: return "offline"; + case TSDB_DNODE_STATUS_READY: return "ready"; + default: return "undefined"; + } +} -char* sdbVnodeDropStateStr[] = {"ready", "dropping"}; +const char* taosGetDnodeBalanceStateStr(int dnodeBalanceStatus) { + switch (dnodeBalanceStatus) { + case LB_DNODE_STATE_BALANCED: return "balanced"; + case LB_DNODE_STATE_BALANCING: return "balancing"; + case LB_DNODE_STATE_OFFLINE_REMOVING: return "offline removing"; + case LB_DNODE_STATE_SHELL_REMOVING: return "removing"; + default: return "undefined"; + } +} + +const char* taosGetVnodeSyncStatusStr(int vnodeSyncStatus) { + switch (vnodeSyncStatus) { + case STDB_SSTATUS_INIT: return "init"; + case TSDB_SSTATUS_SYNCING: return "syncing"; + case TSDB_SSTATUS_SYNC_CACHE: return "sync_cache"; + case TSDB_SSTATUS_SYNC_FILE: return "sync_file"; + default: return "undefined"; + } +} + +const char* taosGetVnodeDropStatusStr(int dropping) { + switch (dropping) { + case 0: return "ready"; + case 1: return "dropping"; + default: return "undefined"; + } +} \ No newline at end of file From d731fc0dbf3dd487a7dec16e3a733dbb687e9629 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 18 Nov 2019 21:36:53 +0800 Subject: [PATCH 63/69] fix #742 --- src/system/detail/src/vnodeImport.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index 260d531200..0d3cd2b338 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -479,7 +479,7 @@ int vnodeImportToFile(SImportInfo *pImport) { slot = (slot + 1 + pInfo->maxBlocks) % pInfo->maxBlocks; } - // last slot, the uncommitted slots shall be shifted + // last slot, the uncommitted slots shall be shifted, a cache block may have empty rows SCacheBlock *pCacheBlock = pInfo->cacheBlocks[slot]; int points = pCacheBlock->numOfPoints - pInfo->commitPoint; if (points > 0) { @@ -568,7 +568,7 @@ int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { } } - // copy the overwritten data into buffer + // copy the overwritten data into buffer, merge cache blocks tpoints = rows; pos = pImport->pos; slot = pImport->slot; @@ -603,6 +603,19 @@ int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { pos = 0; tpoints -= points; + if (tpoints == 0) { + // free the rest of cache blocks, since cache blocks are merged + int currentSlot = slot; + while (slot != pInfo->currentSlot) { + slot = (slot + 1) % pInfo->maxBlocks; + pCacheBlock = pInfo->cacheBlocks[slot]; + vnodeFreeCacheBlock(pCacheBlock); + } + + pInfo->currentSlot = currentSlot; + slot = currentSlot; // make sure to exit from the while loop + } + if (slot == pInfo->currentSlot) break; slot = (slot + 1) % pInfo->maxBlocks; } From 54e7aef8e6e4aaf11312cbe27f900168c1a24e64 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 18 Nov 2019 21:50:05 +0800 Subject: [PATCH 64/69] [TBASE-1186] --- src/system/detail/src/vnodeShell.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index f6c70811ca..5982b7b1b5 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -90,9 +90,9 @@ void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { // if ( vnodeList[vnode].status != TSDB_STATUS_MASTER && pMsg->msgType != TSDB_MSG_TYPE_RETRIEVE ) { #ifdef CLUSTER - if (vnodeList[vnode].status != TSDB_STATUS_MASTER) { + if (vnodeList[vnode].vnodeStatus != TSDB_VNODE_STATUS_MASTER) { taosSendSimpleRsp(thandle, pMsg->msgType + 1, TSDB_CODE_NOT_READY); - dTrace("vid:%d sid:%d, shell msg is ignored since in state:%d", vnode, sid, vnodeList[vnode].status); + dTrace("vid:%d sid:%d, shell msg is ignored since in state:%d", vnode, sid, vnodeList[vnode].vnodeStatus); } else { #endif dTrace("vid:%d sid:%d, msg:%s is received pConn:%p", vnode, sid, taosMsg[pMsg->msgType], thandle); @@ -209,7 +209,7 @@ void vnodeCloseShellVnode(int vnode) { * 2. Free connection may cause *(SRpcConn*)pObj->thandle to be invalid to access. */ dTrace("vid:%d, free resources in 500ms", vnode); - taosTmrStart(vnodeDelayedFreeResource, 5000, v, vnodeTmrCtrl); + taosTmrStart(vnodeDelayedFreeResource, 500, v, vnodeTmrCtrl); } void vnodeCleanUpShell() { From fa3f34dcdaa527d11a5570af6d78623d85fc12e5 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 18 Nov 2019 21:53:20 +0800 Subject: [PATCH 65/69] Update Super Table-ch.md --- documentation/webdocs/markdowndocs/Super Table-ch.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/webdocs/markdowndocs/Super Table-ch.md b/documentation/webdocs/markdowndocs/Super Table-ch.md index 9267e00a70..c5b207b564 100644 --- a/documentation/webdocs/markdowndocs/Super Table-ch.md +++ b/documentation/webdocs/markdowndocs/Super Table-ch.md @@ -169,7 +169,7 @@ SELECT function,… 以温度传感器采集时序数据作为例，示范STable的使用。 在这个例子中，对每个温度计都会建立一张表，表名为温度计的ID，温度计读数的时刻记为ts，采集的值记为degree。通过tags给每个采集器打上不同的标签，其中记录温度计的地区和类型，以方便我们后面的查询。所有温度计的采集量都一样，因此我们用STable来定义表结构。 -###定义STable表结构并使用它创建子表 +###1:定义STable表结构并使用它创建子表 创建STable语句如下： @@ -189,7 +189,7 @@ CREATE TABLE therm4 USING thermometer TAGS ('shanghai', 3); 其中therm1，therm2，therm3，therm4是超级表thermometer四个具体的子表，也即普通的Table。以therm1为例，它表示采集器therm1的数据，表结构完全由thermometer定义，标签location=”beijing”, type=1表示therm1的地区是北京，类型是第1类的温度计。 -###写入数据 +###2:写入数据 注意，写入数据时不能直接对STable操作，而是要对每张子表进行操作。我们分别向四张表therm1，therm2， therm3， therm4写入一条数据，写入语句如下： @@ -200,7 +200,7 @@ INSERT INTO therm3 VALUES ('2018-01-01 00:00:00.000', 24); INSERT INTO therm4 VALUES ('2018-01-01 00:00:00.000', 23); ``` -### 按标签聚合查询 +###3:按标签聚合查询 查询位于北京(beijing)和天津(tianjing)两个地区的温度传感器采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)、最低温度min(degree)，并将结果按所处地域(location)和传感器类型(type)进行聚合。 @@ -211,7 +211,7 @@ WHERE location='beijing' or location='tianjin' GROUP BY location, type ``` -### 按时间周期聚合查询 +###4:按时间周期聚合查询 查询仅位于北京以外地区的温度传感器最近24小时(24h)采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)和最低温度min(degree)，将采集结果按照10分钟为周期进行聚合，并将结果按所处地域(location)和传感器类型(type)再次进行聚合。 From 78158e32a5160c924b5426db841ce4ca16bc5d5d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 19 Nov 2019 10:31:53 +0800 Subject: [PATCH 66/69] fix #742 --- src/system/detail/src/vnodeCache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/system/detail/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c index 94ac006060..d4a654de95 100644 --- a/src/system/detail/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -172,6 +172,7 @@ int vnodeFreeCacheBlock(SCacheBlock *pCacheBlock) { SCachePool *pPool = (SCachePool *)vnodeList[pObj->vnode].pCachePool; if (pCacheBlock->notFree) { pPool->notFreeSlots--; + pPool->unCommittedBlocks--; dTrace("vid:%d sid:%d id:%s, cache block is not free, slot:%d, index:%d notFreeSlots:%d", pObj->vnode, pObj->sid, pObj->meterId, pCacheBlock->slot, pCacheBlock->index, pPool->notFreeSlots); } From 1f953a0f63b1b2fb0fc027c7a33c418d313964ac Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 19 Nov 2019 10:36:39 +0800 Subject: [PATCH 67/69] fix #742 unCommittedBlocks shall be decreased by one --- src/system/detail/src/vnodeCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/detail/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c index d4a654de95..45596ff553 100644 --- a/src/system/detail/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -172,7 +172,7 @@ int vnodeFreeCacheBlock(SCacheBlock *pCacheBlock) { SCachePool *pPool = (SCachePool *)vnodeList[pObj->vnode].pCachePool; if (pCacheBlock->notFree) { pPool->notFreeSlots--; - pPool->unCommittedBlocks--; + pInfo->unCommittedBlocks--; dTrace("vid:%d sid:%d id:%s, cache block is not free, slot:%d, index:%d notFreeSlots:%d", pObj->vnode, pObj->sid, pObj->meterId, pCacheBlock->slot, pCacheBlock->index, pPool->notFreeSlots); } From d0a3a288a2c5cb3cb5292c80ba95c1bb34013866 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 19 Nov 2019 14:23:47 +0800 Subject: [PATCH 68/69] [TBASE-1186] --- src/inc/taoserror.h | 1 + src/rpc/src/tstring.c | 1 + src/system/detail/src/dnodeMgmt.c | 6 ++- src/system/detail/src/mgmtDnodeInt.c | 8 ++-- src/system/detail/src/mgmtMeter.c | 62 ++++++++++++++++++++-------- src/system/detail/src/mgmtShell.c | 55 +++++++++++++++--------- src/system/detail/src/mgmtVgroup.c | 25 +++++++---- src/system/detail/src/vnodeStore.c | 4 +- src/system/lite/src/dnodeMgmt.spec.c | 2 +- 9 files changed, 110 insertions(+), 54 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 9afc3de7a8..5510212dbc 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -135,6 +135,7 @@ extern "C" { #define TSDB_CODE_NOT_ACTIVE_TABLE 114 #define TSDB_CODE_INVALID_TABLE_ID 115 #define TSDB_CODE_INVALID_VNODE_STATUS 116 +#define TSDB_CODE_FAILED_TO_LOCK_RESOURCES 117 #ifdef __cplusplus } diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index d11f75885f..e3daca130a 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -241,4 +241,5 @@ char *tsError[] = {"success", "not active table(not created yet or deleted already)", //114 "invalid table id", "invalid vnode status", //116 + "failed to lock resources", }; diff --git a/src/system/detail/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c index a1668b89cf..630e50b44d 100644 --- a/src/system/detail/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -373,7 +373,11 @@ int vnodeProcessVPeerCfg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { } else { dTrace("vid:%d, vnode is not empty", vnode); if (pCfg->maxSessions > 0) { - dTrace("vid:%d, status:%s, start to update vnode", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + if (vnodeList[vnode].vnodeStatus == TSDB_VNODE_STATUS_DELETING) { + dTrace("vid:%d, status:%s, wait vnode delete finished", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + } else { + dTrace("vid:%d, status:%s, start to update vnode", vnode, taosGetVnodeStatusStr(vnodeList[vnode].vnodeStatus)); + } /* if (pCfg->maxSessions != vnodeList[vnode].cfg.maxSessions) { vnodeCleanUpOneVnode(vnode); diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index af86757d51..79f036d0d7 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -128,7 +128,8 @@ int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) { } if (pDb->vgStatus != TSDB_VG_STATUS_IN_PROGRESS) { - mTrace("dnode:%s, db:%s vpeer rsp already disposed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); + mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); + mTrace("dnode:%s, db:%s vpeer rsp already disposed, vgroup status:%d code:%d", taosIpStr(pObj->privateIp), pRsp->more, pDb->vgStatus, pRsp->code); return 0; } @@ -140,10 +141,11 @@ int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) { if (pRsp->code == TSDB_CODE_VG_COMMITLOG_INIT_FAILED) { pDb->vgStatus = TSDB_VG_STATUS_COMMITLOG_INIT_FAILED; + mError("dnode:%s, db:%s vgroup commit log init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); } else { pDb->vgStatus = TSDB_VG_STATUS_INIT_FAILED; + mError("dnode:%s, db:%s vgroup init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); } - mError("dnode:%s, db:%s vgroup create failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); return 0; } @@ -331,7 +333,6 @@ char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, int vnode) { pCfg->replications = (char)pVgroup->numOfVnodes; pCfg->rowsInFileBlock = htonl(pCfg->rowsInFileBlock); -#ifdef CLUSTER SVPeerDesc *vpeerDesc = pVPeers->vpeerDesc; pMsg = (char *)(pVPeers->vpeerDesc); @@ -341,7 +342,6 @@ char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, int vnode) { vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); pMsg += sizeof(SVPeerDesc); } -#endif return pMsg; } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index d4ffabe4b7..ba9c0a4cb9 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -502,7 +502,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { int numOfTables = sdbGetNumOfRows(meterSdb); if (numOfTables >= tsMaxTables) { - mWarn("numOfTables:%d, exceed tsMaxTables:%d", numOfTables, tsMaxTables); + mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->meterId, numOfTables, tsMaxTables); return TSDB_CODE_TOO_MANY_TABLES; } @@ -510,6 +510,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { assert(pAcct != NULL); int code = mgmtCheckMeterLimit(pAcct, pCreate); if (code != 0) { + mError("table:%s, exceed the limit", pCreate->meterId); return code; } @@ -517,8 +518,10 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter = mgmtGetMeter(pCreate->meterId); if (pMeter) { if (pCreate->igExists) { + mError("table:%s, igExists is true", pCreate->meterId); return TSDB_CODE_SUCCESS; } else { + mError("table:%s, table is already exist", pCreate->meterId); return TSDB_CODE_TABLE_ALREADY_EXIST; } } @@ -533,6 +536,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { char *pTagData = (char *)pCreate->schema; // it is a tag key pMetric = mgmtGetMeter(pTagData); if (pMetric == NULL) { + mError("table:%s, corresponding super table does not exist", pCreate->meterId); return TSDB_CODE_INVALID_TABLE; } @@ -545,6 +549,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->schema = (char *)malloc(size); if (pMeter->schema == NULL) { mgmtDestroyMeter(pMeter); + mError("table:%s, corresponding super table schema is null", pCreate->meterId); return TSDB_CODE_INVALID_TABLE; } memset(pMeter->schema, 0, size); @@ -556,13 +561,13 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->pTagData = pMeter->schema; pMeter->nextColId = pMetric->nextColId; memcpy(pMeter->pTagData, pTagData, size); - } else { int numOfCols = pCreate->numOfColumns + pCreate->numOfTags; size = numOfCols * sizeof(SSchema) + pCreate->sqlLen; pMeter->schema = (char *)malloc(size); if (pMeter->schema == NULL) { mgmtDestroyMeter(pMeter); + mError("table:%s, no schema input", pCreate->meterId); return TSDB_CODE_SERV_OUT_OF_MEMORY; } memset(pMeter->schema, 0, size); @@ -583,7 +588,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->pSql = pMeter->schema + numOfCols * sizeof(SSchema); memcpy(pMeter->pSql, (char *)(pCreate->schema) + numOfCols * sizeof(SSchema), pCreate->sqlLen); pMeter->pSql[pCreate->sqlLen - 1] = 0; - mTrace("stream sql len:%d, sql:%s", pCreate->sqlLen, pMeter->pSql); + mTrace("table:%s, stream sql len:%d sql:%s", pCreate->meterId, pCreate->sqlLen, pMeter->pSql); } else { if (pCreate->numOfTags > 0) { pMeter->meterType = TSDB_METER_METRIC; @@ -596,13 +601,14 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->createdTime = taosGetTimestampMs(); strcpy(pMeter->meterId, pCreate->meterId); if (pthread_rwlock_init(&pMeter->rwLock, NULL)) { - mError("Failed to init meter lock"); + mError("table:%s, failed to init meter lock", pCreate->meterId); mgmtDestroyMeter(pMeter); - return TSDB_CODE_OTHERS; + return TSDB_CODE_FAILED_TO_LOCK_RESOURCES; } code = mgmtCheckMeterGrant(pCreate, pMeter); if (code != 0) { + mError("table:%s, grant expired", pCreate->meterId); return code; } @@ -611,21 +617,26 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { if (pDb->vgStatus == TSDB_VG_STATUS_IN_PROGRESS) { mgmtDestroyMeter(pMeter); + mTrace("table:%s, vgroup in creating progress", pCreate->meterId); + mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); return TSDB_CODE_ACTION_IN_PROGRESS; } if (pDb->vgStatus == TSDB_VG_STATUS_FULL) { mgmtDestroyMeter(pMeter); + mError("table:%s, vgroup is full", pCreate->meterId); return TSDB_CODE_NO_ENOUGH_DNODES; } if (pDb->vgStatus == TSDB_VG_STATUS_COMMITLOG_INIT_FAILED) { mgmtDestroyMeter(pMeter); + mError("table:%s, commit log init failed", pCreate->meterId); return TSDB_CODE_VG_COMMITLOG_INIT_FAILED; } if (pDb->vgStatus == TSDB_VG_STATUS_INIT_FAILED) { mgmtDestroyMeter(pMeter); + mError("table:%s, vgroup init failed", pCreate->meterId); return TSDB_CODE_VG_INIT_FAILED; } @@ -633,12 +644,14 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pDb->vgStatus = TSDB_VG_STATUS_IN_PROGRESS; mgmtCreateVgroup(pDb); mgmtDestroyMeter(pMeter); + mTrace("table:%s, vgroup malloced, wait for create progress finished", pCreate->meterId); + mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); return TSDB_CODE_ACTION_IN_PROGRESS; } int sid = taosAllocateId(pVgroup->idPool); if (sid < 0) { - mWarn("db:%s, vgroup:%d, run out of ID, num:%d", pDb->name, pVgroup->vgId, taosIdPoolNumOfUsed(pVgroup->idPool)); + mWarn("table:%s, vgroup:%d run out of ID, num:%d", pCreate->meterId, pVgroup->vgId, taosIdPoolNumOfUsed(pVgroup->idPool)); pDb->vgStatus = TSDB_VG_STATUS_IN_PROGRESS; mgmtCreateVgroup(pDb); mgmtDestroyMeter(pMeter); @@ -650,18 +663,21 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->uid = (((uint64_t)pMeter->gid.vgId) << 40) + ((((uint64_t)pMeter->gid.sid) & ((1ul << 24) - 1ul)) << 16) + ((uint64_t)sdbVersion & ((1ul << 16) - 1ul)); - mTrace("meter:%s, create meter in vgroup, vgId:%d, sid:%d, vnode:%d, uid:%d", - pMeter->meterId, pVgroup->vgId, sid, pVgroup->vnodeGid[0].vnode, pMeter->uid); + mTrace("table:%s, create table in vgroup, vgId:%d sid:%d vnode:%d uid:%d db:%s", + pMeter->meterId, pVgroup->vgId, sid, pVgroup->vnodeGid[0].vnode, pMeter->uid, pDb->name); } else { pMeter->uid = (((uint64_t)pMeter->createdTime) << 16) + ((uint64_t)sdbVersion & ((1ul << 16) - 1ul)); } - if (sdbInsertRow(meterSdb, pMeter, 0) < 0) return TSDB_CODE_SDB_ERROR; + if (sdbInsertRow(meterSdb, pMeter, 0) < 0) { + mError("table:%s, update sdb error", pCreate->meterId); + return TSDB_CODE_SDB_ERROR; + } // send create message to the selected vnode servers if (pCreate->numOfTags == 0) { - mTrace("meter:%s, send msg to dnode, vgId:%d, sid:%d, vnode:%d, dbname:%s", - pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid, pVgroup->vnodeGid[0].vnode, pDb->name); + mTrace("table:%s, send create msg to dnode, vgId:%d, sid:%d, vnode:%d", + pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid, pVgroup->vnodeGid[0].vnode); grantAddTimeSeries(pMeter->numOfColumns - 1); mgmtSendCreateMsgToVgroup(pMeter, pVgroup); @@ -881,7 +897,10 @@ void mgmtCleanUpMeters() { sdbCloseTable(meterSdb); } int mgmtGetMeterMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { int cols = 0; - if (pConn->pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; SSchema *pSchema = tsGetSchema(pMeta); @@ -916,7 +935,7 @@ int mgmtGetMeterMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; // pShow->numOfRows = sdbGetNumOfRows (meterSdb); - pShow->numOfRows = pConn->pDb->numOfTables; + pShow->numOfRows = pDb->numOfTables; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; return 0; @@ -1208,8 +1227,12 @@ int mgmtRetrieveMeters(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { int numOfRead = 0; char prefix[20] = {0}; - if (pConn->pDb == NULL) return 0; - strcpy(prefix, pConn->pDb->name); + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb == NULL) return 0; + + strcpy(prefix, pDb->name); strcat(prefix, TS_PATH_DELIMITER); prefixLen = strlen(prefix); @@ -1269,7 +1292,10 @@ int mgmtRetrieveMeters(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { int mgmtGetMetricMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { int cols = 0; - if (pConn->pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; SSchema *pSchema = tsGetSchema(pMeta); @@ -1309,8 +1335,8 @@ int mgmtGetMetricMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pShow->offset[0] = 0; for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - pShow->numOfRows = pConn->pDb->numOfMetrics; - pShow->pNode = pConn->pDb->pMetric; + pShow->numOfRows = pDb->numOfMetrics; + pShow->pNode = pDb->pMetric; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; return 0; diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index efd863a7ef..e58938bdad 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -189,8 +189,11 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { int size = sizeof(STaosHeader) + sizeof(STaosRsp) + sizeof(SMeterMeta) + sizeof(SSchema) * TSDB_MAX_COLUMNS + sizeof(SSchema) * TSDB_MAX_TAGS + TSDB_MAX_TAGS_LEN + TSDB_EXTRA_PAYLOAD_SIZE; + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + // todo db check should be extracted - if (pConn->pDb == NULL || (pConn->pDb != NULL && pConn->pDb->dropStatus != TSDB_DB_STATUS_READY)) { + if (pDb == NULL || (pDb != NULL && pDb->dropStatus != TSDB_DB_STATUS_READY)) { if ((pStart = mgmtAllocMsg(pConn, size, &pMsg, &pRsp)) == NULL) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); @@ -223,10 +226,10 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { SDbObj* pMeterDb = mgmtGetDbByMeterId(pCreateMsg->meterId); mTrace("meter:%s, pConnDb:%p, pConnDbName:%s, pMeterDb:%p, pMeterDbName:%s", - pCreateMsg->meterId, pConn->pDb, pConn->pDb->name, pMeterDb, pMeterDb->name); - assert(pConn->pDb == pMeterDb); + pCreateMsg->meterId, pDb, pDb->name, pMeterDb, pMeterDb->name); + assert(pDb == pMeterDb); - int32_t code = mgmtCreateMeter(pConn->pDb, pCreateMsg); + int32_t code = mgmtCreateMeter(pDb, pCreateMsg); char stableName[TSDB_METER_ID_LEN] = {0}; strncpy(stableName, pInfo->tags, TSDB_METER_ID_LEN); @@ -256,7 +259,7 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { } if (pMeterObj == NULL) { - if (pConn->pDb) + if (pDb) pRsp->code = TSDB_CODE_INVALID_TABLE; else pRsp->code = TSDB_CODE_DB_NOT_SELECTED; @@ -274,7 +277,7 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { pMeta->vgid = htonl(pMeterObj->gid.vgId); pMeta->sversion = htons(pMeterObj->sversion); - pMeta->precision = pConn->pDb->cfg.precision; + pMeta->precision = pDb->cfg.precision; pMeta->numOfTags = pMeterObj->numOfTags; pMeta->numOfColumns = htons(pMeterObj->numOfColumns); @@ -505,7 +508,10 @@ int mgmtProcessMetricMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg *)(((char *)pMetricMetaMsg) + pMetricMetaMsg->metaElem[0]); pMetric = mgmtGetMeter(pElem->meterId); - if (pMetric == NULL || (pConn->pDb != NULL && pConn->pDb->dropStatus != TSDB_DB_STATUS_READY)) { + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pMetric == NULL || (pDb != NULL && pDb->dropStatus != TSDB_DB_STATUS_READY)) { pStart = taosBuildRspMsg(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP); if (pStart == NULL) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); @@ -514,7 +520,7 @@ int mgmtProcessMetricMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { pMsg = pStart; pRsp = (STaosRsp *)pMsg; - if (pConn->pDb) + if (pDb) pRsp->code = TSDB_CODE_INVALID_TABLE; else pRsp->code = TSDB_CODE_DB_NOT_SELECTED; @@ -957,17 +963,23 @@ int mgmtProcessCreateTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { pSchema++; } - if (pConn->pDb) { - code = mgmtCreateMeter(pConn->pDb, pCreate); - if (code == 0) { - mTrace("meter:%s is created by %s", pCreate->meterId, pConn->pUser->user); - // mLPrint("meter:%s is created by %s", pCreate->meterId, pConn->pUser->user); - } + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb) { + code = mgmtCreateMeter(pDb, pCreate); } else { code = TSDB_CODE_DB_NOT_SELECTED; } } + if (code != 0) { + mError("table:%s, failed to create table, code:%d", pCreate->meterId, code); + } else { + mTrace("table:%s, table is created by %s", pCreate->meterId, pConn->pUser->user); + //mLPrint("meter:%s is created by %s", pCreate->meterId, pConn->pUser->user); + } + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_TABLE_RSP, code); return 0; @@ -984,7 +996,10 @@ int mgmtProcessDropTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { - code = mgmtDropMeter(pConn->pDb, pDrop->meterId, pDrop->igNotExists); + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + code = mgmtDropMeter(pDb, pDrop->meterId, pDrop->igNotExists); if (code == 0) { mTrace("meter:%s is dropped by user:%s", pDrop->meterId, pConn->pUser->user); // mLPrint("meter:%s is dropped by user:%s", pDrop->meterId, pConn->pUser->user); @@ -1014,12 +1029,15 @@ int mgmtProcessAlterTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { mError("meter:%s error numOfCols:%d in alter table", pAlter->meterId, pAlter->numOfCols); code = TSDB_CODE_APP_ERROR; } else { - if (pConn->pDb) { + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb) { for (int32_t i = 0; i < pAlter->numOfCols; ++i) { pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes); } - code = mgmtAlterMeter(pConn->pDb, pAlter); + code = mgmtAlterMeter(pDb, pAlter); if (code == 0) { mLPrint("meter:%s is altered by %s", pAlter->meterId, pConn->pUser->user); } @@ -1263,8 +1281,7 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { } if (pConn->pAcct) { - if (pConn->pDb == NULL || - strncmp(pConn->pDb->name, pHead->db, tListLen(pConn->pDb->name)) != 0) { + if (pConn->pDb == NULL || strncmp(pConn->pDb->name, pHead->db, tListLen(pConn->pDb->name)) != 0) { pConn->pDb = mgmtGetDb(pHead->db); } diff --git a/src/system/detail/src/mgmtVgroup.c b/src/system/detail/src/mgmtVgroup.c index 3b1da25a0a..97a7343e1e 100644 --- a/src/system/detail/src/mgmtVgroup.c +++ b/src/system/detail/src/mgmtVgroup.c @@ -123,7 +123,7 @@ void mgmtProcessVgTimer(void *handle, void *tmrId) { if (pDb == NULL) return; if (pDb->vgStatus > TSDB_VG_STATUS_IN_PROGRESS) { - mTrace("db:%s, set vgstatus from %d to %d", pDb->name, pDb->vgStatus, TSDB_VG_STATUS_READY); + mTrace("db:%s, set vgroup status from %d to ready", pDb->name, pDb->vgStatus); pDb->vgStatus = TSDB_VG_STATUS_READY; } @@ -143,7 +143,7 @@ SVgObj *mgmtCreateVgroup(SDbObj *pDb) { // based on load balance, create a new one if (mgmtAllocVnodes(pVgroup) != 0) { - mError("no enough free dnode"); + mError("db:%s, no enough free dnode to alloc %d vnodes", pDb->name, pVgroup->numOfVnodes); free(pVgroup); pDb->vgStatus = TSDB_VG_STATUS_FULL; taosTmrReset(mgmtProcessVgTimer, 5000, pDb, mgmtTmr, &pDb->vgTimer); @@ -152,9 +152,9 @@ SVgObj *mgmtCreateVgroup(SDbObj *pDb) { sdbInsertRow(vgSdb, pVgroup, 0); - mTrace("vgroup:%d, db:%s replica:%d is created", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); + mTrace("vgroup:%d, vgroup is created, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int i = 0; i < pVgroup->numOfVnodes; ++i) - mTrace("dnode:%s, vgroup:%d, vnode:%d is created", taosIpStr(pVgroup->vnodeGid[i].ip), pVgroup->vgId, pVgroup->vnodeGid[i].vnode); + mTrace("vgroup:%d, dnode:%s vnode:%d is created", pVgroup->vgId, taosIpStr(pVgroup->vnodeGid[i].ip), pVgroup->vnodeGid[i].vnode); mgmtSendVPeersMsg(pVgroup); @@ -206,7 +206,10 @@ void mgmtCleanUpVgroups() { sdbCloseTable(vgSdb); } int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { int cols = 0; - if (pConn->pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + if (pDb == NULL) return TSDB_CODE_DB_NOT_SELECTED; SSchema *pSchema = tsGetSchema(pMeta); @@ -229,7 +232,7 @@ int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { cols++; int maxReplica = 0; - SVgObj *pVgroup = pConn->pDb->pHead; + SVgObj *pVgroup = pDb->pHead; while (pVgroup != NULL) { maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; pVgroup = pVgroup->next; @@ -267,8 +270,8 @@ int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pShow->offset[0] = 0; for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - pShow->numOfRows = pConn->pDb->numOfVgroups; - pShow->pNode = pConn->pDb->pHead; + pShow->numOfRows = pDb->numOfVgroups; + pShow->pNode = pDb->pHead; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; return 0; @@ -282,7 +285,11 @@ int mgmtRetrieveVgroups(SShowObj *pShow, char *data, int rows, SConnObj *pConn) char ipstr[20]; int maxReplica = 0; - pVgroup = pConn->pDb->pHead; + + SDbObj *pDb = NULL; + if (pConn->pDb != NULL) pDb = mgmtGetDb(pConn->pDb->name); + + pVgroup = pDb->pHead; while (pVgroup != NULL) { maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; pVgroup = pVgroup->next; diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index 8c958c21d7..572bd88cf7 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -141,7 +141,7 @@ static int vnodeCloseVnode(int vnode) { } if (pVnode->vnodeStatus == TSDB_VNODE_STATUS_DELETING) { - dTrace("vid:%d, status:%s, another performed delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); + dTrace("vid:%d, status:%s, another thread performed delete operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); return TSDB_CODE_SUCCESS; } else { dTrace("vid:%d, status:%s, enter close operation", vnode, taosGetVnodeStatusStr(pVnode->vnodeStatus)); @@ -250,7 +250,7 @@ static void vnodeRemoveDataFiles(int vnode) { sprintf(vnodeDir, "%s/vnode%d", tsDirectory, vnode); rmdir(vnodeDir); - dTrace("vnode %d is removed!", vnode); + dTrace("vid:%d, vnode is removed!", vnode); } int vnodeRemoveVnode(int vnode) { diff --git a/src/system/lite/src/dnodeMgmt.spec.c b/src/system/lite/src/dnodeMgmt.spec.c index 00e7e469c0..6c424a85af 100644 --- a/src/system/lite/src/dnodeMgmt.spec.c +++ b/src/system/lite/src/dnodeMgmt.spec.c @@ -50,7 +50,7 @@ char *taosBuildReqMsgToMnode(SMgmtObj *pObj, char type) { } int taosSendMsgToMnode(SMgmtObj *pObj, char *msg, int msgLen) { - mTrace("msg:%s is sent to mnode", taosMsg[*(msg-1)]); + dTrace("msg:%s is sent to mnode", taosMsg[*(msg-1)]); /* * Lite version has no message header, so minus one From eb6be38711b3788637d6a01c29aaf4f0343ebad8 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 19 Nov 2019 14:53:25 +0800 Subject: [PATCH 69/69] remove some un necessary logs --- src/system/detail/src/mgmtDnodeInt.c | 1 - src/system/detail/src/mgmtMeter.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index 79f036d0d7..9310182a14 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -128,7 +128,6 @@ int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) { } if (pDb->vgStatus != TSDB_VG_STATUS_IN_PROGRESS) { - mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); mTrace("dnode:%s, db:%s vpeer rsp already disposed, vgroup status:%d code:%d", taosIpStr(pObj->privateIp), pRsp->more, pDb->vgStatus, pRsp->code); return 0; } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index ba9c0a4cb9..80ea5142e9 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -618,7 +618,6 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { if (pDb->vgStatus == TSDB_VG_STATUS_IN_PROGRESS) { mgmtDestroyMeter(pMeter); mTrace("table:%s, vgroup in creating progress", pCreate->meterId); - mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); return TSDB_CODE_ACTION_IN_PROGRESS; } @@ -645,7 +644,6 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { mgmtCreateVgroup(pDb); mgmtDestroyMeter(pMeter); mTrace("table:%s, vgroup malloced, wait for create progress finished", pCreate->meterId); - mTrace("===> pDb:%s %p status:%d", pDb->name, pDb, pDb->vgStatus); return TSDB_CODE_ACTION_IN_PROGRESS; }