From 4918783d5a3e1f5c6b8c6743e96a87db5ebe232a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 9 Nov 2019 22:30:27 +0800 Subject: [PATCH 01/10] refactor the header file format in client module --- src/os/linux/inc/os.h | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 004896960e..1a10bc3fe2 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -1,17 +1,17 @@ /* -* Copyright (c) 2019 TAOS Data, Inc. -* -* This program is free software: you can use, redistribute, and/or modify -* it under the terms of the GNU Affero General Public License, version 3 -* or later ("AGPL"), as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, but WITHOUT -* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -* FITNESS FOR A PARTICULAR PURPOSE. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #ifndef TDENGINE_PLATFORM_LINUX_H #define TDENGINE_PLATFORM_LINUX_H @@ -25,10 +25,12 @@ extern "C" { #include #include +#include #include #include #include #include +#include #include #include #include @@ -37,7 +39,8 @@ extern "C" { #include #include #include -#include +#include +#include #include #include #include @@ -45,21 +48,19 @@ extern "C" { #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include #include #include #include -#include -#include #define taosCloseSocket(x) \ { \ @@ -81,7 +82,7 @@ extern "C" { #define __sync_sub_and_fetch_32 __sync_sub_and_fetch int32_t __sync_val_load_32(int32_t *ptr); -void __sync_val_restore_32(int32_t *ptr, int32_t newval); +void __sync_val_restore_32(int32_t *ptr, int32_t newval); #define SWAP(a, b, c) \ do { \ @@ -139,9 +140,9 @@ bool taosSkipSocketCheck(); int64_t str2int64(char *str); #define BUILDIN_CLZL(val) __builtin_clzl(val) -#define BUILDIN_CLZ(val) __builtin_clz(val) +#define BUILDIN_CLZ(val) __builtin_clz(val) #define BUILDIN_CTZL(val) __builtin_ctzl(val) -#define BUILDIN_CTZ(val) __builtin_ctz(val) +#define BUILDIN_CTZ(val) __builtin_ctz(val) #ifdef __cplusplus } From c0d6cb743fd1dbfded9bafd5de6fe05231f65d8c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 9 Nov 2019 22:38:22 +0800 Subject: [PATCH 02/10] refactor the header file format in client module --- src/client/src/TSDBJNIConnector.c | 2 -- src/client/src/tscAst.c | 6 ------ src/client/src/tscAsync.c | 3 +-- src/client/src/tscCache.c | 9 +-------- src/client/src/tscFunctionImpl.c | 10 ---------- src/client/src/tscJoinProcess.c | 7 ------- src/client/src/tscLocal.c | 4 +--- src/client/src/tscParseInsert.c | 16 +--------------- src/client/src/tscPrepare.c | 3 --- src/client/src/tscProfile.c | 3 --- src/client/src/tscSQLParserImpl.c | 7 ------- src/client/src/tscSchemaUtil.c | 4 ---- src/client/src/tscSecondaryMerge.c | 7 +------ src/client/src/tscServer.c | 8 +------- src/client/src/tscSql.c | 3 --- src/client/src/tscSub.c | 2 +- src/client/src/tscSyntaxtreefunction.c | 5 +---- src/client/src/tscSystem.c | 9 --------- src/client/src/tscUtil.c | 4 ---- 19 files changed, 8 insertions(+), 104 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index fb175618c0..e98acfc9a6 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include - #include "os.h" #include "com_taosdata_jdbc_TSDBJNIConnector.h" #include "taos.h" diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index 845a9fd36d..fe0921c64d 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tast.h" diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 6e0e1ae7c5..aad0295959 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include -#include +#include "os.h" #include "tlog.h" #include "trpc.h" diff --git a/src/client/src/tscCache.c b/src/client/src/tscCache.c index 866b6e7dbc..f508857ce1 100644 --- a/src/client/src/tscCache.c +++ b/src/client/src/tscCache.c @@ -13,14 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include +#include "os.h" #include "tglobalcfg.h" #include "tlog.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0f9be90412..35011a259b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -15,16 +15,6 @@ #pragma GCC diagnostic ignored "-Wincompatible-pointer-types" -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tast.h" diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index 4965498eff..dfeebc8c61 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tcache.h" #include "tscJoinProcess.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 9afb74fec2..d217d09588 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include +#include "os.h" #include "taosmsg.h" #include "tcache.h" diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 37a32e9ebb..c8ce728de1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -21,22 +21,8 @@ #pragma GCC diagnostic ignored "-Woverflow" #pragma GCC diagnostic ignored "-Wunused-variable" -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "ihash.h" #include "os.h" +#include "ihash.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index e956d6159e..03f644ac36 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "taos.h" #include "tsclient.h" #include "tsql.h" diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 770e61e278..7d74f62dcc 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "os.h" #include "tlog.h" #include "tsclient.h" diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index 7e1b4a7cf1..4fefe1665c 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -13,13 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - #include "os.h" #include "tglobalcfg.h" #include "tsql.h" diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 9728811ae7..5e50c27ab1 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -13,10 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tschemautil.h" diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index ac2638f635..398cd1cca6 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -13,13 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include "tlosertree.h" +#include "os.h" #include "tlosertree.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6edec3d477..73e612962a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include - #include "os.h" #include "tcache.h" #include "trpc.h" @@ -3613,7 +3607,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) { code = tscDoGetMeterMeta(pSql, meterId, 0); // todo ?? } else { - tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", pSql, + tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%lld, addr:%p", pSql, pMeterMetaInfo->pMeterMeta->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, pMeterMetaInfo->pMeterMeta); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e6714bb4cb..1727840a98 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include -#include - #include "os.h" #include "tcache.h" #include "tlog.h" diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index dee8f02118..bcbcaba4c4 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +#include "os.h" #include "shash.h" #include "taos.h" diff --git a/src/client/src/tscSyntaxtreefunction.c b/src/client/src/tscSyntaxtreefunction.c index 00781919e4..bbd30c1fa9 100644 --- a/src/client/src/tscSyntaxtreefunction.c +++ b/src/client/src/tscSyntaxtreefunction.c @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include -#include -#include +#include "os.h" #include "tscSyntaxtreefunction.h" #include "tsql.h" diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 7ebd43cd19..708ceee405 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -13,15 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "os.h" #include "taosmsg.h" #include "tcache.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ec5c980612..5793656477 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -13,10 +13,6 @@ * along with this program. If not, see . */ -#include -#include -#include - #include "os.h" #include "ihash.h" #include "taosmsg.h" From 7ed580d93b6de7f248d8db3ae6b5f6de80a8b1a8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 11:55:49 +0800 Subject: [PATCH 03/10] fix some memory leaks --- src/system/detail/src/mgmtMeter.c | 1 + src/system/detail/src/vnodeQueryImpl.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index bccf9a06c8..2692e39131 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -833,6 +833,7 @@ static void removeMeterFromMetricIndex(STabObj *pMetric, STabObj *pMeter) { } } + tSkipListDestroyKey(&key); if (num != 0) { free(pRes); } diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 187269d30b..9c0d7883a5 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -3436,9 +3436,18 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // only the function of interp needs the corresponding information + if (pCtx->functionId != TSDB_FUNC_INTERP) { + continue; + } + + pCtx->numOfParams = 4; + SInterpInfo * pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; // for primary timestamp column, set the flag From 5400e49c7ede3a2d60eaa4b3adc39a0fdfb29370 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 11:58:20 +0800 Subject: [PATCH 04/10] refactor some code, fix the used time too big bug. --- src/client/src/tscServer.c | 4 ++-- src/inc/ttimer.h | 2 +- src/system/detail/src/vnodeShell.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 73e612962a..49368e4d17 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2842,7 +2842,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql) { return msgLen; } -int tscEstimateBuildHeartBeatMsgLength(SSqlObj *pSql) { +int tscEstimateHeartBeatMsgLength(SSqlObj *pSql) { int size = 0; STscObj *pObj = pSql->pTscObj; @@ -2875,7 +2875,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql) { pthread_mutex_lock(&pObj->mutex); - size = tscEstimateBuildHeartBeatMsgLength(pSql); + size = tscEstimateHeartBeatMsgLength(pSql); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for heartbeat msg", pSql); return -1; diff --git a/src/inc/ttimer.h b/src/inc/ttimer.h index ddfd3f1fdf..5424881e26 100644 --- a/src/inc/ttimer.h +++ b/src/inc/ttimer.h @@ -41,7 +41,7 @@ extern int taosTmrThreads; tprintf("TMR ", tmrDebugFlag, __VA_ARGS__); \ } } while(0) -#define MAX_NUM_OF_TMRCTL 32 +#define MAX_NUM_OF_TMRCTL 512 #define MSECONDS_PER_TICK 5 void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 190c1d6ee6..353b2668ee 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -419,7 +419,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { if (code == TSDB_CODE_SUCCESS) { pRsp->offset = htobe64(vnodeGetOffsetVal(pRetrieve->qhandle)); - pRsp->useconds = ((SQInfo *)(pRetrieve->qhandle))->useconds; + pRsp->useconds = htobe64(((SQInfo *)(pRetrieve->qhandle))->useconds); } else { pRsp->offset = 0; pRsp->useconds = 0; From 1f78d9b859c26daf5127bae0c790ff83208af38f Mon Sep 17 00:00:00 2001 From: localvar Date: Mon, 11 Nov 2019 03:28:55 +0000 Subject: [PATCH 05/10] make(rollback) max timer controller number to 512 and make it configurable. --- src/inc/tglobalcfg.h | 2 ++ src/inc/ttimer.h | 2 +- src/util/src/tglobalcfg.c | 5 +++++ src/util/src/ttimer.c | 21 ++++++++++++++++----- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index e83c89593f..026d48ba08 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -169,6 +169,8 @@ extern uint32_t debugFlag; extern uint32_t odbcdebugFlag; extern uint32_t qdebugFlag; +extern uint32_t taosMaxTmrCtrl; + extern int tsRpcTimer; extern int tsRpcMaxTime; extern int tsUdpDelay; diff --git a/src/inc/ttimer.h b/src/inc/ttimer.h index 5424881e26..b9bbbb04dd 100644 --- a/src/inc/ttimer.h +++ b/src/inc/ttimer.h @@ -25,6 +25,7 @@ typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern uint32_t tmrDebugFlag; extern int taosTmrThreads; +extern uint32_t taosMaxTmrCtrl; #define tmrError(...) \ do { if (tmrDebugFlag & DEBUG_ERROR) { \ @@ -41,7 +42,6 @@ extern int taosTmrThreads; tprintf("TMR ", tmrDebugFlag, __VA_ARGS__); \ } } while(0) -#define MAX_NUM_OF_TMRCTL 512 #define MSECONDS_PER_TICK 5 void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 9abe60220f..586f329001 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -536,6 +536,11 @@ void tsInitGlobalConfig() { 0, 2, 0, TSDB_CFG_UTYPE_NONE); // 0-any, 1-mgmt, 2-dnode + // timer + tsInitConfigOption(cfg++, "maxTmrCtrl", &taosMaxTmrCtrl, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 8, 2048, 0, TSDB_CFG_UTYPE_NONE); + // time tsInitConfigOption(cfg++, "monitorInterval", &tsMonitorInterval, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 798e63a64a..e6d98a3734 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -82,13 +82,16 @@ typedef struct time_wheel_t { } time_wheel_t; uint32_t tmrDebugFlag = DEBUG_ERROR | DEBUG_WARN | DEBUG_FILE; +uint32_t taosMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tmrCtrlMutex; -static tmr_ctrl_t tmrCtrls[MAX_NUM_OF_TMRCTL]; +static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; -void* tmrQhandle; -int taosTmrThreads = 1; +static void* tmrQhandle; +static int numOfTmrCtrl = 0; + +int taosTmrThreads = 1; static uintptr_t nextTimerId = 0; @@ -481,7 +484,13 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, } static void taosTmrModuleInit(void) { - for (int i = 0; i < tListLen(tmrCtrls) - 1; ++i) { + tmrCtrls = malloc(sizeof(tmr_ctrl_t) * taosMaxTmrCtrl); + if (tmrCtrls == NULL) { + tmrError("failed to allocate memory for timer controllers."); + return; + } + + for (int i = 0; i < taosMaxTmrCtrl - 1; ++i) { tmr_ctrl_t* ctrl = tmrCtrls + i; ctrl->next = ctrl + 1; } @@ -526,6 +535,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab tmr_ctrl_t* ctrl = unusedTmrCtrl; if (ctrl != NULL) { unusedTmrCtrl = ctrl->next; + numOfTmrCtrl++; } pthread_mutex_unlock(&tmrCtrlMutex); @@ -536,7 +546,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab strncpy(ctrl->label, label, sizeof(ctrl->label)); ctrl->label[sizeof(ctrl->label) - 1] = 0; - tmrTrace("timer controller[label=%s] is initialized.", label); + tmrTrace("timer controller[label=%s] is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); return ctrl; } @@ -549,6 +559,7 @@ void taosTmrCleanUp(void* handle) { pthread_mutex_lock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; + numOfTmrCtrl--; unusedTmrCtrl = ctrl; pthread_mutex_unlock(&tmrCtrlMutex); } From 48c60fd01a86373e5c656186c5235b22763bfa88 Mon Sep 17 00:00:00 2001 From: localvar Date: Mon, 11 Nov 2019 04:08:14 +0000 Subject: [PATCH 06/10] update timer log format --- src/util/src/ttimer.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index e6d98a3734..97e2189153 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -132,7 +132,7 @@ static void unlockTimerList(timer_list_t* list) { int64_t tid = taosGetPthreadId(); if (__sync_val_compare_and_swap_64(&(list->lockedBy), tid, 0) != tid) { assert(false); - tmrError("trying to unlock a timer list not locked by current thread."); + tmrError("%d trying to unlock a timer list not locked by current thread.", tid); } } @@ -259,13 +259,13 @@ static void processExpiredTimer(void* handle, void* arg) { timer->executedBy = taosGetPthreadId(); uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] execution start."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] execution start."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); (*timer->fp)(timer->param, (tmr_h)timer->id); atomic_store_8(&timer->state, TIMER_STATE_STOPPED); - fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] execution end."; + fmt = "%s timer[id=%lld, fp=%p, param=%p] execution end."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } removeTimer(timer->id); @@ -273,18 +273,21 @@ static void processExpiredTimer(void* handle, void* arg) { } static void addToExpired(tmr_obj_t* head) { - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] expired"; + const char* fmt = "%s adding expired timer[id=%lld, fp=%p, param=%p] to queue."; while (head != NULL) { - tmrTrace(fmt, head->ctrl->label, head->id, head->fp, head->param); - + uintptr_t id = head->id; tmr_obj_t* next = head->next; + tmrTrace(fmt, head->ctrl->label, id, head->fp, head->param); + SSchedMsg schedMsg; schedMsg.fp = NULL; schedMsg.tfp = processExpiredTimer; schedMsg.ahandle = head; schedMsg.thandle = NULL; taosScheduleTask(tmrQhandle, &schedMsg); + + tmrTrace("timer[id=%lld] has been added to queue.", id); head = next; } } @@ -298,7 +301,7 @@ static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int msecon timer->ctrl = ctrl; addTimer(timer); - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] started"; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] started"; tmrTrace(fmt, ctrl->label, timer->id, timer->fp, timer->param); if (mseconds == 0) { @@ -321,7 +324,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle tmr_obj_t* timer = (tmr_obj_t*)calloc(1, sizeof(tmr_obj_t)); if (timer == NULL) { - tmrError("failed to allocated memory for new timer object."); + tmrError("%s failed to allocated memory for new timer object.", ctrl->label); return NULL; } @@ -392,7 +395,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { // we cannot guarantee the thread safety of the timr in all other cases. reusable = true; } - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] is cancelled."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] is cancelled."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } else if (state != TIMER_STATE_EXPIRED) { // timer already stopped or cancelled, has nothing to do in this case @@ -403,7 +406,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { } else { assert(timer->executedBy != taosGetPthreadId()); - const char* fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] fired, waiting..."; + const char* fmt = "%s timer[id=%lld, fp=%p, param=%p] fired, waiting..."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); for (int i = 1; atomic_load_8(&timer->state) != TIMER_STATE_STOPPED; i++) { @@ -412,7 +415,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { } } - fmt = "timer[label=%s, id=%lld, fp=%p, param=%p] stopped."; + fmt = "%s timer[id=%lld, fp=%p, param=%p] stopped."; tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param); } @@ -451,7 +454,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, bool stopped = false; tmr_obj_t* timer = findTimer(id); if (timer == NULL) { - tmrTrace("timer[id=%lld] does not exist", id); + tmrTrace("%s timer[id=%lld] does not exist", ctrl->label, id); } else { uint8_t state = __sync_val_compare_and_swap_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); if (!doStopTimer(timer, state)) { @@ -466,7 +469,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, return stopped; } - tmrTrace("timer[id=%lld] is reused", timer->id); + tmrTrace("%s timer[id=%lld] is reused", ctrl->label, timer->id); // wait until there's no other reference to this timer, // so that we can reuse this timer safely. @@ -540,13 +543,13 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab pthread_mutex_unlock(&tmrCtrlMutex); if (ctrl == NULL) { - tmrError("too many timer controllers, failed to create timer controller[label=%s].", label); + tmrError("%s too many timer controllers, failed to create timer controller.", label); return NULL; } strncpy(ctrl->label, label, sizeof(ctrl->label)); ctrl->label[sizeof(ctrl->label) - 1] = 0; - tmrTrace("timer controller[label=%s] is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); + tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); return ctrl; } @@ -554,7 +557,7 @@ void taosTmrCleanUp(void* handle) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; assert(ctrl != NULL && ctrl->label[0] != 0); - tmrTrace("timer controller[label=%s] is cleaned up.", ctrl->label); + tmrTrace("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; pthread_mutex_lock(&tmrCtrlMutex); From db26cf1e8e54b9d6f7cf22324603453d33591d69 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Mon, 11 Nov 2019 12:39:48 +0800 Subject: [PATCH 07/10] [TBASE-1047] Go connector prints all the SQL. This will harm the performance and consume unnecessary hard disk space. Modified to print only the SQLs which returns error. --- src/connector/go/src/taosSql/taosSqlCgo.go | 57 +++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/connector/go/src/taosSql/taosSqlCgo.go b/src/connector/go/src/taosSql/taosSqlCgo.go index e240ac44ec..fcef14045f 100755 --- a/src/connector/go/src/taosSql/taosSqlCgo.go +++ b/src/connector/go/src/taosSql/taosSqlCgo.go @@ -29,46 +29,47 @@ import ( "unsafe" ) -func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error){ +func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) { cuser := C.CString(user) cpass := C.CString(pass) - cip := C.CString(ip) - cdb := C.CString(db) + cip := C.CString(ip) + cdb := C.CString(db) defer C.free(unsafe.Pointer(cip)) defer C.free(unsafe.Pointer(cuser)) defer C.free(unsafe.Pointer(cpass)) defer C.free(unsafe.Pointer(cdb)) taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.int)(port)) - if taosObj == nil { - return nil, errors.New("taos_connect() fail!") - } + if taosObj == nil { + return nil, errors.New("taos_connect() fail!") + } - return (unsafe.Pointer)(taosObj), nil -} + return (unsafe.Pointer)(taosObj), nil +} func (mc *taosConn) taosQuery(sqlstr string) (int, error) { - taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) + //taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - csqlstr := C.CString(sqlstr) + csqlstr := C.CString(sqlstr) defer C.free(unsafe.Pointer(csqlstr)) - code := int(C.taos_query(mc.taos, csqlstr)) + code := int(C.taos_query(mc.taos, csqlstr)) - if 0 != code { - mc.taos_error() - errStr := C.GoString(C.taos_errstr(mc.taos)) - taosLog.Println("taos_query() failed:", errStr) - return 0, errors.New(errStr) - } + if 0 != code { + mc.taos_error() + errStr := C.GoString(C.taos_errstr(mc.taos)) + taosLog.Println("taos_query() failed:", errStr) + taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) + return 0, errors.New(errStr) + } - // read result and save into mc struct - num_fields := int(C.taos_field_count(mc.taos)) - if 0 == num_fields { // there are no select and show kinds of commands - mc.affectedRows = int(C.taos_affected_rows(mc.taos)) - mc.insertId = 0 - } + // read result and save into mc struct + num_fields := int(C.taos_field_count(mc.taos)) + if 0 == num_fields { // there are no select and show kinds of commands + mc.affectedRows = int(C.taos_affected_rows(mc.taos)) + mc.insertId = 0 + } - return num_fields, nil + return num_fields, nil } func (mc *taosConn) taos_close() { @@ -76,8 +77,8 @@ func (mc *taosConn) taos_close() { } func (mc *taosConn) taos_error() { - // free local resouce: allocated memory/metric-meta refcnt - //var pRes unsafe.Pointer - pRes := C.taos_use_result(mc.taos) - C.taos_free_result(pRes) + // free local resouce: allocated memory/metric-meta refcnt + //var pRes unsafe.Pointer + pRes := C.taos_use_result(mc.taos) + C.taos_free_result(pRes) } From 2a11363e0c57af6e4060a0072283cd1736259078 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 11 Nov 2019 13:04:09 +0800 Subject: [PATCH 08/10] [TBASE-1060] --- src/client/src/tscProfile.c | 45 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 770e61e278..b6de98e712 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -23,6 +23,27 @@ #include "ttimer.h" #include "tutil.h" +void tscSaveSlowQueryFp(void *handle, void *tmrId); +void *tscSlowQueryConn = NULL; +bool tscSlowQueryConnInitialized = false; +TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, int port, void (*fp)(void *, TAOS_RES *, int), + void *param, void **taos); + +void tscInitConnCb(void *param, TAOS_RES *result, int code) { + char *sql = param; + if (code < 0) { + tscError("taos:%p, slow query connect failed, code:%d", tscSlowQueryConn, code); + taos_close(tscSlowQueryConn); + tscSlowQueryConn = NULL; + tscSlowQueryConnInitialized = false; + free(sql); + } else { + tscTrace("taos:%p, slow query connect success, code:%d", tscSlowQueryConn, code); + tscSlowQueryConnInitialized = true; + tscSaveSlowQueryFp(sql, NULL); + } +} + void tscAddIntoSqlList(SSqlObj *pSql) { static uint32_t queryId = 1; @@ -47,26 +68,28 @@ void tscAddIntoSqlList(SSqlObj *pSql) { void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { if (code < 0) { - tscError("failed to save slowquery, code:%d", code); + tscError("failed to save slow query, code:%d", code); + } else { + tscTrace("success to save slow query, code:%d", code); } } void tscSaveSlowQueryFp(void *handle, void *tmrId) { char *sql = handle; - static void *taos = NULL; - if (taos == NULL) { - taos = taos_connect(NULL, "monitor", tsInternalPass, NULL, 0); - if (taos == NULL) { - tscError("failed to save slow query, can't connect to server"); + if (!tscSlowQueryConnInitialized) { + if (tscSlowQueryConn == NULL) { + tscTrace("start to init slow query connect"); + taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, tscInitConnCb, sql, &tscSlowQueryConn); + } else { + tscError("taos:%p, slow query connect is already initialized", tscSlowQueryConn); free(sql); - return; } + } else { + tscTrace("taos:%p, save slow query:%s", tscSlowQueryConn, sql); + taos_query_a(tscSlowQueryConn, sql, tscSaveSlowQueryFpCb, NULL); + free(sql); } - - tscTrace("save slow query:sql", sql); - taos_query_a(taos, sql, tscSaveSlowQueryFpCb, NULL); - free(sql); } void tscSaveSlowQuery(SSqlObj *pSql) { From 1aa5f927f374186b3883f2560ba24ab5373d54ae Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 13:08:50 +0800 Subject: [PATCH 09/10] [TBASE-1098]fix failed to drop a db, of which name start with "log" issue #686 --- src/inc/tutil.h | 2 -- src/system/detail/inc/mgmtUtil.h | 1 + src/system/detail/src/mgmtDb.c | 16 +++++++++++++--- src/system/detail/src/mgmtMeter.c | 10 ++++++---- src/system/detail/src/mgmtUtil.c | 7 +++++++ src/util/src/tutil.c | 7 ------- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/inc/tutil.h b/src/inc/tutil.h index f63f8b6cec..884e448aae 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -169,8 +169,6 @@ int32_t taosInitTimer(void (*callback)(int), int32_t ms); */ uint32_t MurmurHash3_32(const void *key, int32_t len); -bool taosCheckDbName(char *db, char *monitordb); - bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len); bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); diff --git a/src/system/detail/inc/mgmtUtil.h b/src/system/detail/inc/mgmtUtil.h index aecb229dba..6a6b819bf2 100644 --- a/src/system/detail/inc/mgmtUtil.h +++ b/src/system/detail/inc/mgmtUtil.h @@ -30,6 +30,7 @@ char* mgmtMeterGetTag(STabObj* pMeter, int32_t col, SSchema* pTagColSchema); int32_t mgmtFindTagCol(STabObj * pMetric, const char * tagName); int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col); +bool mgmtCheckIsMonitorDB(char *db, char *monitordb); int32_t mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pInfo, int32_t tableIndex, tQueryResultset* pRes); int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes); diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index ae4c5bed7c..93ff1279c0 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -14,9 +14,11 @@ */ #define _DEFAULT_SOURCE +#include "os.h" + #include "mgmt.h" -#include #include "mgmtBalance.h" +#include "mgmtUtil.h" #include "tschemautil.h" void *dbSdb = NULL; @@ -373,10 +375,18 @@ int mgmtDropDbByName(SAcctObj *pAcct, char *name) { if (pDb == NULL) { mWarn("db:%s is not there", name); // return TSDB_CODE_INVALID_DB; - return 0; + return TSDB_CODE_SUCCESS; } - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; +// char dbName[TSDB_DB_NAME_LEN + 1] = {0}; +// extractDBName(pDb->name, dbName); +// if (strncasecmp(dbName, tsMonitorDbName, strlen(dbName)) == 0) { +// return TSDB_CODE_MONITOR_DB_FORBEIDDEN; +// } + + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + } return mgmtDropDb(pDb); } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 2692e39131..2db920fdd7 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -688,8 +688,10 @@ int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore) { pAcct = mgmtGetAcct(pDb->cfg.acct); - // 0.sys - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + // 0.log + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + } if (mgmtIsNormalMeter(pMeter)) { return dropMeterImp(pDb, pMeter, pAcct); @@ -719,8 +721,8 @@ int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter) { return TSDB_CODE_INVALID_TABLE; } - // 0.sys - if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; + // 0.log + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { if (!mgmtIsNormalMeter(pMeter) || !mgmtMeterCreateFromMetric(pMeter)) { diff --git a/src/system/detail/src/mgmtUtil.c b/src/system/detail/src/mgmtUtil.c index a427771e0f..be90180f68 100644 --- a/src/system/detail/src/mgmtUtil.c +++ b/src/system/detail/src/mgmtUtil.c @@ -87,3 +87,10 @@ int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col) { // length before col return len; } + +bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { + char dbName[TSDB_DB_NAME_LEN + 1] = {0}; + extractDBName(db, dbName); + + return (strncasecmp(dbName, monitordb, strlen(dbName)) == 0); +} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 805f27a381..0d1ea9ed50 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -406,13 +406,6 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP return rename(fullPath, *dstPath); } -bool taosCheckDbName(char *db, char *monitordb) { - char *pos = strchr(db, '.'); - if (pos == NULL) return false; - - return strncasecmp(pos + 1, monitordb, strlen(monitordb)) == 0; -} - bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef USE_LIBICONV iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); From d609fe319aff0b70aa48ebd31b9202542f8c7597 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 Nov 2019 13:11:36 +0800 Subject: [PATCH 10/10] remove unused code --- src/system/detail/src/mgmtDb.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index 93ff1279c0..e87266aa4b 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -378,12 +378,6 @@ int mgmtDropDbByName(SAcctObj *pAcct, char *name) { return TSDB_CODE_SUCCESS; } -// char dbName[TSDB_DB_NAME_LEN + 1] = {0}; -// extractDBName(pDb->name, dbName); -// if (strncasecmp(dbName, tsMonitorDbName, strlen(dbName)) == 0) { -// return TSDB_CODE_MONITOR_DB_FORBEIDDEN; -// } - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { return TSDB_CODE_MONITOR_DB_FORBEIDDEN; }