From 77c6f616f9784ecaa7ac562abd3cb58b8fe5909b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 7 Apr 2020 14:39:31 +0800 Subject: [PATCH 01/48] add compile flag to treat all warnings as errors. [TD-118] --- cmake/platform.inc | 14 +++++++------- src/client/src/tscSchemaUtil.c | 4 ++-- src/client/src/tscSub.c | 14 ++++++-------- src/common/src/name.c | 4 ++-- src/inc/taosmsg.h | 4 ++-- src/mnode/inc/mgmtGrant.h | 2 +- src/query/inc/tlosertree.h | 2 +- src/query/src/queryExecutor.c | 5 +++-- src/query/src/tlosertree.c | 2 +- src/util/src/tskiplist.c | 4 ++-- 10 files changed, 27 insertions(+), 28 deletions(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 0d53f0cc43..6087b6f16f 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -107,12 +107,12 @@ IF (TD_LINUX_64) SET(RELEASE_FLAGS "-O0") IF (NOT TD_ARM) IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () ADD_DEFINITIONS(-DLINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) @@ -128,7 +128,7 @@ IF (TD_LINUX_64) ENDIF () SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DLINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) @@ -141,7 +141,7 @@ IF (TD_LINUX_64) ELSEIF (TD_WINDOWS_64) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) - SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") SET(DEBUG_FLAGS "/Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /GL") ENDIF () @@ -151,7 +151,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-DPTW32_BUILD) ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) ELSEIF (TD_DARWIN_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") ADD_DEFINITIONS(-DDARWIN) @@ -159,4 +159,4 @@ IF (TD_LINUX_64) ELSE () MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile") EXIT () - ENDIF () \ No newline at end of file + ENDIF () diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index b487d5d6db..90e1bd4c1d 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -210,7 +210,7 @@ char* tsGetTagsValue(STableMeta* pTableMeta) { } // todo refactor -static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { +__attribute__ ((unused))static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { while (*input != 0 && *input++ != delim) { }; @@ -218,7 +218,7 @@ static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { return input; } -static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { +__attribute__ ((unused)) static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { size_t len = 0; while (*src != delimiter && *src != 0) { *dst++ = *src++; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index fd457a12dd..b7d01941d6 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -108,7 +108,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* if (pSql == NULL) { terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; tscError("failed to allocate SSqlObj for subscription"); - goto failed; + goto _pSql_failed; } pSql->signature = pSql; @@ -137,13 +137,11 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* return pSub; failed: - if (sqlstr != NULL) { - free(sqlstr); - } - if (pSql != NULL) { - free(pSql); - } - free(pSub); + tfree(sqlstr); + +_pSql_failed: + tfree(pSql); + tfree(pSub); return NULL; } diff --git a/src/common/src/name.c b/src/common/src/name.c index b0411d742c..a605b98a14 100644 --- a/src/common/src/name.c +++ b/src/common/src/name.c @@ -6,7 +6,7 @@ #include "ttokendef.h" // todo refactor -static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { +__attribute__((unused)) static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { while (*input != 0 && *input++ != delim) { }; @@ -14,7 +14,7 @@ static FORCE_INLINE const char* skipSegments(const char* input, char delim, int3 return input; } -static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { +__attribute__((unused)) static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { size_t len = 0; while (*src != delimiter && *src != 0) { *dst++ = *src++; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5550ebf177..3fae17170a 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -753,12 +753,12 @@ typedef struct { typedef struct { int32_t numOfQueries; - SQueryDesc qdesc[]; + SQueryDesc *qdesc; } SQqueryList; typedef struct { int32_t numOfStreams; - SStreamDesc sdesc[]; + SStreamDesc *sdesc; } SStreamList; typedef struct { diff --git a/src/mnode/inc/mgmtGrant.h b/src/mnode/inc/mgmtGrant.h index c9e018d9b5..92b20532c4 100644 --- a/src/mnode/inc/mgmtGrant.h +++ b/src/mnode/inc/mgmtGrant.h @@ -14,7 +14,7 @@ */ #ifndef TDENGINE_MGMT_GRANT_H -#define TDENGINE_MGMT_GTANT_H +#define TDENGINE_MGMT_GRANT_H #ifdef __cplusplus "C" { diff --git a/src/query/inc/tlosertree.h b/src/query/inc/tlosertree.h index fb64fd2ee4..197d27a761 100644 --- a/src/query/inc/tlosertree.h +++ b/src/query/inc/tlosertree.h @@ -38,7 +38,7 @@ typedef struct SLoserTreeInfo { SLoserTreeNode *pNode; } SLoserTreeInfo; -uint8_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); +uint32_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); void tLoserTreeInit(SLoserTreeInfo *pTree); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index edb0e0aaed..f0aa13ee3d 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5707,7 +5707,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou SArray *pTableIdList) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { - goto _clean_memory; + goto _clean_pQInfo_memory; } SQuery *pQuery = calloc(1, sizeof(SQuery)); @@ -5836,6 +5836,7 @@ _clean_memory: tfree(pExprs); tfree(pGroupbyExpr); +_clean_pQInfo_memory: tfree(pQInfo); return NULL; @@ -6213,4 +6214,4 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c // vnodeDecRefCount(pObj->qhandle); // pObj->qhandle = NULL; // } -} \ No newline at end of file +} diff --git a/src/query/src/tlosertree.c b/src/query/src/tlosertree.c index 4fe68970b9..e6e45ed8d0 100644 --- a/src/query/src/tlosertree.c +++ b/src/query/src/tlosertree.c @@ -40,7 +40,7 @@ void tLoserTreeDisplay(SLoserTreeInfo* pTree) { printf("\n"); } -uint8_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) { +uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) { int32_t totalEntries = numOfEntries << 1; *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 3af7bfd2f2..0ef99b7244 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -18,7 +18,7 @@ #include "tskiplist.h" #include "tutil.h" -static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level +__attribute__ ((unused)) static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE for (int32_t i = 0; i < level; ++i) { pSkipList->state.nLevelNodeCnt[i]++; @@ -26,7 +26,7 @@ static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level #endif } -static FORCE_INLINE void removeNodeEachLevel(SSkipList *pSkipList, int32_t level) { +__attribute__ ((unused)) static FORCE_INLINE void removeNodeEachLevel(SSkipList *pSkipList, int32_t level) { #if SKIP_LIST_RECORD_PERFORMANCE for (int32_t i = 0; i < level; ++i) { pSkipList->state.nLevelNodeCnt[i]--; From 39ecb90a12a8b829cb3e9308a67d9e4902277a27 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 7 Apr 2020 14:43:25 +0800 Subject: [PATCH 02/48] [TD-116] split dnodeRead to dnodeVnode --- src/dnode/src/dnodeRead.c | 182 +++++++-------------------------- src/inc/taoserror.h | 1 + src/inc/vnode.h | 4 + src/vnode/main/inc/vnodeInt.h | 1 + src/vnode/main/src/vnodeMain.c | 1 + src/vnode/main/src/vnodeRead.c | 110 ++++++++++++++++++++ 6 files changed, 153 insertions(+), 146 deletions(-) create mode 100644 src/vnode/main/src/vnodeRead.c diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 7a57b0a2c0..d4365dae10 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -15,38 +15,25 @@ #define _DEFAULT_SOURCE #include "os.h" - #include "taoserror.h" #include "taosmsg.h" #include "tlog.h" #include "tqueue.h" #include "trpc.h" - #include "twal.h" #include "dnodeMgmt.h" #include "dnodeRead.h" -#include "queryExecutor.h" #include "vnode.h" typedef struct { - int32_t code; - int32_t count; - int32_t numOfVnodes; -} SRpcContext; - -typedef struct { - void *pCont; - int32_t contLen; - SRpcMsg rpcMsg; - SRpcContext *pRpcContext; // RPC message context + SRspRet rspRet; + void *pCont; + int32_t contLen; + SRpcMsg rpcMsg; } SReadMsg; static void *dnodeProcessReadQueue(void *param); -static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead); static void dnodeHandleIdleReadWorker(); -static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg); -static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg); -static void(*dnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(void *pVnode, SReadMsg *pNode); // module global variable static taos_qset readQset; @@ -55,14 +42,11 @@ static int32_t maxThreads; static int32_t minThreads; int32_t dnodeInitRead() { - dnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeProcessQueryMsg; - dnodeProcessReadMsgFp[TSDB_MSG_TYPE_RETRIEVE] = dnodeProcessRetrieveMsg; - readQset = taosOpenQset(); minThreads = 3; - maxThreads = tsNumOfCores*tsNumOfThreadsPerCore; - if (maxThreads <= minThreads*2) maxThreads = 2*minThreads; + maxThreads = tsNumOfCores * tsNumOfThreadsPerCore; + if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads; dPrint("dnode read is opened"); return 0; @@ -77,7 +61,6 @@ void dnodeRead(SRpcMsg *pMsg) { int32_t queuedMsgNum = 0; int32_t leftLen = pMsg->contLen; char *pCont = (char *) pMsg->pCont; - SRpcContext *pRpcContext = NULL; void *pVnode; dTrace("dnode %s msg incoming, thandle:%p", taosMsg[pMsg->msgType], pMsg->handle); @@ -105,7 +88,6 @@ void dnodeRead(SRpcMsg *pMsg) { pRead->rpcMsg = *pMsg; pRead->pCont = pCont; pRead->contLen = pHead->contLen; - pRead->pRpcContext = pRpcContext; taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); @@ -156,6 +138,34 @@ void dnodeFreeRqueue(void *rqueue) { // dynamically adjust the number of threads } +static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) { + SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); + pRead->rpcMsg = pMsg->rpcMsg; + pRead->pCont = qhandle; + pRead->contLen = 0; + pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + + taos_queue queue = vnodeGetRqueue(pVnode); + taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); +} + +void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { + if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; + if (code == TSDB_CODE_ACTION_NEED_REPROCESSED) { + dnodeContinueExecuteQuery(pVnode, pRead->rspRet.qhandle, pRead); + } + + SRpcMsg rpcRsp = { + .handle = pRead->rpcMsg.handle, + .pCont = pRead->rspRet.rsp, + .contLen = pRead->rspRet.len, + .code = pRead->rspRet.code, + }; + + rpcSendResponse(&rpcRsp); + rpcFreeCont(pRead->rpcMsg.pCont); +} + static void *dnodeProcessReadQueue(void *param) { taos_qset qset = (taos_qset)param; SReadMsg *pReadMsg; @@ -168,13 +178,8 @@ static void *dnodeProcessReadQueue(void *param) { continue; } - terrno = 0; - if (dnodeProcessReadMsgFp[pReadMsg->rpcMsg.msgType]) { - (*dnodeProcessReadMsgFp[pReadMsg->rpcMsg.msgType]) (pVnode, pReadMsg); - } else { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - + int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet); + dnodeSendRpcReadRsp(pVnode, pReadMsg, code); taosFreeQitem(pReadMsg); } @@ -193,118 +198,3 @@ static void dnodeHandleIdleReadWorker() { } } -UNUSED_FUNC -static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) { - SRpcContext *pRpcContext = pRead->pRpcContext; - int32_t code = 0; - - if (pRpcContext) { - if (terrno) { - if (pRpcContext->code == 0) pRpcContext->code = terrno; - } - - int32_t count = atomic_add_fetch_32(&pRpcContext->count, 1); - if (count < pRpcContext->numOfVnodes) { - // not over yet, multiple vnodes - return; - } - - // over, result can be merged now - code = pRpcContext->code; - } else { - code = terrno; - } - - //TODO: query handle is returned by dnodeProcessQueryMsg - if (0) { - SRpcMsg rsp; - rsp.handle = pRead->rpcMsg.handle; - rsp.code = code; - rsp.pCont = NULL; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pRead->rpcMsg.pCont); // free the received message -} - -static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) { - - SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); - pRead->rpcMsg = pMsg->rpcMsg; - pRead->pCont = qhandle; - pRead->contLen = 0; - pRead->pRpcContext = pMsg->pRpcContext; - pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - - taos_queue queue = vnodeGetRqueue(pVnode); - taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); -} - -static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { - SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont; - - SQInfo* pQInfo = NULL; - if (pMsg->contLen != 0) { - void* tsdb = vnodeGetTsdb(pVnode); - int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); - - SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); - pRsp->code = code; - pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - - SRpcMsg rpcRsp = { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = sizeof(SQueryTableRsp), - .code = code, - .msgType = 0 - }; - - rpcSendResponse(&rpcRsp); - dTrace("dnode query msg disposed, thandle:%p", pMsg->rpcMsg.handle); - } else { - pQInfo = pMsg->pCont; - } - - qTableQuery(pQInfo); // do execute query -} - -static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) { - SRetrieveTableMsg *pRetrieve = pMsg->pCont; - void *pQInfo = (void*) htobe64(pRetrieve->qhandle); - - dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); - int32_t contLen = 0; - - SRetrieveTableRsp *pRsp = NULL; - - int32_t code = qRetrieveQueryResultInfo(pQInfo); - if (code != TSDB_CODE_SUCCESS) { - contLen = sizeof(SRetrieveTableRsp); - - pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); - } else { - // todo check code and handle error in build result set - code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); - - if (qHasMoreResultsToRetrieve(pQInfo)) { - dnodeContinueExecuteQuery(pVnode, pQInfo, pMsg); - } else { // no further execution invoked, release the ref to vnode - qDestroyQueryInfo(pQInfo); -// dnodeProcessReadResult(pVnode, pMsg); - vnodeRelease(pVnode); - } - } - - SRpcMsg rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; - - rpcSendResponse(&rpcRsp); - dTrace("dnode retrieve msg disposed, thandle:%p", pMsg->rpcMsg.handle); -} diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 841c5b658f..0624af45c3 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -47,6 +47,7 @@ static STaosError errors[] = { // rpc TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, 0, 1, "action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_NEED_REPROCESSED, 0, 3, "action need to be reprocessed") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, 0, 4, "message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_ALREADY_PROCESSED, 0, 5, "message already processed") TAOS_DEFINE_ERROR(TSDB_CODE_REDIRECT, 0, 6, "redirect") diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 9861d1a2ff..3097343a48 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -22,7 +22,9 @@ extern "C" { typedef struct { int len; + int code; void *rsp; + void *qhandle; //used by query and retrieve msg } SRspRet; int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); @@ -42,6 +44,8 @@ void* vnodeGetTsdb(void *pVnode); int32_t vnodeProcessWrite(void *pVnode, int qtype, SWalHead *pHead, void *item); void vnodeBuildStatusMsg(void * param); +int32_t vnodeProcessRead(void *pVnode, int msgType, void *pCont, int32_t contLen, SRspRet *ret); + #ifdef __cplusplus } #endif diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 7e6caf6168..561b6ae61f 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -47,6 +47,7 @@ typedef struct { int vnodeWriteToQueue(void *param, SWalHead *pHead, int type); void vnodeInitWriteFp(void); +void vnodeInitReadFp(void); #ifdef __cplusplus } diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index d0352da3b1..3a70ec0d7d 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -41,6 +41,7 @@ static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; static void vnodeInit() { vnodeInitWriteFp(); + vnodeInitReadFp(); tsDnodeVnodesHash = taosInitIntHash(TSDB_MAX_VNODES, sizeof(SVnodeObj *), taosHashInt); if (tsDnodeVnodesHash == NULL) { diff --git a/src/vnode/main/src/vnodeRead.c b/src/vnode/main/src/vnodeRead.c new file mode 100644 index 0000000000..929a30fbcd --- /dev/null +++ b/src/vnode/main/src/vnodeRead.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "taoserror.h" +#include "tlog.h" +#include "tqueue.h" +#include "trpc.h" +#include "tsdb.h" +#include "twal.h" +#include "dataformat.h" +#include "vnode.h" +#include "vnodeInt.h" +#include "queryExecutor.h" + +static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, int32_t contLen, SRspRet *pRet); +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet); +static int32_t vnodeProcessRetrieveMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet); + +void vnodeInitReadFp(void) { + vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; + vnodeProcessReadMsgFp[TSDB_MSG_TYPE_RETRIEVE] = vnodeProcessRetrieveMsg; +} + +int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, SRspRet *ret) { + SVnodeObj *pVnode = (SVnodeObj *)param; + + if (vnodeProcessReadMsgFp[msgType] == NULL) + return TSDB_CODE_MSG_NOT_PROCESSED; + + if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) + return TSDB_CODE_NOT_ACTIVE_VNODE; + + return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret); +} + +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) { + SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont; + memset(pRet, 0, sizeof(SRspRet)); + + int32_t code = TSDB_CODE_SUCCESS; + + SQInfo* pQInfo = NULL; + if (contLen != 0) { + void* tsdb = vnodeGetTsdb(pVnode); + pRet->code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); + + SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); + pRsp->qhandle = htobe64((uint64_t) (pQInfo)); + pRsp->code = pRet->code; + + pRet->len = sizeof(SQueryTableRsp); + pRet->rsp = pRsp; + + dTrace("pVnode:%p vgId:%d QInfo:%p, dnode query msg disposed", pVnode, pVnode->vgId, pQInfo); + } else { + pQInfo = pCont; + code = TSDB_CODE_ACTION_IN_PROGRESS; + } + + qTableQuery(pQInfo); // do execute query + + return code; +} + +static int32_t vnodeProcessRetrieveMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) { + SRetrieveTableMsg *pRetrieve = pCont; + void *pQInfo = (void*) htobe64(pRetrieve->qhandle); + memset(pRet, 0, sizeof(SRspRet)); + + int32_t code = TSDB_CODE_SUCCESS; + + dTrace("pVnode:%p vgId:%d QInfo:%p, retrieve msg is received", pVnode, pVnode->vgId, pQInfo); + + pRet->code = qRetrieveQueryResultInfo(pQInfo); + if (pRet->code != TSDB_CODE_SUCCESS) { + //TODO + pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); + memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); + } else { + // todo check code and handle error in build result set + pRet->code = qDumpRetrieveResult(pQInfo, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len); + + if (qHasMoreResultsToRetrieve(pQInfo)) { + pRet->qhandle = pQInfo; + code = TSDB_CODE_ACTION_NEED_REPROCESSED; + } else { + // no further execution invoked, release the ref to vnode + qDestroyQueryInfo(pQInfo); + vnodeRelease(pVnode); + } + } + + dTrace("pVnode:%p vgId:%d QInfo:%p, retrieve msg is disposed", pVnode, pVnode->vgId, pQInfo); + return code; +} From 6f67da7aa2f3f4664ea6abd1ef473713162da5a1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 6 Apr 2020 08:39:39 +0800 Subject: [PATCH 03/48] add test execution in CI [TD-96] --- .travis.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74ed80da3f..552d408c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,28 @@ addons: branch_pattern: coverity_scan before_script: - - mkdir build - - cd build + - mkdir debug + - cd debug script: - cmake .. - cmake --build . + - |- + case $TRAVIS_OS_NAME in + linux) + cd ../tests/script + sudo ./test.sh >& out.txt + sync + sleep 2 + cat out.txt + grep success out.txt + total_success=`grep success out.txt | wc -l` + echo "Total $total_success success" + grep failed out.txt + total_failed=`grep failed out.txt | wc -l` + echo "Total $total_failed failed" + ;; + esac # # Build Matrix @@ -58,6 +74,7 @@ matrix: packages: - build-essential - cmake + - net-tools # - os: osx # addons: From 69fec0077ad6a252a04a536f672ea8aecec56372 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 7 Apr 2020 16:30:45 +0800 Subject: [PATCH 04/48] remove script from basic suite --- tests/script/basicSuite.sim | 13 ++----------- tests/script/general/table/basic1.sim | 4 ---- tests/script/general/table/basic2.sim | 3 --- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index d16e85323b..c95695fe00 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,17 +1,8 @@ ################################# -run general/user/basic1.sim -run general/show/dnodes.sim - -run general/db/basic1.sim -run general/db/basic2.sim -run general/db/basic3.sim -run general/db/basic4.sim -run general/db/basic5.sim - -run general/table/basic1.sim +#run general/table/basic1.sim run general/table/basic2.sim -run general/table/basic3.sim +#run general/table/basic3.sim ################################## diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index 9a05fb6d67..00e11483da 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -69,7 +69,3 @@ if $data21 != 3 then return -1 endi -sql drop database d1 - -system sh/exec.sh -n dnode1 -s stop -x SIGINT - diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index 18b98f4a3f..7701ca1c1f 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -64,6 +64,3 @@ if $data21 != 3 then return -1 endi -sql drop database d1 - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 3c77d2c399c2fc0eb42672ecdf994dc93d6753b0 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 09:36:20 +0800 Subject: [PATCH 05/48] add walCfg, syncCfg structure --- src/inc/tsync.h | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ src/inc/twal.h | 5 ++ 2 files changed, 124 insertions(+) create mode 100644 src/inc/tsync.h diff --git a/src/inc/tsync.h b/src/inc/tsync.h new file mode 100644 index 0000000000..16ce1de83c --- /dev/null +++ b/src/inc/tsync.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_H +#define TDENGINE_SYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_SYNC_MAX_REPLICA 5 + +typedef enum _TAOS_SYNC_ROLE { + TAOS_SYNC_ROLE_OFFLINE, + TAOS_SYNC_ROLE_UNSYNCED, + TAOS_SYNC_ROLE_SLAVE, + TAOS_SYNC_ROLE_MASTER, +} ESyncRole; + +typedef enum _TAOS_SYNC_STATUS { + TAOS_SYNC_STATUS_INIT, + TAOS_SYNC_STATUS_START, + TAOS_SYNC_STATUS_FILE, + TAOS_SYNC_STATUS_CACHE, +} ESyncStatus; + +typedef struct { + int8_t msgType; + int8_t reserved[3]; + int32_t len; + uint64_t version; + uint64_t cksum; + char cont[]; +} SWalHead; + +typedef struct { + uint32_t nodeId; // node ID assigned by TDengine + uint32_t nodeIp; // node IP address + char name[TSDB_FILENAME_LEN]; // external node name +} SNodeInfo; + +typedef struct { + int selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int role[TAOS_SYNC_MAX_REPLICA]; +} SNodesRole; + +typedef struct { + uint32_t arbitratorIp; // arbitrator IP address + int8_t quorum; // number of confirms required, >=1 + int8_t replica; // number of replications, >=1 + SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; +} SSyncCfg; + +typedef struct { + char label[20]; // for debug purpose + char path[128]; // path to the file + int8_t replica; // number of replications, >=1 + int8_t quorum; // number of confirms required, >=1 + int32_t vgId; // vgroup ID + void *ahandle; // handle provided by APP + uint64_t version; // initial version + uint32_t arbitratorIp; + SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; + + // if name is null, get the file from index or after, used by master + // if name is provided, get the named file at the specified index, used by unsynced node + // it returns the file magic number and size, if file not there, magic shall be 0. + uint32_t (*getFileInfo)(char *name, int *index, int *size); + + // get the wal file from index or after + // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file + int (*getWalInfo)(char *name, int *index); + + // when a forward pkt is received, call this to handle data + int (*writeToCache)(void *ahandle, SWalHead *, int type); + + // when forward is confirmed by peer, master call this API to notify app + void (*confirmForward)(void *ahandle, void *mhandle, int32_t code); + + // when role is changed, call this to notify app + void (*notifyRole)(void *ahandle, int8_t role); +} SSyncInfo; + +typedef void* tsync_h; + +tsync_h syncStart(SSyncInfo *); +void syncStop(tsync_h shandle); +int syncReconfig(tsync_h shandle, SSyncInfo *); +int syncForwardToPeer(tsync_h shandle, SWalHead *pHead, void *mhandle); +void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); +void syncRecover(tsync_h shandle); // recover from other nodes: +int syncGetNodesRole(tsync_h shandle, SNodesRole *); + +extern char *syncRole[]; + +extern int tsMaxSyncNum; +extern int tsSyncTcpThreads; +extern int tsMaxWatchFiles; +extern short tsSyncPort; +extern int tsMaxFwdInfo; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SYNC_H diff --git a/src/inc/twal.h b/src/inc/twal.h index bac5f87215..3648f5ae29 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -33,6 +33,11 @@ typedef struct { char cont[]; } SWalHead; +typedef struct { + int8_t commitLog; // commitLog + int8_t wals; // number of WAL files; +} SWalCfg; + typedef void* twal_h; // WAL HANDLE twal_h walOpen(char *path, int max, int level); From e1a3d7f295ff31fb5ea080f13609678bc3f56d6e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 09:40:20 +0800 Subject: [PATCH 06/48] remove SWalHead in tsync.h --- src/inc/tsync.h | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 16ce1de83c..39c116c9cb 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -36,34 +36,25 @@ typedef enum _TAOS_SYNC_STATUS { TAOS_SYNC_STATUS_CACHE, } ESyncStatus; -typedef struct { - int8_t msgType; - int8_t reserved[3]; - int32_t len; - uint64_t version; - uint64_t cksum; - char cont[]; -} SWalHead; - typedef struct { uint32_t nodeId; // node ID assigned by TDengine uint32_t nodeIp; // node IP address char name[TSDB_FILENAME_LEN]; // external node name } SNodeInfo; -typedef struct { - int selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int role[TAOS_SYNC_MAX_REPLICA]; -} SNodesRole; - typedef struct { uint32_t arbitratorIp; // arbitrator IP address int8_t quorum; // number of confirms required, >=1 int8_t replica; // number of replications, >=1 SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; } SSyncCfg; - + +typedef struct { + int selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int role[TAOS_SYNC_MAX_REPLICA]; +} SNodesRole; + typedef struct { char label[20]; // for debug purpose char path[128]; // path to the file From cfd3eb2f8a7a7a2fa9a7fc70af97d6623792a74b Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 09:54:52 +0800 Subject: [PATCH 07/48] [TD-114] --- src/inc/mnode.h | 5 +++-- src/mnode/inc/mgmtDnode.h | 11 +++++++++-- src/mnode/inc/mgmtVgroup.h | 6 ++++-- src/mnode/src/mgmtDnode.c | 25 ++++++++++++++----------- src/mnode/src/mgmtVgroup.c | 18 +++++++++++++----- src/util/inc/tstatus.h | 28 +++------------------------- src/util/src/tstatus.c | 8 -------- 7 files changed, 46 insertions(+), 55 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 3f66c46d7b..651ad0036e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -147,9 +147,10 @@ typedef struct _vg_obj { int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t lbIp; + int32_t lbDnodeId; int32_t lbTime; - int8_t lbStatus; + int8_t status; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; diff --git a/src/mnode/inc/mgmtDnode.h b/src/mnode/inc/mgmtDnode.h index f262cd3c5e..9c9ddea346 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/mnode/inc/mgmtDnode.h @@ -21,12 +21,19 @@ extern "C" { #endif #include "mnode.h" +enum _TSDB_DN_STATUS { + TSDB_DN_STATUS_OFFLINE, + TSDB_DN_STATUS_DROPING, + TSDB_DN_STATUS_BALANCING, + TSDB_DN_STATUS_READY +}; + int32_t mgmtInitDnodes(); void mgmtCleanUpDnodes(); int32_t mgmtGetDnodesNum(); void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); -void mgmtIncDnodeRef(SDnodeObj *pDnode); -void mgmtDecDnodeRef(SDnodeObj *pDnode); +void mgmtReleaseDnode(SDnodeObj *pDnode); +char* mgmtGetDnodeStatusStr(int32_t dnodeStatus); SDnodeObj* mgmtGetDnode(int32_t dnodeId); SDnodeObj* mgmtGetDnodeByIp(uint32_t ip); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index d0b1e0de97..b4d50a8318 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -27,10 +27,12 @@ extern "C" { int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); -void mgmtIncVgroupRef(SVgObj *pVgroup); -void mgmtDecVgroupRef(SVgObj *pVgroup); +void mgmtReleaseVgroup(SVgObj *pVgroup); void mgmtDropAllVgroups(SDbObj *pDropDb); +void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup); +void mgmtUpdateVgroup(SVgObj *pVgroup); + void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index ada0bce2e9..7019d8e20a 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -42,8 +42,7 @@ extern int32_t clusterInit(); extern void clusterCleanUp(); extern int32_t clusterGetDnodesNum(); extern void * clusterGetNextDnode(void *pNode, SDnodeObj **pDnode); -extern void clusterIncDnodeRef(SDnodeObj *pDnode); -extern void clusterDecDnodeRef(SDnodeObj *pDnode); +extern void clusterReleaseDnode(SDnodeObj *pDnode); extern SDnodeObj* clusterGetDnode(int32_t dnodeId); extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip); #ifndef _CLUSTER @@ -120,15 +119,9 @@ int32_t mgmtGetDnodesNum() { #endif } -void mgmtIncDnodeRef(SDnodeObj *pDnode) { +void mgmtReleaseDnode(SDnodeObj *pDnode) { #ifdef _CLUSTER - return clusterIncDnodeRef(pDnode); -#endif -} - -void mgmtDecDnodeRef(SDnodeObj *pDnode) { -#ifdef _CLUSTER - return clusterDecDnodeRef(pDnode); + return clusterReleaseDnode(pDnode); #endif } @@ -724,4 +717,14 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi pShow->numOfReads += numOfRows; return numOfRows; -} \ No newline at end of file +} + +char* mgmtGetDnodeStatusStr(int32_t dnodeStatus) { + switch (dnodeStatus) { + case TSDB_DN_STATUS_OFFLINE: return "offline"; + case TSDB_DN_STATUS_DROPING: return "dropping"; + case TSDB_DN_STATUS_BALANCING: return "balancing"; + case TSDB_DN_STATUS_READY: return "ready"; + default: return "undefined"; + } +} diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index bbf960203a..c16cae775c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -190,11 +190,7 @@ int32_t mgmtInitVgroups() { return 0; } -void mgmtIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); -} - -void mgmtDecVgroupRef(SVgObj *pVgroup) { +void mgmtReleaseVgroup(SVgObj *pVgroup) { return sdbDecRef(tsVgroupSdb, pVgroup); } @@ -202,6 +198,18 @@ SVgObj *mgmtGetVgroup(int32_t vgId) { return (SVgObj *)sdbGetRow(tsVgroupSdb, &vgId); } +void mgmtUpdateVgroup(SVgObj *pVgroup) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = tsVgUpdateSize + }; + + sdbUpdateRow(&oper); + mgmtSendCreateVgroupMsg(pVgroup, NULL); +} + SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } diff --git a/src/util/inc/tstatus.h b/src/util/inc/tstatus.h index 223ed1d52d..4fa392ca35 100644 --- a/src/util/inc/tstatus.h +++ b/src/util/inc/tstatus.h @@ -24,15 +24,6 @@ extern "C" { #include #include "taoserror.h" -enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY = TSDB_CODE_SUCCESS, - TSDB_VG_STATUS_IN_PROGRESS = 1, //TSDB_CODE_ACTION_IN_PROGRESS, - TSDB_VG_STATUS_NO_DISK_PERMISSIONS = 73,//TSDB_CODE_NO_DISK_PERMISSIONS, - TSDB_VG_STATUS_SERVER_NO_PACE = 110, //TSDB_CODE_SERV_NO_DISKSPACE, - TSDB_VG_STATUS_SERV_OUT_OF_MEMORY = 69, //TSDB_CODE_SERV_OUT_OF_MEMORY, - TSDB_VG_STATUS_INIT_FAILED = 74, //TSDB_CODE_VG_INIT_FAILED, - TSDB_VG_STATUS_FULL = 48, //TSDB_CODE_NO_ENOUGH_DNODES, -}; enum _TSDB_DB_STATUS { TSDB_DB_STATUS_READY, @@ -75,21 +66,9 @@ enum _TSDB_MN_ROLE { TSDB_MN_ROLE_MASTER }; -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_READY -}; - -enum _TSDB_DN_LB_STATUS { - TSDB_DN_LB_STATUS_BALANCED, - TSDB_DN_LB_STATUS_BALANCING, - TSDB_DN_LB_STATUS_OFFLINE_REMOVING, - TSDB_DN_LB_STATE_SHELL_REMOVING -}; - -enum _TSDB_VG_LB_STATUS { - TSDB_VG_LB_STATUS_READY, - TSDB_VG_LB_STATUS_UPDATE +enum _TSDB_VG_STATUS { + TSDB_VG_STATUS_READY, + TSDB_VG_STATUS_UPDATE }; enum _TSDB_VN_STREAM_STATUS { @@ -111,7 +90,6 @@ char* taosGetDbStatusStr(int32_t dbStatus); char* taosGetVnodeStatusStr(int32_t vnodeStatus); char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus); char* taosGetVnodeDropStatusStr(int32_t dropping); -char* taosGetDnodeStatusStr(int32_t dnodeStatus); char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus); char* taosGetVgroupLbStatusStr(int32_t vglbStatus); char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus); diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index e8534de30b..cfcbea0ca1 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -69,14 +69,6 @@ char* taosGetVnodeDropStatusStr(int32_t dropping) { } } -char* taosGetDnodeStatusStr(int32_t dnodeStatus) { - switch (dnodeStatus) { - case TSDB_DN_STATUS_OFFLINE: return "offline"; - case TSDB_DN_STATUS_READY: return "ready"; - default: return "undefined"; - } -} - char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus) { switch (dnodeBalanceStatus) { case TSDB_DN_LB_STATUS_BALANCED: return "balanced"; From c536cdd39c952dc9454585533b3cc3095b09a1d9 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 12:24:32 +0800 Subject: [PATCH 08/48] [TD-114] add wal config --- src/inc/mnode.h | 8 ++-- src/mnode/inc/mgmtTable.h | 2 +- src/mnode/src/mgmtTable.c | 12 ++--- src/mnode/src/mgmtVgroup.c | 2 +- src/vnode/main/inc/vnodeInt.h | 8 +++- src/vnode/main/src/vnodeMain.c | 87 +++++++++++++++++++++++----------- 6 files changed, 77 insertions(+), 42 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 3f66c46d7b..7c59a5a4b6 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -105,10 +105,10 @@ typedef struct { typedef struct { char tableId[TSDB_TABLE_ID_LEN + 1]; int8_t type; -} STableInfo; +} STableObj; typedef struct SSuperTableObj { - STableInfo info; + STableObj info; uint64_t uid; int64_t createdTime; int32_t sversion; @@ -123,7 +123,7 @@ typedef struct SSuperTableObj { } SSuperTableObj; typedef struct { - STableInfo info; + STableObj info; uint64_t uid; int64_t createdTime; int32_t sversion; //used by normal table @@ -254,7 +254,7 @@ typedef struct { SUserObj *pUser; SDbObj *pDb; SVgObj *pVgroup; - STableInfo *pTable; + STableObj *pTable; } SQueuedMsg; int32_t mgmtInitSystem(); diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h index ddbbfb4a70..4d3e0f6b43 100644 --- a/src/mnode/inc/mgmtTable.h +++ b/src/mnode/inc/mgmtTable.h @@ -27,7 +27,7 @@ extern "C" { int32_t mgmtInitTables(); void mgmtCleanUpTables(); -STableInfo* mgmtGetTable(char* tableId); +STableObj* mgmtGetTable(char* tableId); void mgmtIncTableRef(void *pTable); void mgmtDecTableRef(void *pTable); void mgmtDropAllChildTables(SDbObj *pDropDb); diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index c4996fc4bb..d6d7a6afc0 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -486,8 +486,8 @@ static void *mgmtGetSuperTable(char *tableId) { return sdbGetRow(tsSuperTableSdb, tableId); } -STableInfo *mgmtGetTable(char *tableId) { - STableInfo *tableInfo = sdbGetRow(tsSuperTableSdb, tableId); +STableObj *mgmtGetTable(char *tableId) { + STableObj *tableInfo = sdbGetRow(tsSuperTableSdb, tableId); if (tableInfo != NULL) { return tableInfo; } @@ -501,7 +501,7 @@ STableInfo *mgmtGetTable(char *tableId) { } void mgmtIncTableRef(void *p1) { - STableInfo *pTable = (STableInfo *)p1; + STableObj *pTable = (STableObj *)p1; if (pTable->type == TSDB_SUPER_TABLE) { sdbIncRef(tsSuperTableSdb, pTable); } else { @@ -512,7 +512,7 @@ void mgmtIncTableRef(void *p1) { void mgmtDecTableRef(void *p1) { if (p1 == NULL) return; - STableInfo *pTable = (STableInfo *)p1; + STableObj *pTable = (STableObj *)p1; if (pTable->type == TSDB_SUPER_TABLE) { sdbDecRef(tsSuperTableSdb, pTable); } else { @@ -1302,7 +1302,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pTable = (STableInfo *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); if (pMsg->pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); return; @@ -1641,7 +1641,7 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_ } SChildTableObj *pTable = pVgroup->tableList[sid]; - mgmtIncTableRef((STableInfo *)pTable); + mgmtIncTableRef((STableObj *)pTable); mgmtDecVgroupRef(pVgroup); return pTable; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index bbf960203a..8ca23dc98a 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -293,7 +293,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t maxReplica = 0; SVgObj *pVgroup = NULL; - STableInfo *pTable = NULL; + STableObj *pTable = NULL; if (pShow->payloadLen > 0 ) { pTable = mgmtGetTable(pShow->payload); if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) { diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 561b6ae61f..4d078869c4 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -20,6 +20,9 @@ extern "C" { #endif +#include "tsync.h" +#include "twal.h" + typedef enum _VN_STATUS { VN_STATUS_INIT, VN_STATUS_CREATING, @@ -41,8 +44,9 @@ typedef struct { void *sync; void *events; void *cq; // continuous query - int32_t replicas; - SVnodeDesc vpeers[TSDB_MAX_MPEERS]; + STsdbCfg tsdbCfg; + SSyncCfg syncCfg; + SWalCfg walCfg; } SVnodeObj; int vnodeWriteToQueue(void *param, SWalHead *pHead, int type); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 3a70ec0d7d..30a630b82d 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -60,16 +60,6 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_SUCCESS; } - STsdbCfg tsdbCfg = {0}; - tsdbCfg.precision = pVnodeCfg->cfg.precision; - tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; - tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions; - tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; - tsdbCfg.minRowsPerFileBlock = -1; - tsdbCfg.maxRowsPerFileBlock = -1; - tsdbCfg.keep = -1; - tsdbCfg.maxCacheSize = -1; - char rootDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId); if (mkdir(rootDir, 0755) != 0) { @@ -89,6 +79,16 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { return code; } + STsdbCfg tsdbCfg = {0}; + tsdbCfg.precision = pVnodeCfg->cfg.precision; + tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; + tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions; + tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; + tsdbCfg.minRowsPerFileBlock = -1; + tsdbCfg.maxRowsPerFileBlock = -1; + tsdbCfg.keep = -1; + tsdbCfg.maxCacheSize = -1; + char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); @@ -140,7 +140,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->rqueue = dnodeAllocateRqueue(pVnode); sprintf(temp, "%s/wal", rootDir); - pVnode->wal = walOpen(temp, 3, tsCommitLog); + pVnode->wal = walOpen(temp, pVnode->walCfg.wals, pVnode->walCfg.commitLog); pVnode->sync = NULL; pVnode->events = NULL; pVnode->cq = NULL; @@ -293,9 +293,13 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { FILE *fp = fopen(cfgFile, "w"); if (!fp) return errno; - fprintf(fp, "replicas %d\n", pVnodeCfg->cfg.replications); + fprintf(fp, "commitLog %d\n", pVnodeCfg->cfg.commitLog); + fprintf(fp, "wals %d\n", 3); + fprintf(fp, "arbitratorIp %d\n", pVnodeCfg->vpeerDesc[0].ip); + fprintf(fp, "quorum %d\n", 1); + fprintf(fp, "replica %d\n", pVnodeCfg->cfg.replications); for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { - fprintf(fp, "index%d dnode %d ip %u\n", i, pVnodeCfg->vpeerDesc[i].dnodeId, pVnodeCfg->vpeerDesc[i].ip); + fprintf(fp, "index%d nodeId %d nodeIp %u name n%d\n", i, pVnodeCfg->vpeerDesc[i].dnodeId, pVnodeCfg->vpeerDesc[i].ip, pVnodeCfg->vpeerDesc[i].dnodeId); } fclose(fp); @@ -306,33 +310,60 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { // TODO: this is a simple implement static int32_t vnodeReadCfg(SVnodeObj *pVnode) { + char option[3][16] = {0}; char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(cfgFile, "r"); if (!fp) return errno; - char option[3][32] = {0}; - int32_t replicas = 0; - int32_t num = fscanf(fp, "%s %d", option[0], &replicas); + int32_t commitLog = 0; + int32_t num = fscanf(fp, "%s %d", option[0], &commitLog); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; - if (strcmp(option[0], "replicas") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (replicas == 0) return TSDB_CODE_INVALID_FILE_FORMAT; - pVnode->replicas = replicas; + if (strcmp(option[0], "commitLog") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (commitLog == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + pVnode->walCfg.commitLog = (int8_t)commitLog; - for (int32_t i = 0; i < replicas; ++i) { + int32_t wals = 0; + num = fscanf(fp, "%s %d", option[0], &wals); + if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (wals == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + pVnode->walCfg.wals = (int8_t)wals; + + int32_t arbitratorIp = 0; + num = fscanf(fp, "%s %u", option[0], &arbitratorIp); + if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (arbitratorIp == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + pVnode->syncCfg.arbitratorIp = arbitratorIp; + + int32_t quorum = 0; + num = fscanf(fp, "%s %d", option[0], &quorum); + if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[0], "quorum") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (quorum == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + pVnode->syncCfg.quorum = (int8_t)quorum; + + int32_t replica = 0; + num = fscanf(fp, "%s %d", option[0], &replica); + if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[0], "replica") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (replica == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + pVnode->syncCfg.replica = (int8_t)replica; + + for (int32_t i = 0; i < replica; ++i) { int32_t dnodeId = 0; uint32_t dnodeIp = 0; - num = fscanf(fp, "%s %s %d %s %u", option[0], option[1], &dnodeId, option[2], &dnodeIp); - if (num != 5) return TSDB_CODE_INVALID_FILE_FORMAT; - if (strcmp(option[1], "dnode") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (strcmp(option[2], "ip") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + num = fscanf(fp, "%s %s %d %s %u %s %s", option[0], option[1], &dnodeId, option[2], &dnodeIp, option[3], pVnode->syncCfg.nodeInfo[i].name); + if (num != 7) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[1], "nodeId") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[2], "nodeIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (strcmp(option[3], "name") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (dnodeId == 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (dnodeIp == 0) return TSDB_CODE_INVALID_FILE_FORMAT; - - pVnode->vpeers[i].dnodeId = dnodeId; - pVnode->vpeers[i].ip = dnodeIp; - pVnode->vpeers[i].vgId = pVnode->vgId; + pVnode->syncCfg.nodeInfo[i].nodeId = dnodeId; + pVnode->syncCfg.nodeInfo[i].nodeIp = dnodeIp; } fclose(fp); From 3c8d8bc42894618a3c98bd2a744a401645fa3eb6 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 12:42:02 +0800 Subject: [PATCH 09/48] fix read file error --- src/vnode/main/src/vnodeMain.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 30a630b82d..1211828a47 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -317,51 +317,51 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { FILE *fp = fopen(cfgFile, "r"); if (!fp) return errno; - int32_t commitLog = 0; + int32_t commitLog = -1; int32_t num = fscanf(fp, "%s %d", option[0], &commitLog); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "commitLog") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (commitLog == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (commitLog == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->walCfg.commitLog = (int8_t)commitLog; - int32_t wals = 0; + int32_t wals = -1; num = fscanf(fp, "%s %d", option[0], &wals); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (wals == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->walCfg.wals = (int8_t)wals; - int32_t arbitratorIp = 0; + int32_t arbitratorIp = -1; num = fscanf(fp, "%s %u", option[0], &arbitratorIp); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (arbitratorIp == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->syncCfg.arbitratorIp = arbitratorIp; - int32_t quorum = 0; + int32_t quorum = -1; num = fscanf(fp, "%s %d", option[0], &quorum); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "quorum") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (quorum == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (quorum == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->syncCfg.quorum = (int8_t)quorum; - int32_t replica = 0; + int32_t replica = -1; num = fscanf(fp, "%s %d", option[0], &replica); if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "replica") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (replica == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (replica == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->syncCfg.replica = (int8_t)replica; for (int32_t i = 0; i < replica; ++i) { - int32_t dnodeId = 0; - uint32_t dnodeIp = 0; + int32_t dnodeId = -1; + uint32_t dnodeIp = -1; num = fscanf(fp, "%s %s %d %s %u %s %s", option[0], option[1], &dnodeId, option[2], &dnodeIp, option[3], pVnode->syncCfg.nodeInfo[i].name); if (num != 7) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[1], "nodeId") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[2], "nodeIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[3], "name") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (dnodeId == 0) return TSDB_CODE_INVALID_FILE_FORMAT; - if (dnodeIp == 0) return TSDB_CODE_INVALID_FILE_FORMAT; + if (dnodeId == -1) return TSDB_CODE_INVALID_FILE_FORMAT; + if (dnodeIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->syncCfg.nodeInfo[i].nodeId = dnodeId; pVnode->syncCfg.nodeInfo[i].nodeIp = dnodeIp; } @@ -370,4 +370,4 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { dTrace("pVnode:%p vgId:%d, read vnode cfg successed", pVnode, pVnode->vgId); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} From 871322a485d11887485c4e55e35672c5ee6012f6 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 13:46:36 +0800 Subject: [PATCH 10/48] script --- tests/script/general/table/basic1.sim | 6 +++--- tests/script/general/table/basic3.sim | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index 00e11483da..a8dfb0f21a 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -43,9 +43,9 @@ print $data10 $data11 $data22 print $data20 $data11 $data22 print =============== insert data -sql insert into d1.n1 values(now, 1) -sql insert into d1.n1 values(now, 2) -sql insert into d1.n1 values(now, 3) +sql insert into d1.n1 values(now+1s, 1) +sql insert into d1.n1 values(now+2s, 2) +sql insert into d1.n1 values(now+3s, 3) print =============== query data sql select * from d1.n1 diff --git a/tests/script/general/table/basic3.sim b/tests/script/general/table/basic3.sim index 599d011653..6558617384 100644 --- a/tests/script/general/table/basic3.sim +++ b/tests/script/general/table/basic3.sim @@ -43,9 +43,9 @@ print $data10 $data11 $data22 print $data20 $data11 $data22 print =============== insert data -sql insert into db.n1 values(now, 1) -sql insert into db.n1 values(now, 2) -sql insert into db.n1 values(now, 3) +sql insert into db.n1 values(now+1s, 1) +sql insert into db.n1 values(now+2s, 2) +sql insert into db.n1 values(now+3s, 3) print =============== query data sql select * from db.n1 From ead50301fdfa775029611060ca972a8d84ef3347 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 6 Apr 2020 08:39:39 +0800 Subject: [PATCH 11/48] add test execution in CI [TD-96] --- .travis.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74ed80da3f..552d408c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,28 @@ addons: branch_pattern: coverity_scan before_script: - - mkdir build - - cd build + - mkdir debug + - cd debug script: - cmake .. - cmake --build . + - |- + case $TRAVIS_OS_NAME in + linux) + cd ../tests/script + sudo ./test.sh >& out.txt + sync + sleep 2 + cat out.txt + grep success out.txt + total_success=`grep success out.txt | wc -l` + echo "Total $total_success success" + grep failed out.txt + total_failed=`grep failed out.txt | wc -l` + echo "Total $total_failed failed" + ;; + esac # # Build Matrix @@ -58,6 +74,7 @@ matrix: packages: - build-essential - cmake + - net-tools # - os: osx # addons: From c84cd03f490ad2c16a7479017447a1b620affa03 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 15:27:40 +0800 Subject: [PATCH 12/48] fix memory error --- src/vnode/main/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 1211828a47..9512cf4c6c 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -310,7 +310,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { // TODO: this is a simple implement static int32_t vnodeReadCfg(SVnodeObj *pVnode) { - char option[3][16] = {0}; + char option[5][16] = {0}; char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnode->vgId); From 4f6cb328b2274ebf5efe8f10a3ef7daaf495603a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 16:59:28 +0800 Subject: [PATCH 13/48] change the API to fit the changes in syncCfg --- src/inc/tsync.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 39c116c9cb..555e0503da 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -56,16 +56,13 @@ typedef struct { } SNodesRole; typedef struct { - char label[20]; // for debug purpose - char path[128]; // path to the file - int8_t replica; // number of replications, >=1 - int8_t quorum; // number of confirms required, >=1 int32_t vgId; // vgroup ID - void *ahandle; // handle provided by APP uint64_t version; // initial version - uint32_t arbitratorIp; - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; + SSyncCfg syncCfg; // configuration from mgmt + char path[128]; // path to the file + void *ahandle; // handle provided by APP + // if name is null, get the file from index or after, used by master // if name is provided, get the named file at the specified index, used by unsynced node // it returns the file magic number and size, if file not there, magic shall be 0. @@ -76,7 +73,7 @@ typedef struct { int (*getWalInfo)(char *name, int *index); // when a forward pkt is received, call this to handle data - int (*writeToCache)(void *ahandle, SWalHead *, int type); + int (*writeToCache)(void *ahandle, void *pHead, int type); // when forward is confirmed by peer, master call this API to notify app void (*confirmForward)(void *ahandle, void *mhandle, int32_t code); @@ -89,8 +86,8 @@ typedef void* tsync_h; tsync_h syncStart(SSyncInfo *); void syncStop(tsync_h shandle); -int syncReconfig(tsync_h shandle, SSyncInfo *); -int syncForwardToPeer(tsync_h shandle, SWalHead *pHead, void *mhandle); +int syncReconfig(tsync_h shandle, SSyncCfg *); +int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle); void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); void syncRecover(tsync_h shandle); // recover from other nodes: int syncGetNodesRole(tsync_h shandle, SNodesRole *); From 45b90bf90994ebaeb10be8c413378903a205a910 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 17:04:16 +0800 Subject: [PATCH 14/48] add testcase execution in CI [TD-96] --- .travis.yml | 24 +++++++++++++++++++++--- tests/script/basicSuite.sim | 4 ++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74ed80da3f..5cff3bc72b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,29 @@ addons: branch_pattern: coverity_scan before_script: - - mkdir build - - cd build + - mkdir debug + - cd debug script: - cmake .. - - cmake --build . + - cmake --build . || exit $? + - |- + case $TRAVIS_OS_NAME in + linux) + cd ../tests/script + sudo ./test.sh 2>&1 | tee out.txt + cat out.txt + grep success out.txt + total_success=`grep success out.txt | wc -l` + echo "Total $total_success success" + grep failed out.txt + total_failed=`grep failed out.txt | wc -l` + echo "Total $total_failed failed" + if [ "$total_failed" -ne "0" ]; then + exit $total_failed + fi + ;; + esac # # Build Matrix @@ -58,6 +75,7 @@ matrix: packages: - build-essential - cmake + - net-tools # - os: osx # addons: diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index c95695fe00..d5892d8682 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,8 +1,8 @@ ################################# -#run general/table/basic1.sim -run general/table/basic2.sim +run general/table/basic1.sim +#run general/table/basic2.sim #run general/table/basic3.sim ################################## From 57f96f1b21221a725fcdc774ba6311656e3ce536 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 6 Apr 2020 08:39:39 +0800 Subject: [PATCH 15/48] add test execution in CI [TD-96] --- .travis.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74ed80da3f..552d408c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,28 @@ addons: branch_pattern: coverity_scan before_script: - - mkdir build - - cd build + - mkdir debug + - cd debug script: - cmake .. - cmake --build . + - |- + case $TRAVIS_OS_NAME in + linux) + cd ../tests/script + sudo ./test.sh >& out.txt + sync + sleep 2 + cat out.txt + grep success out.txt + total_success=`grep success out.txt | wc -l` + echo "Total $total_success success" + grep failed out.txt + total_failed=`grep failed out.txt | wc -l` + echo "Total $total_failed failed" + ;; + esac # # Build Matrix @@ -58,6 +74,7 @@ matrix: packages: - build-essential - cmake + - net-tools # - os: osx # addons: From 54bbfe74c7717d2b872d7d380302e6ecf64f0e03 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 18:52:07 +0800 Subject: [PATCH 16/48] add sleep while connect taosd --- tests/tsim/src/simExe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index d844f3b786..912f10ba2a 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -576,6 +576,7 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); void *taos = NULL; + taosMsleep(2000); for (int attempt = 0; attempt < 10; ++attempt) { taos = taos_connect(NULL, user, pass, NULL, tsMnodeShellPort); if (taos == NULL) { From a7aa4e1e5f5199f0ce18f52417c1f0f7e40a1fb8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 19:19:24 +0800 Subject: [PATCH 17/48] fix missing mutex-unlock call in few files. [TD-121] --- src/client/src/tscServer.c | 1 + src/client/src/tscSubquery.c | 1 + src/mnode/src/mgmtSdb.c | 1 + 3 files changed, 3 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1e7ad937ac..66d5e17747 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1787,6 +1787,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { size = tscEstimateHeartBeatMsgLength(pSql); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); return -1; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d616a8bffa..3660c822d7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1381,6 +1381,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } else { // all data has been retrieved to client tscAllDataRetrievedFromDnode(trsupport, pSql); } + pthread_mutex_unlock(&trsupport->queryMutex); } static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 06cb9dca5c..de68baf40f 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -695,6 +695,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)calloc(1, total_size); if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); sdbError("table:%s, failed to allocate row head memory", pTable->tableName); return -1; } From 8568f4c4610df30b8cc68e04ebc98ba61a71fbf4 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 20:02:29 +0800 Subject: [PATCH 18/48] [TD-82] refactor balance module --- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeMgmt.c | 1 - src/inc/mnode.h | 20 +- src/{mnode/inc/mgmtAcct.h => inc/taccount.h} | 29 +-- src/{mnode/inc/mgmtBalance.h => inc/tadmin.h} | 18 +- src/inc/taosmsg.h | 2 +- src/{mnode/inc/mgmtDnode.h => inc/tbalance.h} | 31 ++- src/inc/tcluster.h | 54 +++++ src/{mnode/inc/mgmtGrant.h => inc/tgrant.h} | 4 +- src/inc/vnode.h | 10 +- src/mnode/inc/mgmtDb.h | 7 +- src/mnode/inc/mgmtMnode.h | 13 ++ src/mnode/inc/mgmtUser.h | 3 +- src/mnode/inc/mgmtVgroup.h | 5 + src/mnode/src/mgmtAcct.c | 18 +- src/mnode/src/mgmtBalance.c | 59 ++--- src/mnode/src/mgmtDClient.c | 7 +- src/mnode/src/mgmtDServer.c | 5 +- src/mnode/src/mgmtDb.c | 39 ++-- src/mnode/src/mgmtDnode.c | 210 +++++++----------- src/mnode/src/mgmtGrant.c | 2 +- src/mnode/src/mgmtMain.c | 16 +- src/mnode/src/mgmtMnode.c | 22 +- src/mnode/src/mgmtProfile.c | 12 +- src/mnode/src/mgmtShell.c | 43 +++- src/mnode/src/mgmtTable.c | 58 +++-- src/mnode/src/mgmtUser.c | 28 +-- src/mnode/src/mgmtVgroup.c | 48 ++-- src/query/src/queryExecutor.c | 1 - src/util/inc/tstatus.h | 105 --------- src/util/src/tstatus.c | 149 ------------- src/vnode/main/inc/vnodeInt.h | 10 +- src/vnode/main/src/vnodeMain.c | 13 +- src/vnode/main/src/vnodeRead.c | 2 +- src/vnode/main/src/vnodeWrite.c | 9 +- 35 files changed, 423 insertions(+), 632 deletions(-) rename src/{mnode/inc/mgmtAcct.h => inc/taccount.h} (56%) rename src/{mnode/inc/mgmtBalance.h => inc/tadmin.h} (72%) rename src/{mnode/inc/mgmtDnode.h => inc/tbalance.h} (57%) create mode 100644 src/inc/tcluster.h rename src/{mnode/inc/mgmtGrant.h => inc/tgrant.h} (95%) delete mode 100644 src/util/inc/tstatus.h delete mode 100644 src/util/src/tstatus.c diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index e2de9bf586..d95a344058 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -28,7 +28,7 @@ #include "dnodeRead.h" #include "dnodeShell.h" #include "dnodeWrite.h" -#include "mgmtGrant.h" +#include "tgrant.h" static int32_t dnodeInitSystem(); static int32_t dnodeInitStorage(); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5f1e7a7a94..fa3ad946e3 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -20,7 +20,6 @@ #include "taosmsg.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 70c7f8003d..903b172068 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "os.h" - #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" @@ -40,7 +39,8 @@ extern "C" { struct _vg_obj; struct _db_obj; -struct _acctObj; +struct _acct_obj; +struct _user_obj; typedef struct { int32_t mnodeId; @@ -65,7 +65,7 @@ typedef struct { void *pSync; } SMnodeObj; -typedef struct { +typedef struct _dnode_obj { int32_t dnodeId; uint32_t privateIp; uint32_t publicIp; @@ -79,16 +79,15 @@ typedef struct { uint16_t slot; uint16_t numOfCores; // from dnode status msg int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode - int8_t lbStatus; // set in balance function - float lbScore; // calc in balance function + int8_t status; // set in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; int8_t reserved[15]; int8_t updateEnd[1]; int32_t refCount; SVnodeLoad vload[TSDB_MAX_VNODES]; - int32_t status; uint32_t lastReboot; // time stamp for last reboot + float score; // calc in balance function float diskAvailable; // from dnode status msg int16_t diskAvgUsage; // calc from sys.disk int16_t cpuAvgUsage; // calc from sys.cpu @@ -150,7 +149,6 @@ typedef struct _vg_obj { int32_t lbDnodeId; int32_t lbTime; int8_t status; - int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; @@ -163,7 +161,7 @@ typedef struct _vg_obj { typedef struct _db_obj { char name[TSDB_DB_NAME_LEN + 1]; - int8_t dirty; + int8_t status; int64_t createdTime; SDbCfg cfg; int8_t reserved[15]; @@ -174,7 +172,7 @@ typedef struct _db_obj { int32_t numOfSuperTables; SVgObj *pHead; SVgObj *pTail; - struct _acctObj *pAcct; + struct _acct_obj *pAcct; } SDbObj; typedef struct _user_obj { @@ -187,7 +185,7 @@ typedef struct _user_obj { int8_t reserved[13]; int8_t updateEnd[1]; int32_t refCount; - struct _acctObj * pAcct; + struct _acct_obj * pAcct; SQqueryList * pQList; // query list SStreamList * pSList; // stream list } SUserObj; @@ -210,7 +208,7 @@ typedef struct { int8_t accessState; // Checked by mgmt heartbeat message } SAcctInfo; -typedef struct _acctObj { +typedef struct _acct_obj { char user[TSDB_USER_LEN + 1]; char pass[TSDB_KEY_LEN + 1]; SAcctCfg cfg; diff --git a/src/mnode/inc/mgmtAcct.h b/src/inc/taccount.h similarity index 56% rename from src/mnode/inc/mgmtAcct.h rename to src/inc/taccount.h index 1f8dc5c74a..18a974a574 100644 --- a/src/mnode/inc/mgmtAcct.h +++ b/src/inc/taccount.h @@ -13,31 +13,34 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_ACCT_H -#define TDENGINE_MGMT_ACCT_H +#ifndef TDENGINE_ACCT_H +#define TDENGINE_ACCT_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" +struct _acct_obj; +struct _user_obj; +struct _db_obj; + typedef enum { TSDB_ACCT_USER, TSDB_ACCT_DB, TSDB_ACCT_TABLE } EAcctGrantType; -int32_t acctInit(); -void acctCleanUp(); -SAcctObj *acctGetAcct(char *acctName); -void acctIncRef(SAcctObj *pAcct); -void acctDecRef(SAcctObj *pAcct); -int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type); +int32_t acctInit(); +void acctCleanUp(); +void *acctGetAcct(char *acctName); +void acctIncRef(struct _acct_obj *pAcct); +void acctReleaseAcct(struct _acct_obj *pAcct); +int32_t acctCheck(struct _acct_obj *pAcct, EAcctGrantType type); -void acctAddDb(SAcctObj *pAcct, SDbObj *pDb); -void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb); -void acctAddUser(SAcctObj *pAcct, SUserObj *pUser); -void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser); +void acctAddDb(struct _acct_obj *pAcct, struct _db_obj *pDb); +void acctRemoveDb(struct _acct_obj *pAcct, struct _db_obj *pDb); +void acctAddUser(struct _acct_obj *pAcct, struct _user_obj *pUser); +void acctRemoveUser(struct _acct_obj *pAcct, struct _user_obj *pUser); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtBalance.h b/src/inc/tadmin.h similarity index 72% rename from src/mnode/inc/mgmtBalance.h rename to src/inc/tadmin.h index 05f2ed94a7..4a883965f4 100644 --- a/src/mnode/inc/mgmtBalance.h +++ b/src/inc/tadmin.h @@ -13,18 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_BALANCE_H -#define TDENGINE_MGMT_BALANCE_H +#ifndef TDENGINE_ADMIN_H +#define TDENGINE_ADMIN_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" -int32_t mgmtInitBalance(); -void mgmtCleanupBalance(); -void mgmtBalanceNotify() ; -int32_t mgmtAllocVnodes(SVgObj *pVgroup); +#include +#include + +void adminInit(); + +struct _http_server_obj_; + +extern void (*adminInitHandleFp)(struct _http_server_obj_* pServer); +extern void (*opInitHandleFp)(struct _http_server_obj_* pServer); #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 3fae17170a..5f07d6ec99 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -515,7 +515,7 @@ typedef struct { int64_t compStorage; int64_t pointsWritten; uint8_t status; - uint8_t syncStatus; + uint8_t role; uint8_t accessState; uint8_t reserved[5]; } SVnodeLoad; diff --git a/src/mnode/inc/mgmtDnode.h b/src/inc/tbalance.h similarity index 57% rename from src/mnode/inc/mgmtDnode.h rename to src/inc/tbalance.h index 9c9ddea346..8cf8cb9fb9 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/inc/tbalance.h @@ -13,29 +13,26 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_DNODE_H -#define TDENGINE_MGMT_DNODE_H +#ifndef TDENGINE_BALANCE_H +#define TDENGINE_BALANCE_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_DROPING, - TSDB_DN_STATUS_BALANCING, - TSDB_DN_STATUS_READY -}; +#include +#include +#include -int32_t mgmtInitDnodes(); -void mgmtCleanUpDnodes(); -int32_t mgmtGetDnodesNum(); -void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); -void mgmtReleaseDnode(SDnodeObj *pDnode); -char* mgmtGetDnodeStatusStr(int32_t dnodeStatus); -SDnodeObj* mgmtGetDnode(int32_t dnodeId); -SDnodeObj* mgmtGetDnodeByIp(uint32_t ip); +struct _db_obj; +struct _vg_obj; +struct _dnode_obj; + +int32_t balanceInit(); +void balanceCleanUp(); +void balanceNotify(); +int32_t balanceAllocVnodes(struct _vg_obj *pVgroup); +int32_t balanceDropDnode(struct _dnode_obj *pDnode); #ifdef __cplusplus } diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h new file mode 100644 index 0000000000..60436ee743 --- /dev/null +++ b/src/inc/tcluster.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLUSTER_H +#define TDENGINE_CLUSTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +struct _dnode_obj; + +enum _TSDB_DN_STATUS { + TSDB_DN_STATUS_OFFLINE, + TSDB_DN_STATUS_DROPING, + TSDB_DN_STATUS_BALANCING, + TSDB_DN_STATUS_READY +}; + +int32_t clusterInit(); +void clusterCleanUp(); +char* clusterGetDnodeStatusStr(int32_t dnodeStatus); + +int32_t clusterInitDnodes(); +void clusterCleanupDnodes(); +int32_t clusterGetDnodesNum(); +void * clusterGetNextDnode(void *pNode, struct _dnode_obj **pDnode); +void clusterReleaseDnode(struct _dnode_obj *pDnode); +void * clusterGetDnode(int32_t dnodeId); +void * clusterGetDnodeByIp(uint32_t ip); +void clusterUpdateDnode(struct _dnode_obj *pDnode); +int32_t clusterDropDnode(struct _dnode_obj *pDnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/mnode/inc/mgmtGrant.h b/src/inc/tgrant.h similarity index 95% rename from src/mnode/inc/mgmtGrant.h rename to src/inc/tgrant.h index 92b20532c4..f62a521b6c 100644 --- a/src/mnode/inc/mgmtGrant.h +++ b/src/inc/tgrant.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_GRANT_H -#define TDENGINE_MGMT_GRANT_H +#ifndef TDENGINE_GTANT_H +#define TDENGINE_GTANT_H #ifdef __cplusplus "C" { diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 3097343a48..e54820dffd 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,14 @@ extern "C" { #endif +typedef enum _VN_STATUS { + TAOS_VN_STATUS_INIT, + TAOS_VN_STATUS_CREATING, + TAOS_VN_STATUS_READY, + TAOS_VN_STATUS_CLOSING, + TAOS_VN_STATUS_DELETING, +} EVnStatus; + typedef struct { int len; int code; @@ -41,7 +49,7 @@ void* vnodeGetWqueue(int32_t vgId); void* vnodeGetWal(void *pVnode); void* vnodeGetTsdb(void *pVnode); -int32_t vnodeProcessWrite(void *pVnode, int qtype, SWalHead *pHead, void *item); +int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); void vnodeBuildStatusMsg(void * param); int32_t vnodeProcessRead(void *pVnode, int msgType, void *pCont, int32_t contLen, SRspRet *ret); diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index fb6b8f8b6f..22c92bff6d 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,13 +22,18 @@ extern "C" { #include "mnode.h" +enum _TSDB_DB_STATUS { + TSDB_DB_STATUS_READY, + TSDB_DB_STATUS_DROPPING +}; + // api int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByTableId(char *db); void mgmtIncDbRef(SDbObj *pDb); -void mgmtDecDbRef(SDbObj *pDb); +void mgmtReleaseDb(SDbObj *pDb); bool mgmtCheckIsMonitorDB(char *db, char *monitordb); void mgmtDropAllDbs(SAcctObj *pAcct); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index f82806a645..ad9688c0ee 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,6 +20,19 @@ extern "C" { #endif +enum _TSDB_MN_STATUS { + TSDB_MN_STATUS_OFFLINE, + TSDB_MN_STATUS_UNSYNCED, + TSDB_MN_STATUS_SYNCING, + TSDB_MN_STATUS_SERVING +}; + +enum _TSDB_MN_ROLE { + TSDB_MN_ROLE_UNDECIDED, + TSDB_MN_ROLE_SLAVE, + TSDB_MN_ROLE_MASTER +}; + int32_t mgmtInitMnodes(); void mgmtCleanupMnodes(); diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index f5c9b96f03..6997081c7c 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,8 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -void mgmtIncUserRef(SUserObj *pUser); -void mgmtDecUserRef(SUserObj *pUser); +void mgmtReleaseUser(SUserObj *pUser); SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp); int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); void mgmtDropAllUsers(SAcctObj *pAcct); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b4d50a8318..83e003e063 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -24,6 +24,11 @@ extern "C" { #include #include "mnode.h" +enum _TSDB_VG_STATUS { + TSDB_VG_STATUS_READY, + TSDB_VG_STATUS_UPDATE +}; + int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index 22690c4a44..a22313c52e 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -17,9 +17,10 @@ #include "os.h" #include "taoserror.h" #include "mnode.h" -#include "mgmtAcct.h" +#include "taccount.h" #include "mgmtDb.h" #include "mgmtUser.h" + #ifndef _ACCOUNT static SAcctObj tsAcctObj = {0}; @@ -30,11 +31,12 @@ int32_t acctInit() { return TSDB_CODE_SUCCESS; } -void acctCleanUp() {} -SAcctObj *acctGetAcct(char *acctName) { return &tsAcctObj; } -void acctIncRef(SAcctObj *pAcct) {} -void acctDecRef(SAcctObj *pAcct) {} -int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; } +void acctCleanUp() {} +void *acctGetAcct(char *acctName) { return &tsAcctObj; } +void acctIncRef(struct _acct_obj *pAcct) {} +void acctReleaseAcct(SAcctObj *pAcct) {} +int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; } + #endif void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) { @@ -46,7 +48,7 @@ void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) { void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) { atomic_sub_fetch_32(&pAcct->acctInfo.numOfDbs, 1); pDb->pAcct = NULL; - acctIncRef(pAcct); + acctReleaseAcct(pAcct); } void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) { @@ -58,5 +60,5 @@ void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) { void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) { atomic_sub_fetch_32(&pAcct->acctInfo.numOfUsers, 1); pUser->pAcct = NULL; - acctIncRef(pAcct); + acctReleaseAcct(pAcct); } \ No newline at end of file diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c index e697d70d58..77d68f43d8 100644 --- a/src/mnode/src/mgmtBalance.c +++ b/src/mnode/src/mgmtBalance.c @@ -14,56 +14,34 @@ */ #define _DEFAULT_SOURCE -#include "tstatus.h" -#include "mgmtBalance.h" -#include "mgmtDnode.h" +#include "tbalance.h" +#include "mnode.h" +#include "tcluster.h" +#include "mgmtVgroup.h" -extern int32_t balanceInit(); -extern void balanceCleanUp(); -extern void balanceNotify(); -extern int32_t balanceAllocVnodes(SVgObj *pVgroup); +#ifndef _VPEER +int32_t balanceInit() { return 0; } +void balanceCleanUp() {} +void balanceNotify() {} -int32_t mgmtInitBalance() { -#ifdef _VPEER - return balanceInit(); -#else - return 0; -#endif -} - -void mgmtCleanupBalance() { -#ifdef _VPEER - balanceCleanUp(); -#endif -} - -void mgmtBalanceNotify() { -#ifdef _VPEER - balanceNotify(); -#endif -} - -int32_t mgmtAllocVnodes(SVgObj *pVgroup) { -#ifdef _VPEER - return balanceAllocVnodes(pVgroup); -#else +int32_t balanceAllocVnodes(SVgObj *pVgroup) { void * pNode = NULL; SDnodeObj *pDnode = NULL; SDnodeObj *pSelDnode = NULL; float vnodeUsage = 1.0; while (1) { - mgmtDecDnodeRef(pDnode); - pNode = mgmtGetNextDnode(pNode, &pDnode); + pNode = clusterGetNextDnode(pNode, &pDnode); if (pDnode == NULL) break; - if (pDnode->numOfTotalVnodes <= 0) continue; - if (pDnode->openVnodes == pDnode->numOfTotalVnodes) continue; - float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes; - if (usage <= vnodeUsage) { - pSelDnode = pDnode; - vnodeUsage = usage; + if (pDnode->numOfTotalVnodes > 0 && pDnode->openVnodes < pDnode->numOfTotalVnodes) { + float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes; + if (usage <= vnodeUsage) { + pSelDnode = pDnode; + vnodeUsage = usage; + } } + clusterReleaseDnode(pDnode); } if (pSelDnode == NULL) { @@ -77,5 +55,6 @@ int32_t mgmtAllocVnodes(SVgObj *pVgroup) { mTrace("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes); return TSDB_CODE_SUCCESS; -#endif } + +#endif diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index cbdcb8ff5d..b402a85005 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -17,15 +17,14 @@ #include "os.h" #include "taoserror.h" #include "tsched.h" -#include "tstatus.h" #include "tsystem.h" #include "tutil.h" #include "dnode.h" #include "mnode.h" -#include "mgmtBalance.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index ca23a3cfca..0ac93d429c 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -18,15 +18,14 @@ #include "taoserror.h" #include "trpc.h" #include "tsched.h" -#include "tstatus.h" #include "tsystem.h" #include "tutil.h" #include "dnode.h" #include "mnode.h" -#include "mgmtBalance.h" +#include "tbalance.h" #include "mgmtDb.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 7d13451f7e..9ecc2c6458 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -16,15 +16,14 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tstatus.h" #include "tutil.h" #include "name.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtShell.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -38,7 +37,7 @@ static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(SQueuedMsg *newMsg); -static int32_t mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDropping(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); @@ -146,11 +145,11 @@ SDbObj *mgmtGetDb(char *db) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } -void mgmtIncDbRef(SDbObj *pDb) { +void mgmtIncDbRef(SDbObj *pDb) { return sdbIncRef(tsDbSdb, pDb); } -void mgmtDecDbRef(SDbObj *pDb) { +void mgmtReleaseDb(SDbObj *pDb) { return sdbDecRef(tsDbSdb, pDb); } @@ -289,7 +288,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { SDbObj *pDb = mgmtGetDb(pCreate->db); if (pDb != NULL) { - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_DB_ALREADY_EXIST; } @@ -519,7 +518,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->numOfRows = pUser->pAcct->acctInfo.numOfDbs; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } @@ -631,15 +630,15 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, pDb->dirty != TSDB_DB_STATUS_READY ? "dropping" : "ready"); + strcpy(pWrite, pDb->status != TSDB_DB_STATUS_READY ? "dropping" : "ready"); cols++; numOfRows++; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); } pShow->numOfReads += numOfRows; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return numOfRows; } @@ -659,10 +658,10 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static int32_t mgmtSetDbDirty(SDbObj *pDb) { - if (pDb->dirty) return TSDB_CODE_SUCCESS; +static int32_t mgmtSetDbDropping(SDbObj *pDb) { + if (pDb->status) return TSDB_CODE_SUCCESS; - pDb->dirty = true; + pDb->status = true; SSdbOperDesc oper = { .type = SDB_OPER_TYPE_GLOBAL, .table = tsDbSdb, @@ -850,7 +849,7 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - int32_t code = mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDropping(pDb); if (code != TSDB_CODE_SUCCESS) { mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); mgmtSendSimpleResp(pMsg->thandle, code); @@ -881,11 +880,11 @@ void mgmtDropAllDbs(SAcctObj *pAcct) { if (pDb == NULL) break; if (pDb->pAcct == pAcct) { - mgmtSetDbDirty(pDb); + mgmtSetDbDropping(pDb); numOfDbs++; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); } mTrace("acct:%s, all dbs is is set dirty", pAcct->user, numOfDbs); -} \ No newline at end of file +} diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 7019d8e20a..b60d1cb0ba 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tmodule.h" -#include "tstatus.h" -#include "mgmtBalance.h" -#include "mgmtDnode.h" +#include "tbalance.h" +#include "tcluster.h" +#include "mnode.h" #include "mgmtDClient.h" #include "mgmtMnode.h" #include "mgmtShell.h" @@ -26,45 +26,23 @@ #include "mgmtUser.h" #include "mgmtVgroup.h" -static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); -static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg); -static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg); +static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; +static void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg); +static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -extern int32_t clusterInit(); -extern void clusterCleanUp(); -extern int32_t clusterGetDnodesNum(); -extern void * clusterGetNextDnode(void *pNode, SDnodeObj **pDnode); -extern void clusterReleaseDnode(SDnodeObj *pDnode); -extern SDnodeObj* clusterGetDnode(int32_t dnodeId); -extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip); #ifndef _CLUSTER -static SDnodeObj tsDnodeObj = {0}; -#endif -int32_t mgmtInitDnodes() { - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp); - mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes); - -#ifdef _CLUSTER - return clusterInit(); -#else +static SDnodeObj tsDnodeObj = {0}; + +int32_t clusterInitDnodes() { tsDnodeObj.dnodeId = 1; tsDnodeObj.privateIp = inet_addr(tsPrivateIp); tsDnodeObj.publicIp = inet_addr(tsPublicIp); @@ -82,63 +60,47 @@ int32_t mgmtInitDnodes() { tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR); } return 0; -#endif } -void mgmtCleanUpDnodes() { -#ifdef _CLUSTER - clusterCleanUp(); -#endif -} - -SDnodeObj *mgmtGetDnode(int32_t dnodeId) { -#ifdef _CLUSTER - return clusterGetDnode(dnodeId); -#else - if (dnodeId == 1) { - return &tsDnodeObj; - } else { - return NULL; - } -#endif -} - -SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) { -#ifdef _CLUSTER - return clusterGetDnodeByIp(ip); -#else - return &tsDnodeObj; -#endif -} - -int32_t mgmtGetDnodesNum() { -#ifdef _CLUSTER - return clusterGetDnodesNum(); -#else - return 1; -#endif -} - -void mgmtReleaseDnode(SDnodeObj *pDnode) { -#ifdef _CLUSTER - return clusterReleaseDnode(pDnode); -#endif -} - -void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { -#ifdef _CLUSTER - return clusterGetNextDnode(pNode, pDnode); -#else +void *clusterGetNextDnode(void *pNode, SDnodeObj **pDnode) { if (*pDnode == NULL) { *pDnode = &tsDnodeObj; } else { *pDnode = NULL; } return *pDnode; -#endif } -void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { +void clusterCleanupDnodes() {} +int32_t clusterGetDnodesNum() { return 1; } +void * clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : NULL; } +void * clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; } +void clusterReleaseDnode(struct _dnode_obj *pDnode) {} +void clusterUpdateDnode(struct _dnode_obj *pDnode) {} + +#endif + +int32_t clusterInit() { + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, clusterProcessCfgDnodeMsg); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, clusterProcessCfgDnodeMsgRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, clusterProcessDnodeStatusMsg); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, clusterGetModuleMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, clusterRetrieveModules); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, clusterGetConfigMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, clusterRetrieveConfigs); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, clusterGetVnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, clusterRetrieveVnodes); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, clusterGetDnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, clusterRetrieveDnodes); + + return clusterInitDnodes(); +} + +void clusterCleanUp() { + clusterCleanupDnodes(); +} + +void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont; @@ -174,11 +136,11 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { rpcSendResponse(&rpcRsp); } -static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { +static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { mPrint("cfg vnode rsp is received, result:%s", tstrerror(rpcMsg->code)); } -void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { +void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { if (mgmtCheckRedirect(rpcMsg->handle)) return; SDMStatusMsg *pStatus = rpcMsg->pCont; @@ -198,14 +160,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { - pDnode = mgmtGetDnodeByIp(pStatus->privateIp); + pDnode = clusterGetDnodeByIp(pStatus->privateIp); if (pDnode == NULL) { mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp)); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); return; } } else { - pDnode = mgmtGetDnode(pStatus->dnodeId); + pDnode = clusterGetDnode(pStatus->dnodeId); if (pDnode == NULL) { mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp)); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); @@ -238,16 +200,16 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pDnode->vload[j].vgId); mgmtSendDropVnodeMsg(pDnode->vload[j].vgId, &ipSet, NULL); } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); } if (pDnode->status != TSDB_DN_STATUS_READY) { mTrace("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TSDB_DN_STATUS_READY; - mgmtBalanceNotify(); + balanceNotify(); } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SVnodeAccess); SDMStatusRsp *pRsp = rpcMallocCont(contLen); @@ -277,7 +239,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { rpcSendResponse(&rpcRsp); } -static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -344,16 +306,16 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = mgmtGetDnodesNum(); + pShow->numOfRows = clusterGetDnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; int32_t cols = 0; SDnodeObj *pDnode = NULL; @@ -361,8 +323,8 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi char ipstr[32]; while (numOfRows < rows) { - mgmtDecDnodeRef(pDnode); - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + clusterReleaseDnode(pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; cols = 0; @@ -386,7 +348,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -399,7 +361,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi #ifdef _VPEER pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeLbStatusStr(pDnode->lbStatus)); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; #endif @@ -415,7 +377,7 @@ static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { return status > 0; } -static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); @@ -454,7 +416,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->numOfRows = 0; SDnodeObj *pDnode = NULL; while (1) { - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { if (clusterCheckModuleInDnode(pDnode, moduleType)) { @@ -465,12 +427,12 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SDnodeObj *pDnode = NULL; char * pWrite; @@ -478,8 +440,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo char ipstr[20]; while (numOfRows < rows) { - mgmtDecDnodeRef(pDnode); - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + clusterReleaseDnode(pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { @@ -499,7 +461,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; numOfRows++; @@ -516,7 +478,7 @@ static bool clusterCheckConfigShow(SGlobalConfig *cfg) { return true; } -static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); @@ -553,12 +515,12 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) { @@ -605,7 +567,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } -static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -625,12 +587,6 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "sync_status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -640,7 +596,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo SDnodeObj *pDnode = NULL; if (pShow->payloadLen > 0 ) { uint32_t ip = ip2uint(pShow->payload); - pDnode = mgmtGetDnodeByIp(ip); + pDnode = clusterGetDnodeByIp(ip); if (NULL == pDnode) { return TSDB_CODE_NODE_OFFLINE; } @@ -657,7 +613,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->pNode = pDnode; } else { while (true) { - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; pShow->numOfRows += pDnode->openVnodes; @@ -668,13 +624,13 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDnodeRef(pDnode); - mgmtDecUserRef(pUser); + clusterReleaseDnode(pDnode); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SDnodeObj *pDnode = NULL; char * pWrite; @@ -700,11 +656,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus)); + strcpy(pWrite, pVnode->status ? "ready" : "offline"); cols++; numOfRows++; @@ -719,7 +671,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -char* mgmtGetDnodeStatusStr(int32_t dnodeStatus) { +char* clusterGetDnodeStatusStr(int32_t dnodeStatus) { switch (dnodeStatus) { case TSDB_DN_STATUS_OFFLINE: return "offline"; case TSDB_DN_STATUS_DROPING: return "dropping"; diff --git a/src/mnode/src/mgmtGrant.c b/src/mnode/src/mgmtGrant.c index 097d0fde3d..ed32f97426 100644 --- a/src/mnode/src/mgmtGrant.c +++ b/src/mnode/src/mgmtGrant.c @@ -18,7 +18,7 @@ #include "os.h" #include "taoserror.h" #include "tlog.h" -#include "mgmtGrant.h" +#include "tgrant.h" int32_t grantInit() { return TSDB_CODE_SUCCESS; } void grantCleanUp() {} diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 4a81c9b691..e04630d745 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -19,13 +19,13 @@ #include "tmodule.h" #include "tsched.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" +#include "tcluster.h" #include "mgmtDb.h" #include "mgmtDClient.h" -#include "mgmtDnode.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtVgroup.h" @@ -89,7 +89,7 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitDnodes() < 0) { + if (clusterInit() < 0) { mError("failed to init dnodes"); return -1; } @@ -122,7 +122,7 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitBalance() < 0) { + if (balanceInit() < 0) { mError("failed to init dnode balance") } @@ -148,14 +148,14 @@ void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); grantCleanUp(); mgmtCleanupMnodes(); - mgmtCleanupBalance(); + balanceCleanUp(); mgmtCleanUpShell(); mgmtCleanupDClient(); mgmtCleanupDServer(); mgmtCleanUpTables(); mgmtCleanUpVgroups(); mgmtCleanUpDbs(); - mgmtCleanUpDnodes(); + clusterCleanUp(); mgmtCleanUpUsers(); acctCleanUp(); taosTmrCleanUp(tsMgmtTmr); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 4ccc4b8a04..1e2a4e9066 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tstatus.h" #include "trpc.h" #include "mgmtMnode.h" #include "mgmtSdb.h" @@ -64,6 +63,25 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { return *pMnode; } +char *taosGetMnodeStatusStr(int32_t mnodeStatus) { + switch (mnodeStatus) { + case TSDB_MN_STATUS_OFFLINE: return "offline"; + case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; + case TSDB_MN_STATUS_SYNCING: return "syncing"; + case TSDB_MN_STATUS_SERVING: return "serving"; + default: return "undefined"; + } +} + +char *taosGetMnodeRoleStr(int32_t mnodeRole) { + switch (mnodeRole) { + case TSDB_MN_ROLE_UNDECIDED: return "undicided"; + case TSDB_MN_ROLE_SLAVE: return "slave"; + case TSDB_MN_ROLE_MASTER: return "master"; + default: return "undefined"; + } +} + static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -120,7 +138,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->numOfRows = mgmtGetMnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 4a8b4b1ea1..0360432971 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" +#include "taccount.h" +#include "tcluster.h" #include "mgmtDb.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -787,12 +789,12 @@ void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) { void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { if (pMsg != NULL) { rpcFreeCont(pMsg->pCont); - if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser); - if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb); - if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup); + if (pMsg->pUser) mgmtReleaseUser(pMsg->pUser); + if (pMsg->pDb) mgmtReleaseDb(pMsg->pDb); + if (pMsg->pVgroup) mgmtReleaseVgroup(pMsg->pVgroup); if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable); - // if (pMsg->pAcct) acctDecRef(pMsg->pAcct); - // if (pMsg->pDnode) mgmtDecTableRef(pMsg->pDnode); + if (pMsg->pAcct) acctReleaseAcct(pMsg->pAcct); + if (pMsg->pDnode) clusterReleaseDnode(pMsg->pDnode); free(pMsg); } } diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 4b2f53f012..7b6a2654ae 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -19,15 +19,14 @@ #include "taoserror.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsched.h" #include "dnode.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" @@ -179,6 +178,28 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { } } +char *mgmtGetShowTypeStr(int32_t showType) { + switch (showType) { + case TSDB_MGMT_TABLE_ACCT: return "show accounts"; + case TSDB_MGMT_TABLE_USER: return "show users"; + case TSDB_MGMT_TABLE_DB: return "show databases"; + case TSDB_MGMT_TABLE_TABLE: return "show tables"; + case TSDB_MGMT_TABLE_DNODE: return "show dnodes"; + case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; + case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; + case TSDB_MGMT_TABLE_METRIC: return "show stables"; + case TSDB_MGMT_TABLE_MODULE: return "show modules"; + case TSDB_MGMT_TABLE_QUERIES: return "show queries"; + case TSDB_MGMT_TABLE_STREAMS: return "show streams"; + case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_CONNS: return "show connections"; + case TSDB_MGMT_TABLE_SCORES: return "show scores"; + case TSDB_MGMT_TABLE_GRANTS: return "show grants"; + case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; + default: return "undefined"; + } +} + static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { SCMShowMsg *pShowMsg = pMsg->pCont; if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) { @@ -187,7 +208,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { } if (!tsMgmtShowMetaFp[pShowMsg->type] || !tsMgmtShowRetrieveFp[pShowMsg->type]) { - mError("show type:%s is not support", taosGetShowTypeStr(pShowMsg->type)); + mError("show type:%s is not support", mgmtGetShowTypeStr(pShowMsg->type)); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OPS_NOT_SUPPORT); return; } @@ -209,7 +230,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { mgmtSaveQhandle(pShow); pShowRsp->qhandle = htobe64((uint64_t) pShow); - mTrace("show:%p, type:%s, start to get meta", pShow, taosGetShowTypeStr(pShowMsg->type)); + mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type)); int32_t code = (*tsMgmtShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->thandle); if (code == 0) { SRpcMsg rpcRsp = { @@ -220,7 +241,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { }; rpcSendResponse(&rpcRsp); } else { - mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, taosGetShowTypeStr(pShowMsg->type), tstrerror(code)); + mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code)); mgmtFreeQhandle(pShow); SRpcMsg rpcRsp = { .handle = pMsg->thandle, @@ -248,7 +269,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) { } SShowObj *pShow = (SShowObj *)pRetrieve->qhandle; - mTrace("show:%p, type:%s, retrieve data", pShow, taosGetShowTypeStr(pShow->type)); + mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type)); if (!mgmtCheckQhandle(pRetrieve->qhandle)) { mError("pShow:%p, query memory is corrupted", pShow); @@ -338,11 +359,11 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secr SUserObj *pUser = mgmtGetUser(user); if (pUser == NULL) { *secret = 0; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_INVALID_USER; } else { memcpy(secret, pUser->pass, TSDB_KEY_LEN); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_SUCCESS; } } diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index d6d7a6afc0..f81e13414b 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -19,7 +19,6 @@ #include "tscompression.h" #include "tskiplist.h" #include "ttime.h" -#include "tstatus.h" #include "tutil.h" #include "qast.h" #include "qextbuffer.h" @@ -28,15 +27,14 @@ #include "tscompression.h" #include "tskiplist.h" #include "tsqlfunction.h" -#include "tstatus.h" #include "ttime.h" #include "name.h" -#include "mgmtAcct.h" +#include "taccount.h" #include "mgmtDClient.h" #include "mgmtDb.h" -#include "mgmtDnode.h" +#include "tcluster.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" @@ -98,21 +96,21 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId); return TSDB_CODE_INVALID_VGROUP_ID; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - acctDecRef(pAcct); + acctReleaseAcct(pAcct); if (pTable->info.type == TSDB_CHILD_TABLE) { pTable->superTable = mgmtGetSuperTable(pTable->superTableId); @@ -140,21 +138,21 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { if (pVgroup == NULL) { return TSDB_CODE_INVALID_VGROUP_ID; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - acctDecRef(pAcct); + acctReleaseAcct(pAcct); if (pTable->info.type == TSDB_CHILD_TABLE) { grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1); @@ -272,7 +270,7 @@ static int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -286,7 +284,7 @@ static int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); if (strcmp(pVgroup->dbName, pDb->name) != 0) { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", @@ -354,7 +352,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_SUCCESS; } @@ -366,7 +364,7 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { mgmtRemoveSuperTableFromDb(pDb); mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_SUCCESS; } @@ -554,7 +552,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { } pMsg->pDb = mgmtGetDb(pCreate->db); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to create, db not selected", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -572,7 +570,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->pCont; pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to drop table, db not selected", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -611,7 +609,7 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -860,7 +858,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -897,7 +895,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -963,7 +961,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, pShow->numOfRows = pDb->numOfSuperTables; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } @@ -1028,7 +1026,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v } pShow->numOfReads += numOfRows; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -1106,7 +1104,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { return; } - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum()); + SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * clusterGetDnodesNum()); if (pRsp == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; @@ -1409,7 +1407,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += ncols; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -1443,7 +1441,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries--; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -1633,7 +1631,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { } static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { - SDnodeObj *pObj = mgmtGetDnode(dnodeId); + SDnodeObj *pObj = clusterGetDnode(dnodeId); SVgObj *pVgroup = mgmtGetVgroup(vnode); if (pObj == NULL || pVgroup == NULL) { @@ -1642,7 +1640,7 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_ SChildTableObj *pTable = pVgroup->tableList[sid]; mgmtIncTableRef((STableObj *)pTable); - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); return pTable; } @@ -1863,7 +1861,7 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void pShow->numOfRows = pDb->numOfTables; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } @@ -1940,7 +1938,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, const int32_t NUM_OF_COLUMNS = 4; mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -1950,7 +1948,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle); pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to alter table, db not selected", pAlter->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 0ee41b6b87..fe7d40e120 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -18,8 +18,8 @@ #include "trpc.h" #include "ttime.h" #include "tutil.h" -#include "mgmtAcct.h" -#include "mgmtGrant.h" +#include "taccount.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -117,7 +117,7 @@ int32_t mgmtInitUsers() { mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); - acctDecRef(pAcct); + acctReleaseAcct(pAcct); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg); @@ -137,11 +137,7 @@ SUserObj *mgmtGetUser(char *name) { return (SUserObj *)sdbGetRow(tsUserSdb, name); } -void mgmtIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); -} - -void mgmtDecUserRef(SUserObj *pUser) { +void mgmtReleaseUser(SUserObj *pUser) { return sdbDecRef(tsUserSdb, pUser); } @@ -174,7 +170,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { SUserObj *pUser = mgmtGetUser(name); if (pUser != NULL) { mTrace("user:%s is already there", name); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_USER_ALREADY_EXIST; } @@ -264,7 +260,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon pShow->numOfRows = pUser->pAcct->acctInfo.numOfUsers; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } @@ -299,7 +295,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void cols++; numOfRows++; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } pShow->numOfReads += numOfRows; return numOfRows; @@ -351,7 +347,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { if (strcmp(pUser->user, "monitor") == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return; } @@ -427,7 +423,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { @@ -446,7 +442,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (strcmp(pUser->user, "monitor") == 0 || strcmp(pUser->user, pUser->acct) == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return ; } @@ -475,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { } mgmtSendSimpleResp(pMsg->thandle, code); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } void mgmtDropAllUsers(SAcctObj *pAcct) { @@ -501,7 +497,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { numOfUsers++; } - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index a6ceaa1f41..a0b80c8ea3 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -17,12 +17,11 @@ #include "os.h" #include "taoserror.h" #include "tlog.h" -#include "tstatus.h" +#include "tbalance.h" +#include "tcluster.h" #include "mnode.h" -#include "mgmtBalance.h" #include "mgmtDb.h" #include "mgmtDClient.h" -#include "mgmtDnode.h" #include "mgmtDServer.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -54,11 +53,11 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { } for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId); if (pDnode) { atomic_sub_fetch_32(&pDnode->openVnodes, 1); } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); } tfree(pOper->pObj); @@ -71,7 +70,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { if (pDb == NULL) { return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); pVgroup->pDb = pDb; pVgroup->prev = NULL; @@ -92,12 +91,12 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { } for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId); if (pDnode != NULL) { pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; atomic_add_fetch_32(&pDnode->openVnodes, 1); - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); } } @@ -114,7 +113,7 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { mgmtRemoveVgroupFromDb(pVgroup); } - mgmtDecDbRef(pVgroup->pDb); + mgmtReleaseDb(pVgroup->pDb); return TSDB_CODE_SUCCESS; } @@ -214,12 +213,16 @@ SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } +void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { + return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); +} + void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) { SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); - if (mgmtAllocVnodes(pVgroup) != 0) { + if (balanceAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); @@ -310,7 +313,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { mgmtDecTableRef(pTable); pVgroup = mgmtGetVgroup(((SChildTableObj*)pTable)->vgId); if (NULL == pVgroup) return TSDB_CODE_INVALID_TABLE_ID; - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; } else { SVgObj *pVgroup = pDb->pHead; @@ -356,18 +359,18 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->pNode = pVgroup; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { - SDnodeObj *pDnode = mgmtGetDnode(pVnode->dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVnode->dnodeId); if (pDnode == NULL) { mError("vgroup:%d, not exist in dnode:%d", pVgroup->vgId, pDnode->dnodeId); return "null"; } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); if (pDnode->status == TSDB_DN_STATUS_OFFLINE) { return "offline"; @@ -375,7 +378,7 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { for (int i = 0; i < pDnode->openVnodes; ++i) { if (pDnode->vload[i].vgId == pVgroup->vgId) { - return (char*)taosGetVnodeStatusStr(pDnode->vload[i].status); + return pDnode->vload[i].status ? "ready" : "offline"; } } @@ -415,7 +418,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVgroupLbStatusStr(pVgroup->lbStatus)); + strcpy(pWrite, pVgroup->status ? "updating" : "ready"); cols++; for (int32_t i = 0; i < maxReplica; ++i) { @@ -442,7 +445,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo } pShow->numOfReads += numOfRows; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -653,13 +656,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); - SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pCfg->dnodeId); if (pDnode == NULL) { mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); return; } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId); if (pVgroup == NULL) { @@ -667,7 +670,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); return; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); @@ -683,7 +686,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { SVgObj *pVgroup = NULL; while (1) { - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; @@ -712,4 +715,5 @@ void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { } else { mgmtAddToShellQueue(ahandle); } -} \ No newline at end of file +} + diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index f0aa13ee3d..a6419d3549 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -20,7 +20,6 @@ #include "tlog.h" #include "tlosertree.h" #include "tscompression.h" -#include "tstatus.h" #include "ttime.h" #include "qast.h" diff --git a/src/util/inc/tstatus.h b/src/util/inc/tstatus.h deleted file mode 100644 index 4fa392ca35..0000000000 --- a/src/util/inc/tstatus.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSTATUS_H -#define TDENGINE_TSTATUS_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "taoserror.h" - - -enum _TSDB_DB_STATUS { - TSDB_DB_STATUS_READY, - TSDB_DB_STATUS_DROPPING, - TSDB_DB_STATUS_DROP_FROM_SDB -}; - -typedef enum _TSDB_VN_STATUS { - TSDB_VN_STATUS_NOT_READY, - TSDB_VN_STATUS_UNSYNCED, - TSDB_VN_STATUS_SLAVE, - TSDB_VN_STATUS_MASTER, - TSDB_VN_STATUS_CREATING, - TSDB_VN_STATUS_CLOSING, - TSDB_VN_STATUS_DELETING, -} EVnodeStatus; - -enum _TSDB_VN_SYNC_STATUS { - TSDB_VN_SYNC_STATUS_INIT, - TSDB_VN_SYNC_STATUS_SYNCING, - TSDB_VN_SYNC_STATUS_SYNC_CACHE, - TSDB_VN_SYNC_STATUS_SYNC_FILE -}; - -enum _TSDB_VN_DROP_STATUS { - TSDB_VN_DROP_STATUS_READY, - TSDB_VN_DROP_STATUS_DROPPING -}; - -enum _TSDB_MN_STATUS { - TSDB_MN_STATUS_OFFLINE, - TSDB_MN_STATUS_UNSYNCED, - TSDB_MN_STATUS_SYNCING, - TSDB_MN_STATUS_SERVING -}; - -enum _TSDB_MN_ROLE { - TSDB_MN_ROLE_UNDECIDED, - TSDB_MN_ROLE_SLAVE, - TSDB_MN_ROLE_MASTER -}; - -enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY, - TSDB_VG_STATUS_UPDATE -}; - -enum _TSDB_VN_STREAM_STATUS { - TSDB_VN_STREAM_STATUS_STOP, - TSDB_VN_STREAM_STATUS_START -}; - -enum TSDB_TABLE_STATUS { - TSDB_METER_STATE_READY = 0x00, - TSDB_METER_STATE_INSERTING = 0x01, - TSDB_METER_STATE_IMPORTING = 0x02, - TSDB_METER_STATE_UPDATING = 0x04, - TSDB_METER_STATE_DROPPING = 0x10, - TSDB_METER_STATE_DROPPED = 0x18, -}; - -char* taosGetVgroupStatusStr(int32_t vgroupStatus); -char* taosGetDbStatusStr(int32_t dbStatus); -char* taosGetVnodeStatusStr(int32_t vnodeStatus); -char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus); -char* taosGetVnodeDropStatusStr(int32_t dropping); -char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus); -char* taosGetVgroupLbStatusStr(int32_t vglbStatus); -char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus); -char* taosGetTableStatusStr(int32_t tableStatus); -char *taosGetShowTypeStr(int32_t showType); -char *taosGetMnodeStatusStr(int32_t mnodeStatus); -char *taosGetMnodeRoleStr(int32_t mnodeRole); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSTATUS_H diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c deleted file mode 100644 index cfcbea0ca1..0000000000 --- a/src/util/src/tstatus.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "taosmsg.h" -#include "tstatus.h" - -char* taosGetVgroupStatusStr(int32_t vgroupStatus) { - switch (vgroupStatus) { - case TSDB_VG_STATUS_READY: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_IN_PROGRESS: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_NO_DISK_PERMISSIONS: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERVER_NO_PACE: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_INIT_FAILED: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_FULL: return (char*)tstrerror(vgroupStatus); - default: return "undefined"; - } -} - -char* taosGetDbStatusStr(int32_t dbStatus) { - switch (dbStatus) { - case TSDB_DB_STATUS_READY: return "ready"; - case TSDB_DB_STATUS_DROPPING: return "dropping"; - case TSDB_DB_STATUS_DROP_FROM_SDB: return "drop_from_sdb"; - default: return "undefined"; - } -} - -char* taosGetVnodeStatusStr(int32_t vnodeStatus) { - switch (vnodeStatus) { - case TSDB_VN_STATUS_NOT_READY:return "not_ready"; - case TSDB_VN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_VN_STATUS_SLAVE: return "slave"; - case TSDB_VN_STATUS_MASTER: return "master"; - case TSDB_VN_STATUS_CREATING: return "creating"; - case TSDB_VN_STATUS_CLOSING: return "closing"; - case TSDB_VN_STATUS_DELETING: return "deleting"; - default: return "undefined"; - } -} - -char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus) { - switch (vnodeSyncStatus) { - case TSDB_VN_SYNC_STATUS_INIT: return "ready"; - case TSDB_VN_SYNC_STATUS_SYNCING: return "syncing"; - case TSDB_VN_SYNC_STATUS_SYNC_CACHE: return "sync_cache"; - case TSDB_VN_SYNC_STATUS_SYNC_FILE: return "sync_file"; - default: return "undefined"; - } -} - -char* taosGetVnodeDropStatusStr(int32_t dropping) { - switch (dropping) { - case TSDB_VN_DROP_STATUS_READY: return "ready"; - case TSDB_VN_DROP_STATUS_DROPPING: return "dropping"; - default: return "undefined"; - } -} - -char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus) { - switch (dnodeBalanceStatus) { - case TSDB_DN_LB_STATUS_BALANCED: return "balanced"; - case TSDB_DN_LB_STATUS_BALANCING: return "balancing"; - case TSDB_DN_LB_STATUS_OFFLINE_REMOVING: return "offline removing"; - case TSDB_DN_LB_STATE_SHELL_REMOVING: return "removing"; - default: return "undefined"; - } -} - -char* taosGetVgroupLbStatusStr(int32_t vglbStatus) { - switch (vglbStatus) { - case TSDB_VG_LB_STATUS_READY: return "ready"; - case TSDB_VG_LB_STATUS_UPDATE: return "updating"; - default: return "undefined"; - } -} - -char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus) { - switch (vnodeStreamStatus) { - case TSDB_VN_STREAM_STATUS_START: return "start"; - case TSDB_VN_STREAM_STATUS_STOP: return "stop"; - default: return "undefined"; - } -} - -char* taosGetTableStatusStr(int32_t tableStatus) { - switch(tableStatus) { - case TSDB_METER_STATE_INSERTING:return "inserting"; - case TSDB_METER_STATE_IMPORTING:return "importing"; - case TSDB_METER_STATE_UPDATING: return "updating"; - case TSDB_METER_STATE_DROPPING: return "deleting"; - case TSDB_METER_STATE_DROPPED: return "dropped"; - case TSDB_METER_STATE_READY: return "ready"; - default:return "undefined"; - } -} - -char *taosGetShowTypeStr(int32_t showType) { - switch (showType) { - case TSDB_MGMT_TABLE_ACCT: return "show accounts"; - case TSDB_MGMT_TABLE_USER: return "show users"; - case TSDB_MGMT_TABLE_DB: return "show databases"; - case TSDB_MGMT_TABLE_TABLE: return "show tables"; - case TSDB_MGMT_TABLE_DNODE: return "show dnodes"; - case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; - case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_METRIC: return "show stables"; - case TSDB_MGMT_TABLE_MODULE: return "show modules"; - case TSDB_MGMT_TABLE_QUERIES: return "show queries"; - case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; - case TSDB_MGMT_TABLE_CONNS: return "show connections"; - case TSDB_MGMT_TABLE_SCORES: return "show scores"; - case TSDB_MGMT_TABLE_GRANTS: return "show grants"; - case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; - default: return "undefined"; - } -} - -char *taosGetMnodeStatusStr(int32_t mnodeStatus) { - switch (mnodeStatus) { - case TSDB_MN_STATUS_OFFLINE: return "offline"; - case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_MN_STATUS_SYNCING: return "syncing"; - case TSDB_MN_STATUS_SERVING: return "serving"; - default: return "undefined"; - } -} - -char *taosGetMnodeRoleStr(int32_t mnodeRole) { - switch (mnodeRole) { - case TSDB_MN_ROLE_UNDECIDED: return "undicided"; - case TSDB_MN_ROLE_SLAVE: return "slave"; - case TSDB_MN_ROLE_MASTER: return "master"; - default: return "undefined"; - } -} diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 4d078869c4..96526d7209 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -23,18 +23,10 @@ extern "C" { #include "tsync.h" #include "twal.h" -typedef enum _VN_STATUS { - VN_STATUS_INIT, - VN_STATUS_CREATING, - VN_STATUS_READY, - VN_STATUS_CLOSING, - VN_STATUS_DELETING, -} EVnStatus; - typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count - EVnStatus status; + int status; int role; int64_t version; void *wqueue; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 1211828a47..786e6f63a0 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -20,7 +20,6 @@ #include "taosmsg.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" @@ -112,7 +111,7 @@ int32_t vnodeDrop(int32_t vgId) { } dTrace("pVnode:%p vgId:%d, vnode will be dropped", pVnode, pVnode->vgId); - pVnode->status = VN_STATUS_DELETING; + pVnode->status = TAOS_VN_STATUS_DELETING; vnodeCleanUp(pVnode); return TSDB_CODE_SUCCESS; @@ -124,7 +123,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1); pVnode->vgId = vnode; - pVnode->status = VN_STATUS_INIT; + pVnode->status = TAOS_VN_STATUS_INIT; pVnode->refCount = 1; pVnode->version = 0; taosAddIntHash(tsDnodeVnodesHash, pVnode->vgId, (char *)(&pVnode)); @@ -161,7 +160,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { walRestore(pVnode->wal, pVnode, vnodeWriteToQueue); - pVnode->status = VN_STATUS_READY; + pVnode->status = TAOS_VN_STATUS_READY; dTrace("pVnode:%p vgId:%d, vnode is opened in %s", pVnode, pVnode->vgId, rootDir); tsOpennedVnodes++; @@ -174,7 +173,7 @@ int32_t vnodeClose(int32_t vgId) { if (pVnode == NULL) return 0; dTrace("pVnode:%p vgId:%d, vnode will be closed", pVnode, pVnode->vgId); - pVnode->status = VN_STATUS_CLOSING; + pVnode->status = TAOS_VN_STATUS_CLOSING; vnodeCleanUp(pVnode); return 0; @@ -198,7 +197,7 @@ void vnodeRelease(void *pVnodeRaw) { dnodeFreeWqueue(pVnode->wqueue); pVnode->wqueue = NULL; - if (pVnode->status == VN_STATUS_DELETING) { + if (pVnode->status == TAOS_VN_STATUS_DELETING) { // remove the whole directory } @@ -258,7 +257,7 @@ void vnodeBuildStatusMsg(void *param) { static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeObj *pVnode = *(SVnodeObj **) pNode; - if (pVnode->status == VN_STATUS_DELETING) return; + if (pVnode->status == TAOS_VN_STATUS_DELETING) return; SDMStatusMsg *pStatus = param; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; diff --git a/src/vnode/main/src/vnodeRead.c b/src/vnode/main/src/vnodeRead.c index 929a30fbcd..59f392eb82 100644 --- a/src/vnode/main/src/vnodeRead.c +++ b/src/vnode/main/src/vnodeRead.c @@ -42,7 +42,7 @@ int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, if (vnodeProcessReadMsgFp[msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) + if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_NOT_ACTIVE_VNODE; return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..605a57ebd0 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -41,18 +41,19 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessDropStableMsg; } -int32_t vnodeProcessWrite(void *param, int qtype, SWalHead *pHead, void *item) { +int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { int32_t code = 0; - SVnodeObj *pVnode = (SVnodeObj *)param; + SVnodeObj *pVnode = (SVnodeObj *)param1; + SWalHead *pHead = param2; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) + if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_NOT_ACTIVE_VNODE; if (pHead->version == 0) { // from client - if (pVnode->status != VN_STATUS_READY) + if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; // if (pVnode->replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) From 770392fbde4635a7764066386b57d58c899bc8a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 20:25:03 +0800 Subject: [PATCH 19/48] fix use-after-free issues. [TD-122] --- src/client/src/tscAsync.c | 2 +- src/client/src/tscProfile.c | 3 +-- src/client/src/tscServer.c | 2 +- src/client/src/tscSql.c | 2 +- src/client/src/tscStream.c | 4 +++- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 36cd42332b..6288823401 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -342,8 +342,8 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { (*pSql->fp)(pSql->param, taosres, code); if (shouldFree) { - tscFreeSqlObj(pSql); tscTrace("%p Async sql is automatically freed in async res", pSql); + tscFreeSqlObj(pSql); } } diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 00c8d77619..739ae7848e 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -292,7 +292,6 @@ void tscKillConnection(STscObj *pObj) { pthread_mutex_unlock(&pObj->mutex); - taos_close(pObj); - tscTrace("connection:%p is killed", pObj); + taos_close(pObj); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 66d5e17747..5ea6941fff 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -343,8 +343,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { - tscFreeSqlObj(pSql); tscTrace("%p Async sql is automatically freed", pSql); + tscFreeSqlObj(pSql); } } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f6c1fee633..199b4150e8 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -757,8 +757,8 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); if (tscShouldFreeAsyncSqlObj(pSql)) { - tscFreeSqlObj(pSql); tscTrace("%p Async SqlObj is freed by app", pSql); + tscFreeSqlObj(pSql); } else { if (keepCmd) { tscFreeSqlResult(pSql); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 4fadad5021..f586db3d08 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -582,10 +582,12 @@ void taos_close_stream(TAOS_STREAM *handle) { tscRemoveFromStreamList(pStream, pSql); taosTmrStopA(&(pStream->pTimer)); + + tscTrace("%p stream:%p is closed", pSql, pStream); + tscFreeSqlObj(pSql); pStream->pSql = NULL; - tscTrace("%p stream:%p is closed", pSql, pStream); tfree(pStream); } } From dc3c33a820f95657ff3138d51df54e84e6e31f2b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 21:05:19 +0800 Subject: [PATCH 20/48] integrate WAL module --- src/inc/dnode.h | 2 +- src/inc/tsync.h | 6 ++-- src/inc/twal.h | 4 +-- src/vnode/main/inc/vnodeInt.h | 4 +-- src/vnode/main/src/vnodeMain.c | 54 +++++++++++++++++++++++++++------ src/vnode/main/src/vnodeWrite.c | 3 +- src/vnode/wal/src/walMain.c | 12 ++++---- src/vnode/wal/test/waltest.c | 10 ++++-- 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/inc/dnode.h b/src/inc/dnode.h index fa5e3a3b3d..db39906c68 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -42,7 +42,7 @@ void *dnodeAllocateWqueue(void *pVnode); void dnodeFreeWqueue(void *queue); void *dnodeAllocateRqueue(void *pVnode); void dnodeFreeRqueue(void *rqueue); -void dnodeSendWriteResponse(void *pVnode, void *param, int32_t code); +void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code); #ifdef __cplusplus } diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 555e0503da..ea20ceec79 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -66,11 +66,11 @@ typedef struct { // if name is null, get the file from index or after, used by master // if name is provided, get the named file at the specified index, used by unsynced node // it returns the file magic number and size, if file not there, magic shall be 0. - uint32_t (*getFileInfo)(char *name, int *index, int *size); + uint32_t (*getFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file - int (*getWalInfo)(char *name, int *index); + int (*getWalInfo)(void *ahandle, char *name, uint32_t *index); // when a forward pkt is received, call this to handle data int (*writeToCache)(void *ahandle, void *pHead, int type); @@ -94,11 +94,13 @@ int syncGetNodesRole(tsync_h shandle, SNodesRole *); extern char *syncRole[]; +//global configurable parameters extern int tsMaxSyncNum; extern int tsSyncTcpThreads; extern int tsMaxWatchFiles; extern short tsSyncPort; extern int tsMaxFwdInfo; +extern int sDebugFlag; #ifdef __cplusplus } diff --git a/src/inc/twal.h b/src/inc/twal.h index 3648f5ae29..53d4f835b0 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -40,12 +40,12 @@ typedef struct { typedef void* twal_h; // WAL HANDLE -twal_h walOpen(char *path, int max, int level); +twal_h walOpen(char *path, SWalCfg *pCfg); void walClose(twal_h); int walRenew(twal_h); int walWrite(twal_h, SWalHead *); void walFsync(twal_h); -int walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, SWalHead *pHead, int type)); +int walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, void *pHead, int type)); int walGetWalFile(twal_h, char *name, uint32_t *index); extern int wDebugFlag; diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 4d078869c4..75d3117eac 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -35,7 +35,7 @@ typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count EVnStatus status; - int role; + int8_t role; int64_t version; void *wqueue; void *rqueue; @@ -49,7 +49,7 @@ typedef struct { SWalCfg walCfg; } SVnodeObj; -int vnodeWriteToQueue(void *param, SWalHead *pHead, int type); +int vnodeWriteToQueue(void *param, void *pHead, int type); void vnodeInitWriteFp(void); void vnodeInitReadFp(void); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 1211828a47..a3c60c4387 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -29,13 +29,18 @@ #include "vnode.h" #include "vnodeInt.h" -static void *tsDnodeVnodesHash; -static void vnodeCleanUp(SVnodeObj *pVnode); -static void vnodeBuildVloadMsg(char *pNode, void * param); -static int vnodeWALCallback(void *arg); -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); -static int32_t vnodeReadCfg(SVnodeObj *pVnode); +static void *tsDnodeVnodesHash; +static void vnodeCleanUp(SVnodeObj *pVnode); +static void vnodeBuildVloadMsg(char *pNode, void * param); +static int vnodeWalCallback(void *arg); +static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); +static int32_t vnodeReadCfg(SVnodeObj *pVnode); +static int vnodeWalCallback(void *arg); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size); +static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); +static void vnodeNotifyRole(void *ahandle, int8_t role); +// module global static int tsOpennedVnodes; static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; @@ -140,14 +145,27 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->rqueue = dnodeAllocateRqueue(pVnode); sprintf(temp, "%s/wal", rootDir); - pVnode->wal = walOpen(temp, pVnode->walCfg.wals, pVnode->walCfg.commitLog); - pVnode->sync = NULL; + pVnode->wal = walOpen(temp, &pVnode->walCfg); + + SSyncInfo syncInfo; + syncInfo.vgId = pVnode->vgId; + syncInfo.vgId = pVnode->version; + syncInfo.syncCfg = pVnode->syncCfg; + sprintf(syncInfo.path, "%s/tsdb/", rootDir); + syncInfo.ahandle = pVnode; + syncInfo.getWalInfo = vnodeGetWalInfo; + syncInfo.getFileInfo = vnodeGetFileInfo; + syncInfo.writeToCache = vnodeWriteToQueue; + syncInfo.confirmForward = dnodeSendRpcWriteRsp; + syncInfo.notifyRole = vnodeNotifyRole; + // pVnode->sync = syncStart(&syncInfo);; + pVnode->events = NULL; pVnode->cq = NULL; STsdbAppH appH = {0}; appH.appH = (void *)pVnode; - appH.walCallBack = vnodeWALCallback; + appH.walCallBack = vnodeWalCallback; sprintf(temp, "%s/tsdb", rootDir); void *pTsdb = tsdbOpenRepo(temp, &appH); @@ -281,11 +299,27 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // TODO: this is a simple implement -static int vnodeWALCallback(void *arg) { +static int vnodeWalCallback(void *arg) { SVnodeObj *pVnode = arg; return walRenew(pVnode->wal); } +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) { + // SVnodeObj *pVnode = ahandle; + //tsdbGetFileInfo(pVnode->tsdb, name, index, size); + return 0; +} + +static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { + SVnodeObj *pVnode = ahandle; + return walGetWalFile(pVnode->wal, name, index); +} + +static void vnodeNotifyRole(void *ahandle, int8_t role) { + SVnodeObj *pVnode = ahandle; + pVnode->role = role; +} + static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnodeCfg->cfg.vgId); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..fc73850f40 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -248,8 +248,9 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet return code; } -int vnodeWriteToQueue(void *param, SWalHead *pHead, int type) { +int vnodeWriteToQueue(void *param, void *data, int type) { SVnodeObj *pVnode = param; + SWalHead *pHead = data; int size = sizeof(SWalHead) + pHead->len; SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index 99de5cceda..7c5602680f 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -49,18 +49,18 @@ int wDebugFlag = 135; static uint32_t walSignature = 0xFAFBFDFE; static int walHandleExistingFiles(char *path); -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int)); +static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)); static int walRemoveWalFiles(char *path); -void *walOpen(char *path, int max, int level) { +void *walOpen(char *path, SWalCfg *pCfg) { SWal *pWal = calloc(sizeof(SWal), 1); if (pWal == NULL) return NULL; pWal->fd = -1; - pWal->max = max; + pWal->max = pCfg->wals; pWal->id = 0; pWal->num = 0; - pWal->level = level; + pWal->level = pCfg->commitLog; strcpy(pWal->path, path); pthread_mutex_init(&pWal->mutex, NULL); @@ -169,7 +169,7 @@ void walFsync(void *handle) { fsync(pWal->fd); } -int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) { +int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { SWal *pWal = (SWal *)handle; int code = 0; struct dirent *ent; @@ -246,7 +246,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) { return code; } -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) { +static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)) { int code = 0; char *buffer = malloc(1024000); // size for one record diff --git a/src/vnode/wal/test/waltest.c b/src/vnode/wal/test/waltest.c index 37d1d8e84c..8e10bc11e5 100644 --- a/src/vnode/wal/test/waltest.c +++ b/src/vnode/wal/test/waltest.c @@ -21,8 +21,10 @@ int64_t ver = 0; void *pWal = NULL; -int writeToQueue(void *pVnode, SWalHead *pHead, int type) { +int writeToQueue(void *pVnode, void *data, int type) { // do nothing + SWalHead *pHead = data; + if (pHead->version > ver) ver = pHead->version; @@ -74,7 +76,11 @@ int main(int argc, char *argv[]) { taosInitLog("wal.log", 100000, 10); - pWal = walOpen(path, max, level); + SWalCfg walCfg; + walCfg.commitLog = level; + walCfg.wals = max; + + pWal = walOpen(path, &walCfg); if (pWal == NULL) { printf("failed to open wal\n"); exit(-1); From 0ba919dbacadd0a0d0890c34461e24478326b209 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 21:16:58 +0800 Subject: [PATCH 21/48] add compile option for sync --- src/dnode/CMakeLists.txt | 2 +- src/vnode/main/src/vnodeMain.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 4bd89e238e..a81f8c0c9d 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) ENDIF () IF (TD_VPEER) - TARGET_LINK_LIBRARIES(taosd balance) + TARGET_LINK_LIBRARIES(taosd balance sync) ENDIF () SET(PREPARE_ENV_CMD "prepare_env_cmd") diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index a3c60c4387..d51b9ff846 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -158,7 +158,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.writeToCache = vnodeWriteToQueue; syncInfo.confirmForward = dnodeSendRpcWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; - // pVnode->sync = syncStart(&syncInfo);; + pVnode->sync = syncStart(&syncInfo);; pVnode->events = NULL; pVnode->cq = NULL; From 4f945ec62a3519819eadfe0786d91cefe350bf36 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 21:37:39 +0800 Subject: [PATCH 22/48] return after free --- src/rpc/src/rpcMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6f86c2dd7c..52d05dc626 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -331,6 +331,7 @@ void *rpcReallocCont(void *ptr, int contLen) { char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); if (contLen == 0 ) { free(start); + return NULL; } int size = contLen + RPC_MSG_OVERHEAD; From 769ef5be3c22ad80e9513f1960eeedf5461b51de Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 22:19:46 +0800 Subject: [PATCH 23/48] remove the hard code for max number of vnodes --- src/dnode/src/dnodeMgmt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5f1e7a7a94..4f9f03f179 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -132,7 +132,7 @@ static int32_t dnodeOpenVnodes() { char vnodeDir[TSDB_FILENAME_LEN * 3]; int32_t failed = 0; - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000); + int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes = dnodeGetVnodeList(vnodeList); for (int32_t i = 0; i < numOfVnodes; ++i) { @@ -147,7 +147,7 @@ static int32_t dnodeOpenVnodes() { } static void dnodeCloseVnodes() { - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000); + int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes = dnodeGetVnodeList(vnodeList); for (int32_t i = 0; i < numOfVnodes; ++i) { From 366068e287ca12f7ddc5292b8045c54d0a17128a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 22:34:05 +0800 Subject: [PATCH 24/48] make the add/sub atomic --- src/vnode/main/src/vnodeMain.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index d51b9ff846..3211c0aff5 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -41,7 +41,7 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); // module global -static int tsOpennedVnodes; +static int32_t tsOpennedVnodes; static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; static void vnodeInit() { @@ -182,7 +182,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->status = VN_STATUS_READY; dTrace("pVnode:%p vgId:%d, vnode is opened in %s", pVnode, pVnode->vgId, rootDir); - tsOpennedVnodes++; + atomic_add_fetch_32(&tsOpennedVnodes, 1); return TSDB_CODE_SUCCESS; } @@ -223,8 +223,8 @@ void vnodeRelease(void *pVnodeRaw) { dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId); free(pVnode); - tsOpennedVnodes--; - if (tsOpennedVnodes <= 0) { + int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1); + if (count <= 0) { taosCleanUpIntHash(tsDnodeVnodesHash); vnodeModuleInit = PTHREAD_ONCE_INIT; tsDnodeVnodesHash = NULL; From 4926dddebb203dc7016bece9e59ac2c33ebd95af Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 23:14:44 +0800 Subject: [PATCH 25/48] fix kill script to avoid others' path with taosdata be killed by mistake. --- tests/script/sh/stop_dnodes.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index acdfbe9e6a..fe4205824a 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -6,8 +6,8 @@ if [ -n "$PID" ]; then sudo systemctl stop taosd fi -PID=`ps -ef|grep taosd | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` if [ -n "$PID" ]; then echo sudo kill -9 $PID - sudo kill -9 $PID + sudo pkill taosd fi From 8f54cb6768ab1352f92a05e4c9fd843e433cf1a3 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 23:27:10 +0800 Subject: [PATCH 26/48] fix crash while drop databases --- src/vnode/main/src/vnodeMain.c | 5 ++--- src/vnode/wal/src/walMain.c | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 9512cf4c6c..13210e496f 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -36,7 +36,7 @@ static int vnodeWALCallback(void *arg); static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); static int32_t vnodeReadCfg(SVnodeObj *pVnode); -static int tsOpennedVnodes; +static int32_t tsOpennedVnodes; static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; static void vnodeInit() { @@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } int32_t vnodeDrop(int32_t vgId) { - - SVnodeObj *pVnode = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, vgId); + SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); if (pVnode == NULL) { dTrace("vgId:%d, failed to drop, vgId not exist", vgId); return TSDB_CODE_INVALID_VGROUP_ID; diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index 99de5cceda..504e370279 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -80,7 +80,8 @@ void *walOpen(char *path, int max, int level) { } void walClose(void *handle) { - + if (handle == NULL) return; + SWal *pWal = (SWal *)handle; close(pWal->fd); From 4f350284abafcd954ea36b04815c3c2cb539ac01 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 23:36:37 +0800 Subject: [PATCH 27/48] [TD-82] first edition of balance --- src/dnode/src/dnodeMain.c | 1 - src/inc/tcluster.h | 10 +++++----- src/mnode/src/mgmtDnode.c | 14 +++++++------- src/mnode/src/mgmtSdb.c | 4 ++-- src/mnode/src/mgmtVgroup.c | 2 +- src/util/inc/tmodule.h | 2 -- src/vnode/main/src/vnodeMain.c | 1 + 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index d95a344058..01e2c4dfcc 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -220,7 +220,6 @@ static int32_t dnodeInitStorage() { sprintf(tsMnodeDir, "%s/mnode", dataDir); sprintf(tsVnodeDir, "%s/vnode", dataDir); sprintf(tsDnodeDir, "%s/dnode", dataDir); - mkdir(tsMnodeDir, 0755); mkdir(tsVnodeDir, 0755); mkdir(tsDnodeDir, 0755); diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h index 60436ee743..28467ca683 100644 --- a/src/inc/tcluster.h +++ b/src/inc/tcluster.h @@ -26,11 +26,11 @@ extern "C" { struct _dnode_obj; -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_DROPING, - TSDB_DN_STATUS_BALANCING, - TSDB_DN_STATUS_READY +enum _TAOS_DN_STATUS { + TAOS_DN_STATUS_OFFLINE, + TAOS_DN_STATUS_DROPPING, + TAOS_DN_STATUS_BALANCING, + TAOS_DN_STATUS_READY }; int32_t clusterInit(); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index b60d1cb0ba..e6d186ac1b 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -48,7 +48,7 @@ int32_t clusterInitDnodes() { tsDnodeObj.publicIp = inet_addr(tsPublicIp); tsDnodeObj.createdTime = taosGetTimestampMs(); tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes; - tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE; + tsDnodeObj.status = TAOS_DN_STATUS_OFFLINE; tsDnodeObj.lastReboot = taosGetTimestampSec(); sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId); @@ -203,9 +203,9 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtReleaseVgroup(pVgroup); } - if (pDnode->status != TSDB_DN_STATUS_READY) { + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { mTrace("dnode:%d, from offline to online", pDnode->dnodeId); - pDnode->status = TSDB_DN_STATUS_READY; + pDnode->status = TAOS_DN_STATUS_READY; balanceNotify(); } @@ -673,10 +673,10 @@ static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, char* clusterGetDnodeStatusStr(int32_t dnodeStatus) { switch (dnodeStatus) { - case TSDB_DN_STATUS_OFFLINE: return "offline"; - case TSDB_DN_STATUS_DROPING: return "dropping"; - case TSDB_DN_STATUS_BALANCING: return "balancing"; - case TSDB_DN_STATUS_READY: return "ready"; + case TAOS_DN_STATUS_OFFLINE: return "offline"; + case TAOS_DN_STATUS_DROPPING: return "dropping"; + case TAOS_DN_STATUS_BALANCING: return "balancing"; + case TAOS_DN_STATUS_READY: return "ready"; default: return "undefined"; } } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 06cb9dca5c..60614157f1 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -435,7 +435,7 @@ void sdbIncRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - if (0) { + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } } @@ -446,7 +446,7 @@ void sdbDecRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0) { + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } int8_t* updateEnd = pRow + pTable->refCountPos - 1; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index a0b80c8ea3..4c969124a0 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -372,7 +372,7 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { } clusterReleaseDnode(pDnode); - if (pDnode->status == TSDB_DN_STATUS_OFFLINE) { + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { return "offline"; } diff --git a/src/util/inc/tmodule.h b/src/util/inc/tmodule.h index 9f8ef147f8..f5ad6a6b60 100644 --- a/src/util/inc/tmodule.h +++ b/src/util/inc/tmodule.h @@ -28,8 +28,6 @@ enum _module { TSDB_MOD_MGMT, TSDB_MOD_HTTP, TSDB_MOD_MONITOR, - TSDB_MOD_DCLUSTER, - TSDB_MOD_MSTORAGE, TSDB_MOD_MAX }; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 786e6f63a0..4c2d86a518 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -266,6 +266,7 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { pLoad->vgId = htonl(pVnode->vgId); pLoad->vnode = htonl(pVnode->vgId); pLoad->status = pVnode->status; + pLoad->role = pVnode->role; } static void vnodeCleanUp(SVnodeObj *pVnode) { From b37185e5403c514d5b80208bc92ae53408c6ccc8 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 04:30:33 +0000 Subject: [PATCH 28/48] fix part of mem-leak --- src/common/src/dataformat.c | 2 +- src/vnode/main/src/vnodeWrite.c | 3 +++ src/vnode/tsdb/src/tsdbFile.c | 1 + src/vnode/tsdb/src/tsdbMain.c | 8 ++++++-- src/vnode/tsdb/src/tsdbMeta.c | 13 ++++++++++++- src/vnode/tsdb/src/tsdbMetaFile.c | 1 + 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index bff041df1b..45850d1788 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -141,7 +141,7 @@ STSchema *tdDupSchema(STSchema *pSchema) { * Free the SSchema object created by tdNewSchema or tdDupSchema */ void tdFreeSchema(STSchema *pSchema) { - if (pSchema == NULL) free(pSchema); + if (pSchema != NULL) free(pSchema); } /** diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..648e7197b0 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -25,6 +25,7 @@ #include "dataformat.h" #include "vnode.h" #include "vnodeInt.h" +#include "tutil.h" static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *, SRspRet *); static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *); @@ -157,6 +158,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe void *pTsdb = vnodeGetTsdb(pVnode); code = tsdbCreateTable(pTsdb, &tCfg); + tfree(pDestSchema); + dTrace("pVnode:%p vgId:%d, table:%s is created, result:%x", pVnode, pVnode->vgId, pTable->tableId, code); return code; } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index bd6699eb84..8bdfe63002 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -62,6 +62,7 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { // TODO } } + closedir(dir); return pFileH; } diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 61f4995e43..aa185f4bf2 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -268,6 +268,9 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) { tsdbFreeCache(pRepo->tsdbCache); + tfree(pRepo->rootDir); + tfree(pRepo); + return 0; } @@ -847,6 +850,7 @@ static void *tsdbCommitData(void *arg) { tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); + tdListFree(pCache->imem->list); free(pCache->imem); pCache->imem = NULL; pRepo->commit = 0; @@ -1125,11 +1129,11 @@ static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsTo *len += pCompCol->len; } - if (pCompData == NULL) free((void *)pCompData); + tfree(pCompData); return 0; _err: - if (pCompData == NULL) free((void *)pCompData); + tfree(pCompData); return -1; } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0c6fc61701..22f7357135 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -312,6 +312,14 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { // return 0; // } +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + } + + free(pMemTable); +} + static int tsdbFreeTable(STable *pTable) { // TODO: finish this function if (pTable->type == TSDB_CHILD_TABLE) { @@ -323,7 +331,10 @@ static int tsdbFreeTable(STable *pTable) { // Free content if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) { tSkipListDestroy(pTable->pIndex); - } + } + + tsdbFreeMemTable(pTable->mem); + tsdbFreeMemTable(pTable->imem); free(pTable); return 0; diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c index d3cff1772c..6821fc2d98 100644 --- a/src/vnode/tsdb/src/tsdbMetaFile.c +++ b/src/vnode/tsdb/src/tsdbMetaFile.c @@ -177,6 +177,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) { close(mfh->fd); taosHashCleanup(mfh->map); + tfree(mfh); } static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { From 64e59d94630e4a56464cbbdead2eff52041b12a9 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 14:38:00 +0800 Subject: [PATCH 29/48] fix log print error --- src/util/src/tstring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/src/tstring.c b/src/util/src/tstring.c index 7aca939f47..5a134df129 100644 --- a/src/util/src/tstring.c +++ b/src/util/src/tstring.c @@ -110,6 +110,8 @@ char *taosMsg[] = { "", "", "", + "", + "", "", //90 "config-table", From 3899ae87075e4542357313df60def450ebfab045 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 9 Apr 2020 15:57:32 +0800 Subject: [PATCH 30/48] fix the bug in sync integration --- src/rpc/src/rpcMain.c | 5 ++++- src/vnode/main/src/vnodeMain.c | 4 ++-- src/vnode/main/src/vnodeWrite.c | 5 +---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6f86c2dd7c..a241a8ce07 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -934,6 +934,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps); rpcSendReqToServer(pRpc, pContext); + } else if (pHead->code == TSDB_CODE_NOT_READY) { + pConn->pContext->code = pHead->code; + rpcProcessConnError(pConn->pContext, NULL); } else { rpcNotifyClient(pContext, &rpcMsg); } @@ -1078,7 +1081,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort, - pHead->code, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index bbc17c3440..f5a780214b 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -148,7 +148,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; - syncInfo.vgId = pVnode->version; + syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; sprintf(syncInfo.path, "%s/tsdb/", rootDir); syncInfo.ahandle = pVnode; @@ -370,7 +370,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; - pVnode->syncCfg.arbitratorIp = arbitratorIp; + pVnode->syncCfg.arbitratorIp = 0; int32_t quorum = -1; num = fscanf(fp, "%s %d", option[0], &quorum); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 849f5f95da..1504c92151 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -77,11 +77,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); if (code < 0) return code; -/* forward - if (pVnode->replica > 1 && pVnode->role == TAOS_SYNC_ROLE_MASTER) { + if (pVnode->syncCfg.replica > 1) code = syncForwardToPeer(pVnode->sync, pHead, item); - } -*/ return code; } From d5de8dc1e3ca4843994f0417d674fba34c3048a8 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 16:15:21 +0800 Subject: [PATCH 31/48] [TD-82] fix error while balance --- src/inc/tcluster.h | 1 + src/mnode/src/mgmtDnode.c | 6 +++--- src/mnode/src/mgmtSdb.c | 5 +++++ src/util/src/tglobalcfg.c | 7 +++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h index 28467ca683..769a819b90 100644 --- a/src/inc/tcluster.h +++ b/src/inc/tcluster.h @@ -36,6 +36,7 @@ enum _TAOS_DN_STATUS { int32_t clusterInit(); void clusterCleanUp(); char* clusterGetDnodeStatusStr(int32_t dnodeStatus); +bool clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType); int32_t clusterInitDnodes(); void clusterCleanupDnodes(); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index e6d186ac1b..e4ab114090 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -323,8 +323,7 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, char ipstr[32]; while (numOfRows < rows) { - clusterReleaseDnode(pDnode); - pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, &pDnode); if (pDnode == NULL) break; cols = 0; @@ -366,13 +365,14 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, #endif numOfRows++; + clusterReleaseDnode(pDnode); } pShow->numOfReads += numOfRows; return numOfRows; } -static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { +bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { uint32_t status = pDnode->moduleStatus & (1 << moduleType); return status > 0; } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 60614157f1..ef2a23cd12 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -520,6 +520,11 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { if (pTable->keyType == SDB_KEY_TYPE_AUTO) { *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + + // let vgId increase from 2 + if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } } pTable->version++; sdbVersion++; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 8a0d66068e..88a3be0e02 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -111,7 +111,12 @@ short tsDaysPerFile = 10; int tsDaysToKeep = 3650; int tsReplications = TSDB_REPLICA_MIN_NUM; +#ifdef _MPEER int tsNumOfMPeers = 3; +#else +int tsNumOfMPeers = 1; +#endif + int tsMaxShellConns = 2000; int tsMaxTables = 100000; @@ -552,9 +557,11 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "tblocks", &tsNumOfBlocksPerMeter, TSDB_CFG_VTYPE_SHORT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 32, 4096, 0, TSDB_CFG_UTYPE_NONE); +#ifdef _MPEER tsInitConfigOption(cfg++, "numOfMPeers", &tsNumOfMPeers, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, 1, 3, 0, TSDB_CFG_UTYPE_NONE); +#endif tsInitConfigOption(cfg++, "balanceInterval", &tsBalanceStartInterval, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, 1, 30000, 0, TSDB_CFG_UTYPE_NONE); From 7b5c13a857950b81233c32ea86968d685822bfba Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 9 Apr 2020 17:08:11 +0800 Subject: [PATCH 32/48] add query in shared library [TD-96] --- src/client/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 55fa45475a..ecbef79ae4 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -22,7 +22,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) # generate dynamic library (*.so) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common trpc tutil pthread m rt) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) #set version of .so From 55fa8b060ab1350a35fa0b06a522ffa050f8591b Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 17:49:35 +0800 Subject: [PATCH 33/48] fix compile while intergreate sync module --- src/client/src/tscServer.c | 3 ++- src/vnode/main/src/vnodeMain.c | 26 ++++++++++++++++---------- tests/script/general/table/basic2.sim | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3042e3e59b..46c7dd589b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -517,7 +517,8 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pRetrieveMsg->free = htons(pQueryInfo->type); pMsg += sizeof(pQueryInfo->type); - pRetrieveMsg->header.vgId = htonl(1); + STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); pMsg += sizeof(SRetrieveTableMsg); pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 987fd69a70..182b4b6257 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -28,6 +28,7 @@ #include "vnode.h" #include "vnodeInt.h" +static int32_t tsOpennedVnodes; static void *tsDnodeVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); static void vnodeBuildVloadMsg(char *pNode, void * param); @@ -41,6 +42,11 @@ static void vnodeNotifyRole(void *ahandle, int8_t role); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; +#ifndef _VPEER +tsync_h syncStart(SSyncInfo *info) { return NULL; } +int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; } +#endif + static void vnodeInit() { vnodeInitWriteFp(); vnodeInitReadFp(); @@ -56,7 +62,6 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { pthread_once(&vnodeModuleInit, vnodeInit); SVnodeObj *pTemp = (SVnodeObj *)taosGetIntHashData(tsDnodeVnodesHash, pVnodeCfg->cfg.vgId); - if (pTemp != NULL) { dPrint("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp); return TSDB_CODE_SUCCESS; @@ -106,12 +111,13 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } int32_t vnodeDrop(int32_t vgId) { - SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) { dTrace("vgId:%d, failed to drop, vgId not exist", vgId); return TSDB_CODE_INVALID_VGROUP_ID; } + SVnodeObj *pVnode = *ppVnode; dTrace("pVnode:%p vgId:%d, vnode will be dropped", pVnode, pVnode->vgId); pVnode->status = TAOS_VN_STATUS_DELETING; vnodeCleanUp(pVnode); @@ -183,10 +189,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } int32_t vnodeClose(int32_t vgId) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) return 0; - SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) return 0; - + SVnodeObj *pVnode = *ppVnode; dTrace("pVnode:%p vgId:%d, vnode will be closed", pVnode, pVnode->vgId); pVnode->status = TAOS_VN_STATUS_CLOSING; vnodeCleanUp(pVnode); @@ -228,13 +234,13 @@ void vnodeRelease(void *pVnodeRaw) { } void *vnodeGetVnode(int32_t vgId) { - SVnodeObj *pVnode = *(SVnodeObj **) taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_INVALID_VGROUP_ID; - return NULL; + assert(false); } - return pVnode; + return *ppVnode; } void *vnodeAccquireVnode(int32_t vgId) { diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index 7701ca1c1f..73b17471e5 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -29,7 +29,7 @@ if $data03 != 1 then endi sql show d1.vgroups -if $data00 != 1 then +if $data00 != 2 then return -1 endi From 0ba8636c848c2ce1c5f2c23bcd1767b94e6184fb Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 17:52:52 +0800 Subject: [PATCH 34/48] add some scripts --- tests/script/general/db/basic4.sim | 4 ++-- tests/script/general/db/basic5.sim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/db/basic4.sim b/tests/script/general/db/basic4.sim index f620112cfb..deac9d47b4 100644 --- a/tests/script/general/db/basic4.sim +++ b/tests/script/general/db/basic4.sim @@ -36,7 +36,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 4 then @@ -75,7 +75,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 3 then diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index ec619389bd..ebb7d3c967 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -36,7 +36,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 4 then From 37b293720337b7cf443b4cec4e889b0dcabf9d64 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 9 Apr 2020 19:08:26 +0800 Subject: [PATCH 35/48] fix stop_dnodes.sh with postpone socket cleanup and same timestamp issue in general/table/basic2.sim. --- tests/script/basicSuite.sim | 4 ++-- tests/script/general/table/basic1.sim | 1 - tests/script/general/table/basic2.sim | 5 +++-- tests/script/sh/stop_dnodes.sh | 8 +++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index d5892d8682..290472b15d 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -2,7 +2,7 @@ run general/table/basic1.sim -#run general/table/basic2.sim -#run general/table/basic3.sim +run general/table/basic2.sim +run general/table/basic3.sim ################################## diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index fb924ac453..a8dfb0f21a 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -1,6 +1,5 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system ifconfig system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index 7701ca1c1f..2afd6baf36 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -43,11 +43,12 @@ sql_error insert into d1.n2 values(now, 1) print =============== insert data2 sql insert into d1.n3 values(now, 1) -sql insert into d1.n3 values(now, 2) -sql insert into d1.n3 values(now, 3) +sql insert into d1.n3 values(now+1s, 2) +sql insert into d1.n3 values(now+2s, 3) print =============== query data sql select * from d1.n3 +print $rows if $rows != 3 then return -1 endi diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index fe4205824a..a873eb5654 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -7,7 +7,9 @@ if [ -n "$PID" ]; then fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -if [ -n "$PID" ]; then +while [ -n "$PID" ]; do echo sudo kill -9 $PID - sudo pkill taosd -fi + sudo pkill -9 taosd + sudo fuser -k -n tcp 6030 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done From 945f3ac25ec4888b0978d5fab403ed0ad7aa379a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 9 Apr 2020 23:31:52 +0800 Subject: [PATCH 36/48] NOT_READY is not processed in a right way --- src/rpc/src/rpcMain.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a241a8ce07..b9602696d6 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -725,10 +725,6 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { return TSDB_CODE_INVALID_RESPONSE_TYPE; } - if (pHead->code == TSDB_CODE_NOT_READY) { - return TSDB_CODE_ALREADY_PROCESSED; - } - taosTmrStopA(&pConn->pTimer); pConn->retry = 0; @@ -935,8 +931,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps); rpcSendReqToServer(pRpc, pContext); } else if (pHead->code == TSDB_CODE_NOT_READY) { - pConn->pContext->code = pHead->code; - rpcProcessConnError(pConn->pContext, NULL); + pContext->code = pHead->code; + rpcProcessConnError(pContext, NULL); } else { rpcNotifyClient(pContext, &rpcMsg); } @@ -1101,7 +1097,7 @@ static void rpcProcessConnError(void *param, void *id) { tTrace("%s connection error happens", pRpc->label); - if ( pContext->numOfTry >= pContext->ipSet.numOfIps ) { + if ( pContext->numOfTry >= pContext->ipSet.numOfIps*2 ) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.handle = pContext->ahandle; rpcMsg.code = pContext->code; From 358dad34ba4a2da1e47a2d030426e6be7d50be9a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 08:40:08 +0800 Subject: [PATCH 37/48] a minor change --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index b9602696d6..73ea23cbf8 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1097,7 +1097,7 @@ static void rpcProcessConnError(void *param, void *id) { tTrace("%s connection error happens", pRpc->label); - if ( pContext->numOfTry >= pContext->ipSet.numOfIps*2 ) { + if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.handle = pContext->ahandle; rpcMsg.code = pContext->code; From be2f915fb44e09af743d9895db2ef158bc5d7591 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 08:59:17 +0800 Subject: [PATCH 38/48] option index is out of range --- src/vnode/main/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index f5a780214b..7c92ad4f4f 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -344,7 +344,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { // TODO: this is a simple implement static int32_t vnodeReadCfg(SVnodeObj *pVnode) { - char option[3][16] = {0}; + char option[4][16] = {0}; char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnode->vgId); From 909876fa05f157bdbef59b172b1678297ec78891 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 11:08:59 +0800 Subject: [PATCH 39/48] change errno index --- src/inc/taoserror.h | 138 ++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 0624af45c3..5cd38f6d6b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -76,89 +76,89 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE, 0, 28, "node offline") TAOS_DEFINE_ERROR(TSDB_CODE_NETWORK_UNAVAIL, 0, 29, "network unavailable") // db & user -TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 30, "db not selected") -TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST, 0, 31, "database aleady exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB, 0, 32, "invalid database") -TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN, 0, 33, "monitor db forbidden") -TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST, 0, 34, "user already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER, 0, 35, "invalid user") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS, 0, 36, "invalid password") +TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 100, "db not selected") +TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST, 0, 101, "database aleady exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB, 0, 102, "invalid database") +TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN, 0, 103, "monitor db forbidden") +TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST, 0, 104, "user already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER, 0, 105, "invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS, 0, 106, "invalid password") // table -TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 41, "table already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID, 0, 42, "invalid table id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 43, "invalid table typee") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 44, "invalid table name") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 45, "no super table") // operation only available for super table -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 46, "not active table") -TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 47, "table id mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 200, "table already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID, 0, 201, "invalid table id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 202, "invalid table typee") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table name") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table") +TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") // dnode & mnode -TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 50, "no enough dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 51, "dnode already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 52, "dnode not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 53, "no master") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 54, "no remove master") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 55, "invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 56, "invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 57, "invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 58, "sdb error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 300, "no enough dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 301, "dnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 302, "dnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 303, "no master") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 304, "no remove master") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 305, "invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 306, "invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 307, "invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 308, "sdb error") // acct -TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 60, "accounts already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 61, "invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 62, "invalid account parameter") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 63, "too many accounts") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 64, "too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 65, "too many tables") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 66, "too many databases") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 67, "not enough time series") +TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 400, "accounts already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 401, "invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 402, "invalid account parameter") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 403, "too many accounts") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 404, "too many users") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 405, "too many tables") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 406, "too many databases") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 407, "not enough time series") // grant -TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 70, "auth failure") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 71, "no rights") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 72, "no write access") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 73, "no read access") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 74, "grant expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 75, "grant dnode limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 76, "grant account limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 77, "grant timeseries limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 78, "grant db limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 79, "grant user limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 80, "grant conn limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 81, "grant stream limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 82, "grant speed limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 83, "grant storage limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 84, "grant query time limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 85, "grant cpu limited") +TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 400, "auth failure") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 401, "no rights") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 402, "no write access") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 403, "no read access") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 404, "grant expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 405, "grant dnode limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 406, "grant account limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 407, "grant timeseries limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 408, "grant db limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 409, "grant user limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 410, "grant conn limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 411, "grant stream limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 412, "grant speed limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 413, "grant storage limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 414, "grant query time limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 415, "grant cpu limited") // server -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 90, "invalid vgroup id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 91, "invalid vnode id") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 92, "not active vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 93, "vg init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 94, "server no diskspace") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 95, "server out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 96, "no disk permissions") -TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 97, "file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 98, "memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 500, "invalid vgroup id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 501, "invalid vnode id") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 502, "not active vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 503, "vg init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 504, "server no diskspace") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 505, "server out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 506, "no disk permissions") +TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 507, "file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 508, "memory corrupted") // client -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 101, "invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 102, "client out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 103, "client no disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 104, "invalid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 105, "invalid sql") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 106, "query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 107, "invalid query message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 108, "sorted res too many") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 109, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 110, "query cancelled") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 111, "invalid ie") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 112, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 601, "invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 602, "client out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 603, "client no disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 604, "invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 605, "invalid sql") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 606, "query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 607, "invalid query message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 608, "sorted res too many") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 609, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 610, "query cancelled") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 611, "invalid ie") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 612, "invalid value") // others -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 120, "invalid file format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 700, "invalid file format") #ifdef TAOS_ERROR_C From b17e7329388899107763295102dc965269c2b298 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 11:40:46 +0800 Subject: [PATCH 40/48] make output clear and add general/db testcases. --- .travis.yml | 13 +++++++++---- tests/script/basicSuite.sim | 7 ++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5cff3bc72b..63c62d1a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,14 +51,19 @@ script: case $TRAVIS_OS_NAME in linux) cd ../tests/script - sudo ./test.sh 2>&1 | tee out.txt - cat out.txt - grep success out.txt + sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt + total_success=`grep success out.txt | wc -l` + + if [ "$total_success" -gt "0" ]; then + total_success=`expr $total_success - 1` + fi + echo "Total $total_success success" - grep failed out.txt + total_failed=`grep failed out.txt | wc -l` echo "Total $total_failed failed" + if [ "$total_failed" -ne "0" ]; then exit $total_failed fi diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 290472b15d..440ec5592b 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,8 +1,13 @@ ################################# - run general/table/basic1.sim run general/table/basic2.sim run general/table/basic3.sim +run general/db/basic1.sim +run general/db/basic2.sim +run general/db/basic3.sim +run general/db/basic4.sim +run general/db/basic5.sim + ################################## From 0bb86882a5a5682291028c07cb3932924a6fbac1 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 11:48:17 +0800 Subject: [PATCH 41/48] Update administrator-ch.md --- documentation/webdocs/markdowndocs/administrator-ch.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index 35beb610f2..cbccc03cbb 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -102,6 +102,11 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 日志文件目录，客户端和服务器的运行日志将写入该目录。 +**shellActivityTimer** +- 默认值：3 + +系统在服务端保持结果集的最长时间，范围[1-120]。 + **maxUsers** - 默认值：10,000 @@ -411,4 +416,4 @@ TDengine启动后，会自动创建一个监测数据库`LOG`，并自动将服 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项`monitor`将其关闭或打开。 -[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData \ No newline at end of file +[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData From 4ecb5e0f7fd29d2e9e70efe6b623bc1ecf75de40 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 11:51:00 +0800 Subject: [PATCH 42/48] refCount may be negative is vnodeRelease is called appropriately add assert there in case it happens --- src/vnode/main/src/vnodeMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 182b4b6257..8d36f59da0 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -202,11 +202,13 @@ int32_t vnodeClose(int32_t vgId) { void vnodeRelease(void *pVnodeRaw) { SVnodeObj *pVnode = pVnodeRaw; + int32_t vgId = pVnode->vgId; int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + assert(refCount >= 0); if (refCount > 0) { - dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, pVnode->vgId, refCount); + dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, vgId, refCount); return; } From 8c86feba0a60440419789fac1d3f66404de51e91 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 12:21:56 +0800 Subject: [PATCH 43/48] fix print typo in general/db/basic5.sim. --- tests/script/general/db/basic5.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index ebb7d3c967..55a42899ec 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -46,7 +46,7 @@ if $data02 != ready then return -1 endi -print =============== drop table +print =============== drop database sql drop database d1 sql show databases From ed2254dbcc13a1e578fba06ce371549358c679c3 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 14:09:50 +0800 Subject: [PATCH 44/48] [TD-114] let create table wait until master selected --- src/inc/mnode.h | 5 +++- src/inc/taosmsg.h | 1 - src/mnode/inc/mgmtShell.h | 1 + src/mnode/inc/mgmtVgroup.h | 1 + src/mnode/src/mgmtDnode.c | 13 +++++---- src/mnode/src/mgmtProfile.c | 2 ++ src/mnode/src/mgmtShell.c | 9 +++++++ src/mnode/src/mgmtTable.c | 47 ++++++++++++++++++++++----------- src/mnode/src/mgmtVgroup.c | 13 +++++++++ src/vnode/main/src/vnodeMain.c | 1 - src/vnode/main/src/vnodeWrite.c | 3 ++- tests/script/tmp/prepare.sim | 3 ++- 12 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 903b172068..dec9292209 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -149,7 +149,8 @@ typedef struct _vg_obj { int32_t lbDnodeId; int32_t lbTime; int8_t status; - int8_t reserved[14]; + int8_t inUse; + int8_t reserved[13]; int8_t updateEnd[1]; int32_t refCount; struct _vg_obj *prev, *next; @@ -243,6 +244,8 @@ typedef struct { int8_t received; int8_t successed; int8_t expected; + int8_t retry; + int8_t maxRetry; int32_t contLen; int32_t code; void *ahandle; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5f07d6ec99..f5168a2c9e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -510,7 +510,6 @@ typedef struct SRetrieveTableRsp { typedef struct { int32_t vgId; - int32_t vnode; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; diff --git a/src/mnode/inc/mgmtShell.h b/src/mnode/inc/mgmtShell.h index b92e9de1f4..171c93a390 100644 --- a/src/mnode/inc/mgmtShell.h +++ b/src/mnode/inc/mgmtShell.h @@ -31,6 +31,7 @@ void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); void mgmtAddToShellQueue(SQueuedMsg *queuedMsg); +void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg); void mgmtSendSimpleResp(void *thandle, int32_t code); #ifdef __cplusplus diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 83e003e063..072c616f3d 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -37,6 +37,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb); void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup); void mgmtUpdateVgroup(SVgObj *pVgroup); +void mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload); void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index e4ab114090..2f4abc5cfe 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -189,18 +189,21 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { - pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); - pDnode->vload[j].totalStorage = htobe64(pStatus->load[j].totalStorage); - pDnode->vload[j].compStorage = htobe64(pStatus->load[j].compStorage); - pDnode->vload[j].pointsWritten = htobe64(pStatus->load[j].pointsWritten); + SVnodeLoad *pVload = &pStatus->load[j]; + pDnode->vload[j].vgId = htonl(pVload->vgId); + pDnode->vload[j].totalStorage = htobe64(pVload->totalStorage); + pDnode->vload[j].compStorage = htobe64(pVload->compStorage); + pDnode->vload[j].pointsWritten = htobe64(pVload->pointsWritten); SVgObj *pVgroup = mgmtGetVgroup(pDnode->vload[j].vgId); if (pVgroup == NULL) { SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pDnode->vload[j].vgId); mgmtSendDropVnodeMsg(pDnode->vload[j].vgId, &ipSet, NULL); + } else { + mgmtUpdateVgroupStatus(pVgroup, pDnode->dnodeId, pVload); + mgmtReleaseVgroup(pVgroup); } - mgmtReleaseVgroup(pVgroup); } if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 0360432971..2b22fae47a 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -806,6 +806,8 @@ void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) { pDestMsg->msgType = pSrcMsg->msgType; pDestMsg->pCont = pSrcMsg->pCont; pDestMsg->contLen = pSrcMsg->contLen; + pDestMsg->retry = pSrcMsg->retry; + pDestMsg->maxRetry= pSrcMsg->maxRetry; pDestMsg->pUser = pSrcMsg->pUser; pDestMsg->usePublicIp = pSrcMsg->usePublicIp; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 7b6a2654ae..880c6d0c10 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -128,6 +128,15 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { taosScheduleTask(tsMgmtTranQhandle, &schedMsg); } +static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) { + mgmtAddToShellQueue(param); +} + +void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) { + void *unUsed = NULL; + taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed); +} + static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { if (rpcMsg == NULL || rpcMsg->pCont == NULL) { return; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index a14bdd058f..b536bb5ac9 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -540,7 +540,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; pMsg->pTable = mgmtGetTable(pCreate->tableId); - if (pMsg->pTable != NULL) { + if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pCreate->igExists) { mTrace("table:%s, is already exist", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); @@ -1300,7 +1300,11 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + if (pMsg->retry == 0) { + pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + } else { + pMsg->pTable = mgmtGetTable(pCreate->tableId); + } if (pMsg->pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); return; @@ -1315,6 +1319,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); newMsg->ahandle = pMsg->pTable; + newMsg->maxRetry = 5; mgmtIncTableRef(pMsg->pTable); SRpcMsg rpcMsg = { .handle = newMsg, @@ -1737,30 +1742,40 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { queueMsg->received++; SChildTableObj *pTable = queueMsg->ahandle; - mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, - rpcMsg->handle, tstrerror(rpcMsg->code)); + mTrace("table:%s, create table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - sdbDeleteRow(&oper); + if (queueMsg->retry++ < queueMsg->maxRetry) { + mTrace("table:%s, create table rsp received, retry:%d thandle:%p result:%s", pTable->info.tableId, + queueMsg->retry, queueMsg->thandle, tstrerror(rpcMsg->code)); + mgmtDealyedAddToShellQueue(queueMsg); + } else { + mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId, + queueMsg->thandle, tstrerror(rpcMsg->code)); + + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); - mError("table:%s, failed to create in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtFreeQueuedMsg(queueMsg); + } } else { - mTrace("table:%s, created in dnode", pTable->info.tableId); + mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + tstrerror(rpcMsg->code)); + if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { mTrace("table:%s, start to get meta", pTable->info.tableId); - mgmtAddToShellQueue(mgmtCloneQueuedMsg(queueMsg)); + mgmtAddToShellQueue(queueMsg); } else { mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtFreeQueuedMsg(queueMsg); } } - - mgmtFreeQueuedMsg(queueMsg); } // not implemented yet diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 4c969124a0..6b27fbbc83 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "tlog.h" #include "tbalance.h" +#include "tsync.h" #include "tcluster.h" #include "mnode.h" #include "mgmtDb.h" @@ -209,6 +210,18 @@ void mgmtUpdateVgroup(SVgObj *pVgroup) { mgmtSendCreateVgroupMsg(pVgroup, NULL); } +void mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload) { + if (pVload->role == TAOS_SYNC_ROLE_MASTER) { + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + if (pVgid->dnodeId == dnodeId) { + pVgroup->inUse = i; + break; + } + } + } +} + SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 182b4b6257..8db00ed1b1 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -285,7 +285,6 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); - pLoad->vnode = htonl(pVnode->vgId); pLoad->status = pVnode->status; pLoad->role = pVnode->role; } diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 5e03305487..6f06585fa9 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -57,7 +57,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; - // if (pVnode->replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) + if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) + return TSDB_CODE_NO_MASTER; // assign version pVnode->version++; diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 1f0b893e6d..731b707434 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -1,3 +1,4 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 \ No newline at end of file +system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 +system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 \ No newline at end of file From b723946960c88ce5cb4a7be2a778a14ea0b7d08f Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 14:23:25 +0800 Subject: [PATCH 45/48] [TD-114] let replica 3 can create table --- src/vnode/main/src/vnodeWrite.c | 92 ++++++++++++++------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 6f06585fa9..1692bc14eb 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -110,36 +110,27 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe int32_t code = 0; dTrace("pVnode:%p vgId:%d, table:%s, start to create", pVnode, pVnode->vgId, pTable->tableId); - pTable->numOfColumns = htons(pTable->numOfColumns); - pTable->numOfTags = htons(pTable->numOfTags); - pTable->sid = htonl(pTable->sid); - pTable->sversion = htonl(pTable->sversion); - pTable->tagDataLen = htonl(pTable->tagDataLen); - pTable->sqlDataLen = htonl(pTable->sqlDataLen); - pTable->uid = htobe64(pTable->uid); - pTable->superTableUid = htobe64(pTable->superTableUid); - pTable->createdTime = htobe64(pTable->createdTime); + int16_t numOfColumns = htons(pTable->numOfColumns); + int16_t numOfTags = htons(pTable->numOfTags); + int32_t sid = htonl(pTable->sid); + uint64_t uid = htobe64(pTable->uid); SSchema *pSchema = (SSchema *) pTable->data; - int totalCols = pTable->numOfColumns + pTable->numOfTags; - for (int i = 0; i < totalCols; i++) { - pSchema[i].colId = htons(pSchema[i].colId); - pSchema[i].bytes = htons(pSchema[i].bytes); - } - + int32_t totalCols = numOfColumns + numOfTags; + STableCfg tCfg; - tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid); + tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid); - STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns); - for (int i = 0; i < pTable->numOfColumns; i++) { - tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + STSchema *pDestSchema = tdNewSchema(numOfColumns); + for (int i = 0; i < numOfColumns; i++) { + tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetSchema(&tCfg, pDestSchema, false); - if (pTable->numOfTags != 0) { - STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags); - for (int i = pTable->numOfColumns; i < totalCols; i++) { - tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + if (numOfTags != 0) { + STSchema *pDestTagSchema = tdNewSchema(numOfTags); + for (int i = numOfColumns; i < totalCols; i++) { + tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false); @@ -147,9 +138,9 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe int accumBytes = 0; SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema); - for (int i = 0; i < pTable->numOfTags; i++) { + for (int i = 0; i < numOfTags; i++) { tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i); - accumBytes += pSchema[i + pTable->numOfColumns].bytes; + accumBytes += htons(pSchema[i + numOfColumns].bytes); } tsdbTableSetTagValue(&tCfg, dataRow, false); } @@ -183,51 +174,46 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, table:%s, start to alter", pVnode, pVnode->vgId, pTable->tableId); - pTable->numOfColumns = htons(pTable->numOfColumns); - pTable->numOfTags = htons(pTable->numOfTags); - pTable->sid = htonl(pTable->sid); - pTable->sversion = htonl(pTable->sversion); - pTable->tagDataLen = htonl(pTable->tagDataLen); - pTable->sqlDataLen = htonl(pTable->sqlDataLen); - pTable->uid = htobe64(pTable->uid); - pTable->superTableUid = htobe64(pTable->superTableUid); - pTable->createdTime = htobe64(pTable->createdTime); + int16_t numOfColumns = htons(pTable->numOfColumns); + int16_t numOfTags = htons(pTable->numOfTags); + int32_t sid = htonl(pTable->sid); + uint64_t uid = htobe64(pTable->uid); SSchema *pSchema = (SSchema *) pTable->data; - int totalCols = pTable->numOfColumns + pTable->numOfTags; - for (int i = 0; i < totalCols; i++) { - pSchema[i].colId = htons(pSchema[i].colId); - pSchema[i].bytes = htons(pSchema[i].bytes); - } - + int32_t totalCols = numOfColumns + numOfTags; + STableCfg tCfg; - tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid); + tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid); - STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns); - for (int i = 0; i < pTable->numOfColumns; i++) { - tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + STSchema *pDestSchema = tdNewSchema(numOfColumns); + for (int i = 0; i < numOfColumns; i++) { + tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetSchema(&tCfg, pDestSchema, false); - if (pTable->numOfTags != 0) { - STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags); - for (int i = pTable->numOfColumns; i < totalCols; i++) { - tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + if (numOfTags != 0) { + STSchema *pDestTagSchema = tdNewSchema(numOfTags); + for (int i = numOfColumns; i < totalCols; i++) { + tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } - tsdbTableSetSchema(&tCfg, pDestTagSchema, false); + tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false); char *pTagData = pTable->data + totalCols * sizeof(SSchema); int accumBytes = 0; SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema); - for (int i = 0; i < pTable->numOfTags; i++) { + for (int i = 0; i < numOfTags; i++) { tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i); - accumBytes += pSchema[i + pTable->numOfColumns].bytes; + accumBytes += htons(pSchema[i + numOfColumns].bytes); } tsdbTableSetTagValue(&tCfg, dataRow, false); } - code = tsdbAlterTable(pVnode->tsdb, &tCfg); + void *pTsdb = vnodeGetTsdb(pVnode); + code = tsdbAlterTable(pTsdb, &tCfg); + + tfree(pDestSchema); + dTrace("pVnode:%p vgId:%d, table:%s, alter table result:%d", pVnode, pVnode->vgId, pTable->tableId, code); return code; @@ -238,7 +224,7 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId); - pTable->uid = htobe64(pTable->uid); + // int64_t uid = htobe64(pTable->uid); // TODO: drop stable in vvnode //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); From e4d2b5bb5e0d82c0843ea4519e3da782396592d8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 15:09:37 +0800 Subject: [PATCH 46/48] turn on the arbitrator code --- src/inc/tsync.h | 1 + src/vnode/main/src/vnodeMain.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index ea20ceec79..cdc92b2366 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -99,6 +99,7 @@ extern int tsMaxSyncNum; extern int tsSyncTcpThreads; extern int tsMaxWatchFiles; extern short tsSyncPort; +extern int tsSyncTimer; extern int tsMaxFwdInfo; extern int sDebugFlag; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 8d36f59da0..7ee468fa54 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -375,7 +375,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; - pVnode->syncCfg.arbitratorIp = 0; + pVnode->syncCfg.arbitratorIp = arbitratorIp; int32_t quorum = -1; num = fscanf(fp, "%s %d", option[0], &quorum); From 0c2e506772d234f39c1593a5dc2092c65d6d4040 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 17:35:29 +0800 Subject: [PATCH 47/48] add test --- tests/CMakeLists.txt | 1 + tests/test/c/CMakeLists.txt | 11 ++ tests/test/c/benchmarkPerTable.c | 319 +++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 tests/test/c/CMakeLists.txt create mode 100644 tests/test/c/benchmarkPerTable.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 872201f178..a6070a2fc1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,3 +11,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON) ADD_SUBDIRECTORY(examples/c) ADD_SUBDIRECTORY(tsim) +ADD_SUBDIRECTORY(test/c) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt new file mode 100644 index 0000000000..532f304b99 --- /dev/null +++ b/tests/test/c/CMakeLists.txt @@ -0,0 +1,11 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) + add_executable(benchmarkPerTable benchmarkPerTable.c) + target_link_libraries(benchmarkPerTable taos_static pthread) +ENDIF() diff --git a/tests/test/c/benchmarkPerTable.c b/tests/test/c/benchmarkPerTable.c new file mode 100644 index 0000000000..ee4e0fecbd --- /dev/null +++ b/tests/test/c/benchmarkPerTable.c @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tlog.h" +#include "ttimer.h" +#include "tutil.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +typedef struct { + int64_t rowsPerTable; + int64_t pointsPerTable; + int64_t tableBeginIndex; + int64_t tableEndIndex; + int threadIndex; + char dbName[32]; + char stableName[64]; + pthread_t thread; +} SInfo; + +void *syncTest(void *param); +void generateRandomPoints(); +void shellParseArgument(int argc, char *argv[]); +void createDbAndTable(); +void insertData(); + +int32_t randomData[MAX_RANDOM_POINTS]; +int64_t rowsPerTable = 10000; +int64_t pointsPerTable = 1; +int64_t numOfThreads = 1; +int64_t numOfTablesPerThread = 1; +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t cache = 16384; +int32_t tables = 1000; + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + generateRandomPoints(); + taos_init(); + createDbAndTable(); + insertData(); +} + +void createDbAndTable() { + dPrint("start to create table"); + + TAOS * con; + struct timeval systemTime; + int64_t st, et; + char qstr[64000]; + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); + if (taos_query(con, qstr)) { + dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); + exit(0); + } + + sprintf(qstr, "use %s", dbName); + if (taos_query(con, qstr)) { + dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = numOfTablesPerThread * numOfThreads; + + if (strcmp(stableName, "no") != 0) { + int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ") tags(t int)"); + + if (taos_query(con, qstr)) { + dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + for (int64_t t = 0; t < totalTables; ++t) { + sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); + if (taos_query(con, qstr)) { + dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } else { + for (int64_t t = 0; t < totalTables; ++t) { + int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ")"); + + if (taos_query(con, qstr)) { + dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables); +} + +void insertData() { + struct timeval systemTime; + int64_t st, et; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + dPrint("%d threads are spawned to insert data", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + + // Start threads to write + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].rowsPerTable = rowsPerTable; + pInfo[i].pointsPerTable = pointsPerTable; + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + double seconds = (et - st) / 1000.0 / 1000.0; + + int64_t totalTables = numOfTablesPerThread * numOfThreads; + int64_t totalRows = totalTables * rowsPerTable; + int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable; + double speedOfRows = totalRows / seconds; + double speedOfPoints = totalPoints / seconds; + + dPrint( + "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf " + "PointsPerSecond:%.1lf%s", + GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC); + + dPrint("threads exit"); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void *syncTest(void *param) { + TAOS * con; + SInfo * pInfo = (SInfo *)param; + struct timeval systemTime; + int64_t st, et; + char qstr[65000]; + int maxBytes = 60000; + + dPrint("thread:%d, start to run", pInfo->threadIndex); + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + taos_query(con, qstr); + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + int64_t start = 1430000000000; + int64_t interval = 1000; // 1000 ms + + char *sql = qstr; + char inserStr[] = "insert into"; + int len = sprintf(sql, "%s", inserStr); + + for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) { + len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table); + for (int64_t row = 0; row < pInfo->rowsPerTable; row++) { + len += sprintf(sql + len, "(%ld", start + row * interval); + for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) { + len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]); + // len += sprintf(sql + len, ",%ld", row); + } + len += sprintf(sql + len, ")"); + if (len > maxBytes) { + if (taos_query(con, qstr)) { + dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, + table, row, taos_errstr(con)); + } + + // "insert into" + len = sprintf(sql, "%s", inserStr); + + // "insert into st1 values" + if (row != pInfo->rowsPerTable - 1) { + len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table); + } + } + } + } + + if (len != strlen(inserStr)) { + taos_query(con, qstr); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + int64_t totalRows = totalTables * pInfo->rowsPerTable; + int64_t totalPoints = totalRows * pInfo->pointsPerTable; + dPrint("thread:%d, insert finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex, + (et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints); + + return NULL; +} + +void generateRandomPoints() { + for (int r = 0; r < MAX_RANDOM_POINTS; ++r) { + randomData[r] = rand() % 1000; + } +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of TDengine, the insert method is table-by-table\n"); + + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table"); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread); + printf("%s%s\n", indent, "-tables"); + printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables); + printf("%s%s\n", indent, "-cache"); + printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + rowsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + pointsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTablesPerThread = atoi(argv[++i]); + } else if (strcmp(argv[i], "-tables") == 0) { + tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-cache") == 0) { + cache = atoi(argv[++i]); + } else { + } + } + + dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC); + dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC); + dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); + dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC); + dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC); + dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC); + dPrint("%sdbName:%s%s", GREEN, dbName, NC); + dPrint("%stableName:%s%s", GREEN, stableName, NC); + dPrint("%sstart to run%s", GREEN, NC); +} From 7b900a5dd120b20380a68bfa13b84b18e3b7e2a7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 22:50:31 +0800 Subject: [PATCH 48/48] check null pointer in tscSql.c. --- src/client/src/tscSql.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 199b4150e8..f2a3cdcab0 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -329,7 +329,10 @@ int taos_num_fields(TAOS_RES *res) { } SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo; - return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); + if (pFieldsInfo) + return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); + else + return 0; } int taos_field_count(TAOS *taos) { @@ -351,7 +354,11 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - return pQueryInfo->fieldsInfo.pFields; + + if (pQueryInfo) + return pQueryInfo->fieldsInfo.pFields; + else + return NULL; } int taos_retrieve(TAOS_RES *res) { @@ -401,6 +408,9 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + if (pQueryInfo == NULL) + return 0; + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i); }