From dc15a3dadf06d55531e037ff20e51d2df9e5a529 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 11:23:36 +0800 Subject: [PATCH 01/20] opt rpc send/recv --- source/libs/transport/inc/transComm.h | 16 ++----- source/libs/transport/src/transComm.c | 63 +++++---------------------- 2 files changed, 13 insertions(+), 66 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a81d6db80f..853f24b0ce 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -127,7 +127,7 @@ typedef struct { int8_t retryCnt; int8_t retryLimit; - // bool setMaxRetry; + STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -194,17 +194,7 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transLabel(trans) ((STrans*)trans)->label -// int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -// void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -//// int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); -// -// int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -// void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -// bool transCompressMsg(char* msg, int32_t len, int32_t* flen); -// bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); - void transFreeMsg(void* msg); - // typedef struct SConnBuffer { char* buf; @@ -321,8 +311,8 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); // request list typedef struct STransReq { - queue q; - void* data; + queue q; + uv_write_t wreq; } STransReq; void transReqQueueInit(queue* q); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8cf525a506..ca405fa536 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -23,33 +23,6 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static int32_t refMgt; static int32_t instMgt; -int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - int ret = -1; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; - - return ret; -} - -void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - memcpy(pAuth, context.digest, sizeof(context.digest)); -} - bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return false; // SRpcHead* pHead = rpcHeadFromCont(pCont); @@ -176,7 +149,6 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { * info--->| */ SConnBuffer* p = connBuf; - uvBuf->base = p->buf + p->len; if (p->left == -1) { uvBuf->len = p->cap - p->len; @@ -267,14 +239,9 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { uv_async_t* async = &(pool->asyncs[idx]); SAsyncItem* item = async->data; - int64_t st = taosGetTimestampUs(); taosThreadMutexLock(&item->mtx); QUEUE_PUSH(&item->qmsg, q); taosThreadMutexUnlock(&item->mtx); - int64_t el = taosGetTimestampUs() - st; - if (el > 50) { - // tInfo("lock and unlock cost:%d", (int)el); - } return uv_async_send(async); } @@ -350,30 +317,21 @@ void transReqQueueInit(queue* q) { QUEUE_INIT(q); } void* transReqQueuePush(queue* q) { - uv_write_t* req = taosMemoryCalloc(1, sizeof(uv_write_t)); - STransReq* wreq = taosMemoryCalloc(1, sizeof(STransReq)); - wreq->data = req; - req->data = wreq; - QUEUE_PUSH(q, &wreq->q); - return req; + STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); + req->wreq.data = req; + QUEUE_PUSH(q, &req->q); + return &req->wreq; } void* transReqQueueRemove(void* arg) { void* ret = NULL; - uv_write_t* req = arg; - STransReq* wreq = req && req->data ? req->data : NULL; + uv_write_t* wreq = arg; - assert(wreq->data == req); - if (wreq == NULL || wreq->data == NULL) { - taosMemoryFree(wreq->data); - taosMemoryFree(wreq); - return req; - } + STransReq* req = wreq ? wreq->data : NULL; + if (req == NULL) return NULL; + QUEUE_REMOVE(&req->q); - QUEUE_REMOVE(&wreq->q); - - ret = req && req->handle ? req->handle->data : NULL; - taosMemoryFree(wreq->data); - taosMemoryFree(wreq); + ret = wreq && wreq->handle ? wreq->handle->data : NULL; + taosMemoryFree(req); return ret; } @@ -382,7 +340,6 @@ void transReqQueueClear(queue* q) { queue* h = QUEUE_HEAD(q); QUEUE_REMOVE(h); STransReq* req = QUEUE_DATA(h, STransReq, q); - taosMemoryFree(req->data); taosMemoryFree(req); } } From 00d4efb4ea3dfb9cf94e404fa0c4d57186a3e677 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 11:28:49 +0800 Subject: [PATCH 02/20] opt rpc send --- source/libs/transport/src/transCli.c | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..140d2ef792 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1380,11 +1380,12 @@ int transReleaseCliHandle(void* handle) { } int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) { - transFreeMsg(pReq->pCont); - return -1; - } + // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + // if (pTransInst == NULL) { + // transFreeMsg(pReq->pCont); + // return -1; + // } + STrans* pTransInst = shandle; bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); @@ -1404,7 +1405,6 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran if (ctx != NULL) { pCtx->appCtx = *ctx; } - assert(pTransInst->connType == TAOS_CONN_CLIENT); SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; @@ -1421,22 +1421,23 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; } int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) { - transFreeMsg(pReq->pCont); - return -1; - } + STrans* pTransInst = shandle; + // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + // if (pTransInst == NULL) { + // transFreeMsg(pReq->pCont); + // return -1; + // } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } @@ -1474,7 +1475,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs _RETURN: tsem_destroy(sem); taosMemoryFree(sem); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return ret; } /* From 46c2dfa452487ead247889fc29dd629b262770bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 13:58:58 +0800 Subject: [PATCH 03/20] opt send --- source/libs/transport/src/trans.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 7633820292..5fa57fdbc2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -80,7 +80,8 @@ void* rpcOpen(const SRpcInit* pInit) { int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); transAcquireExHandle(transGetInstMgt(), refId); pRpc->refId = refId; - return (void*)refId; + return pRpc; + // return (void*)refId; } void rpcClose(void* arg) { tInfo("start to close rpc"); From a5420dfaad260a06907080b3540b0d1b56546c99 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 18:36:19 +0800 Subject: [PATCH 04/20] fix rpc perf --- source/libs/transport/src/transCli.c | 24 ++++++++++++++++-------- source/libs/transport/src/transSvr.c | 3 +-- source/libs/transport/test/svrBench.c | 17 ++++++++--------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 140d2ef792..11ba2a50d3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -16,7 +16,8 @@ #include "transComm.h" typedef struct SConnList { - queue conn; + queue conn; + int32_t size; } SConnList; typedef struct SCliConn { @@ -518,15 +519,18 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { if (QUEUE_IS_EMPTY(&plist->conn)) { return NULL; } + + plist->size -= 1; queue* h = QUEUE_HEAD(&plist->conn); SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); - transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); - conn->task = NULL; - + if (conn->task != NULL) { + transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + conn->task = NULL; + } return conn; } static void addConnToPool(void* pool, SCliConn* conn) { @@ -555,13 +559,17 @@ static void addConnToPool(void* pool, SCliConn* conn) { assert(conn->list != NULL); QUEUE_INIT(&conn->q); QUEUE_PUSH(&conn->list->conn, &conn->q); + conn->list->size += 1; + conn->task = NULL; assert(!QUEUE_IS_EMPTY(&conn->list->conn)); - STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); - arg->param1 = conn; - arg->param2 = thrd; - conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); + if (conn->list->size >= 10) { + STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); + arg->param1 = conn; + arg->param2 = thrd; + conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); + } } static int32_t allocConnRef(SCliConn* conn, bool update) { if (update) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8b27d95e52..14b8b35478 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -75,7 +75,6 @@ typedef struct SWorkThrd { SAsyncPool* asyncPool; uv_prepare_t* prepare; queue msg; - TdThreadMutex msgMtx; queue conn; void* pTransInst; @@ -499,6 +498,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } + // release handle to rpc init if (msg->type == Quit) { (*transAsyncHandle[msg->type])(msg, pThrd); @@ -743,7 +743,6 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->pipe->data = pThrd; QUEUE_INIT(&pThrd->msg); - taosThreadMutexInit(&pThrd->msgMtx, NULL); pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); uv_prepare_init(pThrd->loop, pThrd->prepare); diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 224f527385..6eb80c8504 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -75,15 +75,14 @@ void processShellMsg() { void *handle = pRpcMsg->info.handle; taosFreeQitem(pRpcMsg); - - { - SRpcMsg nRpcMsg = {0}; - nRpcMsg.pCont = rpcMallocCont(msgSize); - nRpcMsg.contLen = msgSize; - nRpcMsg.info.handle = handle; - nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; - rpcSendResponse(&nRpcMsg); - } + //{ + // SRpcMsg nRpcMsg = {0}; + // nRpcMsg.pCont = rpcMallocCont(msgSize); + // nRpcMsg.contLen = msgSize; + // nRpcMsg.info.handle = handle; + // nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; + // rpcSendResponse(&nRpcMsg); + //} } taosUpdateItemSize(qinfo.queue, numOfMsgs); From f97b972191809130052c0ea5a20948889a3b7033 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 20:55:01 +0800 Subject: [PATCH 05/20] opt rpc perf --- source/libs/transport/src/trans.c | 3 +-- source/libs/transport/src/transCli.c | 32 +++++++++++++-------------- source/libs/transport/src/transComm.c | 3 ++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5fa57fdbc2..7633820292 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -80,8 +80,7 @@ void* rpcOpen(const SRpcInit* pInit) { int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); transAcquireExHandle(transGetInstMgt(), refId); pRpc->refId = refId; - return pRpc; - // return (void*)refId; + return (void*)refId; } void rpcClose(void* arg) { tInfo("start to close rpc"); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 11ba2a50d3..61888db74e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -564,7 +564,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->task = NULL; assert(!QUEUE_IS_EMPTY(&conn->list->conn)); - if (conn->list->size >= 10) { + if (conn->list->size >= 50) { STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); arg->param1 = conn; arg->param2 = thrd; @@ -1372,7 +1372,7 @@ int transReleaseCliHandle(void* handle) { } STransMsg tmsg = {.info.handle = handle}; - TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + // TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; @@ -1388,12 +1388,11 @@ int transReleaseCliHandle(void* handle) { } int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - // if (pTransInst == NULL) { - // transFreeMsg(pReq->pCont); - // return -1; - // } - STrans* pTransInst = shandle; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return -1; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); @@ -1429,23 +1428,22 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; } int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = shandle; - // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - // if (pTransInst == NULL) { - // transFreeMsg(pReq->pCont); - // return -1; - // } + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return -1; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } @@ -1483,7 +1481,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs _RETURN: tsem_destroy(sem); taosMemoryFree(sem); - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return ret; } /* diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ca405fa536..f0b1885d66 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -156,7 +156,8 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { if (p->left < p->cap - p->len) { uvBuf->len = p->left; } else { - p->buf = taosMemoryRealloc(p->buf, p->left + p->len); + p->cap = p->left + p->len; + p->buf = taosMemoryRealloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->left; } From bab1b9c62196792a497a0b0bc6bcef394c77387e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 21:18:12 +0800 Subject: [PATCH 06/20] fix rpc perf --- source/util/src/tref.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 9cd849b9be..c984ef3f34 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -44,11 +44,11 @@ typedef struct { void (*fp)(void *); } SRefSet; -static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; +static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; static TdThreadOnce tsRefModuleInit = PTHREAD_ONCE_INIT; static TdThreadMutex tsRefMutex; -static int32_t tsRefSetNum = 0; -static int32_t tsNextId = 0; +static int32_t tsRefSetNum = 0; +static int32_t tsNextId = 0; static void taosInitRefModule(void); static void taosLockList(int64_t *lockedBy); From eeb71acdf661ffcd36ae53932082d90d82084b2c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 21:28:34 +0800 Subject: [PATCH 07/20] fix rpc perf --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 6d893765fc..a7747b9b45 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -396,7 +396,7 @@ typedef enum ELogicConditionType { #ifdef WINDOWS #define TSDB_MAX_RPC_THREADS 4 // windows pipe only support 4 connections. #else -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_RPC_THREADS 10 #endif #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type From 7f8195b3a588e467e1e3efeb1ba9e7c8adbfb252 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 11:27:13 +0800 Subject: [PATCH 08/20] fix uninited value --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 61888db74e..3206a2ff39 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -510,7 +510,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { SHashObj* pPool = pool; SConnList* plist = taosHashGet(pPool, key, strlen(key)); if (plist == NULL) { - SConnList list; + SConnList list = {0}; taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list)); plist = taosHashGet(pPool, key, strlen(key)); QUEUE_INIT(&plist->conn); From 7fc09d84752979921bad2445c447471272f20ab8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 20:51:21 +0800 Subject: [PATCH 09/20] start timer for particular msg --- source/libs/transport/src/transCli.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 48d1829aa2..bd6fcac806 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -340,8 +340,8 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); uv_timer_stop(conn->timer); } - conn->timer->data = NULL; taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; conn->timer = NULL; } @@ -483,6 +483,7 @@ void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb SCliConn* conn = handle->data; tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_read_stop(conn->stream); cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); } @@ -542,6 +543,13 @@ static void addConnToPool(void* pool, SCliConn* conn) { allocConnRef(conn, true); + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosArrayPush(thrd->timerList, &conn->timer); + conn->timer->data = NULL; + conn->timer = NULL; + } + STrans* pTransInst = thrd->pTransInst; cliReleaseUnfinishedMsg(conn); transQueueClear(&conn->cliMsgs); From 6930b393d60eca1edf887ae454b179cf2d20146c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 20:53:42 +0800 Subject: [PATCH 10/20] start timer for particular msg --- include/libs/transport/trpc.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 467f0a9ff0..655c903c0b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -47,8 +47,6 @@ typedef struct SRpcHandleInfo { int8_t persistHandle; // persist handle or not int8_t hasEpSet; - STraceId traceId; - // app info void *ahandle; // app handle set by client void *wrapper; // wrapper handle @@ -58,7 +56,8 @@ typedef struct SRpcHandleInfo { void *rsp; int32_t rspLen; - // conn info + STraceId traceId; + SRpcConnInfo conn; } SRpcHandleInfo; From 3f0c3076a93ab8529a0ccd358a9877e9535cbf84 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 9 Aug 2022 22:29:54 +0800 Subject: [PATCH 11/20] test: open sim testcase of sync --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 23676153b5..0bb24826a3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -332,8 +332,8 @@ ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim # --- sync -#./test.sh -f tsim/sync/3Replica1VgElect.sim -#./test.sh -f tsim/sync/3Replica5VgElect.sim +./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim From 8665f8056df482ae9e07d4d5b304debf9389def8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 00:21:42 +0800 Subject: [PATCH 12/20] build: release ver-3.0.0.10003 --- packaging/release.bat | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index d58e19cece..ffd3a68048 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -11,13 +11,13 @@ if !%2==! GOTO USAGE if "%1" == "cluster" ( set work_dir=%internal_dir% set packagServerName_x64=TDengine-enterprise-server-%2-beta-Windows-x64 - set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 + @REM set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 set packagClientName_x64=TDengine-enterprise-client-%2-beta-Windows-x64 set packagClientName_x86=TDengine-enterprise-client-%2-beta-Windows-x86 ) else ( set work_dir=%community_dir% set packagServerName_x64=TDengine-server-%2-Windows-x64 - set packagServerName_x86=TDengine-server-%2-Windows-x86 + @REM set packagServerName_x86=TDengine-server-%2-Windows-x86 set packagClientName_x64=TDengine-client-%2-Windows-x64 set packagClientName_x86=TDengine-client-%2-Windows-x86 ) @@ -59,8 +59,8 @@ rd /s /Q C:\TDengine cmake --install . if not %errorlevel% == 0 ( call :RUNFAILED build x86 failed & exit /b 1) cd %package_dir% -iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release -if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) +@REM iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +@REM if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) iscc /DMyAppInstallName="%packagClientName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x86% failed & exit /b 1) From 3fe50ef41e56c82b5af79e52d70c6c47bebaebfb Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 00:29:08 +0800 Subject: [PATCH 13/20] build: release ver-3.0.0.10003 --- cmake/cmake.platform | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index cd1be78fbc..887fbd86d5 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -97,13 +97,13 @@ IF ("${CPUTYPE}" STREQUAL "") ELSE () # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 - IF (${CPUTYPE} MATCHES "aarch32" or ${CPUTYPE} MATCHES "arm32") + IF (${CPUTYPE} MATCHES "aarch32" OR ${CPUTYPE} MATCHES "arm32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") SET(TD_ARM_32 TRUE) - ELSEIF (${CPUTYPE} MATCHES "aarch64" or ${CPUTYPE} MATCHES "arm64") + ELSEIF (${CPUTYPE} MATCHES "aarch64" OR ${CPUTYPE} MATCHES "arm64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") ADD_DEFINITIONS("-D_TD_ARM_") From 38807616e6e493a45c7db0736f75d00c2eca959f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 10:29:47 +0800 Subject: [PATCH 14/20] build: release ver-3.0.0.10003 --- cmake/cmake.install | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cmake.install b/cmake/cmake.install index d9f759217a..4b1ccbf6d5 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -22,6 +22,7 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) From 67fbb7391031e561b6bfe633e18b67de273f6a21 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 11:23:37 +0800 Subject: [PATCH 15/20] build: release ver-3.0.0.10003 --- include/os/osSystem.h | 1 + source/os/src/osSysinfo.c | 12 +++++++++++- source/os/src/osSystem.c | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 6770be6e46..c5b8fc7be7 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -32,6 +32,7 @@ extern "C" { typedef struct TdCmd *TdCmdPtr; TdCmdPtr taosOpenCmd(const char* cmd); +int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf); int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf); int32_t taosEOFCmd(TdCmdPtr pCmd); int64_t taosCloseCmd(TdCmdPtr* ppCmd); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 3e68b6e086..3a75e18a7f 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -398,7 +398,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); - if (code != 0) { + if (code != 0 && (done & 1) == 0) { TdFilePtr pFile1 = taosOpenFile("/proc/device-tree/model", TD_FILE_READ | TD_FILE_STREAM); if (pFile1 == NULL) return code; taosGetsFile(pFile1, maxLen, cpuModel); @@ -407,6 +407,16 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { done |= 1; } + if (code != 0 && (done & 1) == 0) { + TdCmdPtr pCmd = taosOpenCmd("uname -a"); + if (pCmd == NULL) return code; + if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) { + code = 0; + done |= 1; + } + taosCloseCmd(&pCmd); + } + if ((done & 2) == 0) { *numOfCores = coreCount; done |= 2; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index c86cd19e32..b6f6637601 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -248,6 +248,16 @@ TdCmdPtr taosOpenCmd(const char* cmd) { #endif } +int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf) { + if (pCmd == NULL || buf == NULL) { + return -1; + } + if (fgets(buf, maxSize, (FILE*)pCmd) == NULL) { + return -1; + } + return strlen(buf); +} + int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf) { if (pCmd == NULL || ptrBuf == NULL) { return -1; From 6cabe5847e1a6f922cf657195c3666db9e2158c4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 10 Aug 2022 11:26:23 +0800 Subject: [PATCH 16/20] fix(sync): delete memory after optimized --- source/libs/sync/src/syncReplication.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 29370fd4a5..24f75de5d3 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -199,7 +199,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { // send msg syncNodeAppendEntriesBatch(pSyncNode, pDestId, pMsg); - syncAppendEntriesBatchDestroy(pMsg); // speed up if (pMsg->dataCount > 0 && pSyncNode->commitIndex - pMsg->prevLogIndex > SYNC_SLOW_DOWN_RANGE) { @@ -216,6 +215,8 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { } while (0); #endif } + + syncAppendEntriesBatchDestroy(pMsg); } return ret; From 712f7e8cce85efdf79727f8ffc847a6c62b0e7e5 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 11:47:45 +0800 Subject: [PATCH 17/20] os: fix copy appname overflow --- source/os/src/osSemaphore.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 63c414386b..5da78f002f 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -57,7 +57,7 @@ int32_t taosGetAppName(char* name, int32_t* len) { end = filepath; } - strcpy(name, end); + tstrncpy(name, end, TSDB_APP_NAME_LEN); if (len != NULL) { *len = (int32_t)strlen(end); @@ -625,7 +625,7 @@ int32_t taosGetAppName(char *name, int32_t *len) { buf[PATH_MAX] = '\0'; size_t n = strlen(buf); if (len) *len = n; - if (name) strcpy(name, buf); + if (name) tstrncpy(name, buf, TSDB_APP_NAME_LEN); return 0; } @@ -668,7 +668,7 @@ int32_t taosGetAppName(char* name, int32_t* len) { ++end; - strcpy(name, end); + tstrncpy(name, end, TSDB_APP_NAME_LEN); if (len != NULL) { *len = strlen(name); From 59d982333cd52e6042d5fd0c4a4f3fc4b4fff3c4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 13:05:46 +0800 Subject: [PATCH 18/20] os: fix copy appname overflow --- source/os/src/osSemaphore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 5da78f002f..c57ea33028 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" +#include "tdef.h" #include "pthread.h" #ifdef WINDOWS From 10266ec7fa1255b2b8b13c7a18c8b4b6d1c84a82 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 13:47:38 +0800 Subject: [PATCH 19/20] docs: update k8s doc for 3.0 (#15852) * docs: update k8s doc for 3.0 * docs: update 03-k8s.md --- docs/zh/10-deployment/03-k8s.md | 327 ++++++++++++++------------------ 1 file changed, 140 insertions(+), 187 deletions(-) diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index d45a3b8030..ab1bff4449 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -3,27 +3,11 @@ sidebar_label: Kubernetes title: 在 Kubernetes 上部署 TDengine 集群 --- -## 配置 ConfigMap +以下配置文件可以从 [GitHub 仓库](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine) 下载。 -为 TDengine 创建 `taoscfg.yaml`，此文件中的配置将作为环境变量传入 TDengine 镜像，更新此配置将导致所有 TDengine POD 重启。 +## 配置 Service 服务 -```yaml ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: taoscfg - labels: - app: tdengine -data: - CLUSTER: "1" - TAOS_KEEP: "3650" - TAOS_DEBUG_FLAG: "135" -``` - -## 配置服务 - -创建一个 service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的所有端口： +创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的所有端口： ```yaml --- @@ -38,45 +22,9 @@ spec: - name: tcp6030 protocol: "TCP" port: 6030 - - name: tcp6035 - protocol: "TCP" - port: 6035 - name: tcp6041 protocol: "TCP" port: 6041 - - name: udp6030 - protocol: "UDP" - port: 6030 - - name: udp6031 - protocol: "UDP" - port: 6031 - - name: udp6032 - protocol: "UDP" - port: 6032 - - name: udp6033 - protocol: "UDP" - port: 6033 - - name: udp6034 - protocol: "UDP" - port: 6034 - - name: udp6035 - protocol: "UDP" - port: 6035 - - name: udp6036 - protocol: "UDP" - port: 6036 - - name: udp6037 - protocol: "UDP" - port: 6037 - - name: udp6038 - protocol: "UDP" - port: 6038 - - name: udp6039 - protocol: "UDP" - port: 6039 - - name: udp6040 - protocol: "UDP" - port: 6040 selector: app: "tdengine" ``` @@ -109,7 +57,7 @@ spec: spec: containers: - name: "tdengine" - image: "zitsen/taosd:develop" + image: "tdengine/tdengine:3.0.0.0" imagePullPolicy: "Always" envFrom: - configMapRef: @@ -118,45 +66,9 @@ spec: - name: tcp6030 protocol: "TCP" containerPort: 6030 - - name: tcp6035 - protocol: "TCP" - containerPort: 6035 - name: tcp6041 protocol: "TCP" containerPort: 6041 - - name: udp6030 - protocol: "UDP" - containerPort: 6030 - - name: udp6031 - protocol: "UDP" - containerPort: 6031 - - name: udp6032 - protocol: "UDP" - containerPort: 6032 - - name: udp6033 - protocol: "UDP" - containerPort: 6033 - - name: udp6034 - protocol: "UDP" - containerPort: 6034 - - name: udp6035 - protocol: "UDP" - containerPort: 6035 - - name: udp6036 - protocol: "UDP" - containerPort: 6036 - - name: udp6037 - protocol: "UDP" - containerPort: 6037 - - name: udp6038 - protocol: "UDP" - containerPort: 6038 - - name: udp6039 - protocol: "UDP" - containerPort: 6039 - - name: udp6040 - protocol: "UDP" - containerPort: 6040 env: # POD_NAME for FQDN config - name: POD_NAME @@ -190,14 +102,13 @@ spec: readinessProbe: exec: command: - - taos - - -s - - "show mnodes" + - taos-check initialDelaySeconds: 5 timeoutSeconds: 5000 livenessProbe: - tcpSocket: - port: 6030 + exec: + command: + - taos-check initialDelaySeconds: 15 periodSeconds: 20 volumeClaimTemplates: @@ -206,44 +117,78 @@ spec: spec: accessModes: - "ReadWriteOnce" - storageClassName: "csi-rbd-sc" + storageClassName: "standard" resources: requests: storage: "10Gi" ``` -## 启动集群 +## 使用 kubectl 命令部署 TDengine 集群 -将前述三个文件添加到 Kubernetes 集群中： +顺序执行以下命令。 ```bash -kubectl apply -f taoscfg.yaml kubectl apply -f taosd-service.yaml kubectl apply -f tdengine.yaml ``` -上面的配置将生成一个两节点的 TDengine 集群，dnode 是自动配置的，可以使用 `show dnodes` 命令查看当前集群的节点： +上面的配置将生成一个三节点的 TDengine 集群，dnode 是自动配置的，可以使用 show dnodes 命令查看当前集群的节点： ```bash kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" - +kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" ``` 输出如下： ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 17:13:24.181 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 17:14:09.257 | | -Query OK, 2 row(s) in set (0.000997s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | +Query OK, 3 rows in database (0.003655s) +``` +## 使能端口转发 + +利用 kubectl 端口转发功能可以使应用可以访问 Kubernetes 环境运行的 TDengine 集群。 + +``` +kubectl port-forward tdengine-0 6041:6041 & +``` + +使用 curl 命令验证 TDengine REST API 使用的 6041 接口。 + +``` +$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +Handling connection for 6041 +{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +``` + +## 使用 dashboard 进行图形化管理 + + minikube 提供 dashboard 命令支持图形化管理界面。 + +``` +$ minikube dashboard +* Verifying dashboard health ... +* Launching proxy ... +* Verifying proxy health ... +* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... +http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ +``` + +对于某些公有云环境，minikube 绑定在 127.0.0.1 IP 地址上无法通过远程访问，需要使用 kubectl proxy 命令将端口映射到 0.0.0.0 IP 地址上，再通过浏览器访问虚拟机公网 IP 和端口以及相同的 dashboard URL 路径即可远程访问 dashboard。 + +``` +$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' ``` ## 集群扩容 @@ -252,14 +197,12 @@ TDengine 集群支持自动扩容： ```bash kubectl scale statefulsets tdengine --replicas=4 - ``` 上面命令行中参数 `--replica=4` 表示要将 TDengine 集群扩容到 4 个节点，执行后首先检查 POD 的状态： ```bash kubectl get pods -l app=tdengine - ``` 输出如下： @@ -270,102 +213,112 @@ tdengine-0 1/1 Running 0 161m tdengine-1 1/1 Running 0 161m tdengine-2 1/1 Running 0 32m tdengine-3 1/1 Running 0 32m - ``` 此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： ```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" - +kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" ``` 扩容后的四节点 TDengine 集群的 dnode 列表: ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 14:07:27.078 | | - 4 | tdengine-3.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 14:07:48.362 | | -Query OK, 4 row(s) in set (0.001293s) - + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | + 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | +Query OK, 4 rows in database (0.008377s) ``` ## 集群缩容 -TDengine 的缩容并没有自动化，我们尝试将一个三节点集群缩容到两节点。 +由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令，节点删除完成后再进行 Kubernetes 集群缩容。 -首先，确认一个三节点 TDengine 集群正常工作，在 TDengine CLI 中查看 dnode 的状态： +注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。 + +``` +$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" +``` ```bash +$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:28:49.787 | | -Query OK, 3 row(s) in set (0.001101s) - + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | +Query OK, 3 rows in database (0.004861s) ``` -想要安全的缩容，首先需要将节点从 dnode 列表中移除，也即从集群中移除： +确认移除成功后（使用 kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" 查看和确认 dnode 列表），使用 kubectl 命令移除 POD： + +``` +kubectl scale statefulsets tdengine --replicas=3 +``` + +最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态： + +``` +$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 4m7s +tdengine-1 1/1 Running 0 3m55s +tdengine-2 1/1 Running 0 2m28s +``` + +POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 ```bash -kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 'tdengine-2.taosd.default.svc.cluster.local:6030'" - -``` - -通过 `show dondes` 命令确认移除成功后，移除相应的 POD： - -```bash -kubectl scale statefulsets tdengine --replicas=2 - -``` - -最后一个 POD 会被删除，使用 `kubectl get pods -l app=tdengine` 查看集群状态： - -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 3h40m -tdengine-1 1/1 Running 0 3h40m - -``` - -POD 删除后，需要手动删除 PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 - -```bash -kubectl delete pvc taosdata-tdengine-2 - +$ kubectl delete pvc taosdata-tdengine-3 ``` 此时的集群状态是安全的，需要时还可以再次进行扩容： ```bash -kubectl scale statefulsets tdengine --replicas=3 +$ kubectl scale statefulsets tdengine --replicas=4 +statefulset.apps/tdengine scaled +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 35m +tdengine-1 1/1 Running 0 34m +tdengine-2 1/1 Running 0 12m +tdengine-3 0/1 ContainerCreating 0 4s +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 35m +tdengine-1 1/1 Running 0 34m +tdengine-2 1/1 Running 0 12m +tdengine-3 0/1 Running 0 7s +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + +Server is Community Edition. - -``` - -`show dnodes` 输出如下： - -``` taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | +id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | ====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | - 4 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:40:49.177 | | - - +1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | +2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | +5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | +6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | +Query OK, 4 row(s) in set (0.001348s) ``` -## 删除集群 +## 清理 TDengine 集群 完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 @@ -381,26 +334,26 @@ kubectl delete configmap taoscfg ### 错误一 -扩容到四节点之后缩容到两节点，删除的 POD 会进入 offline 状态： +未进行 "drop dnode" 直接进行缩容，由于 TDengine 尚未删除节点，缩容 pod 导致 TDengine 集群中部分节点处于 offline 状态。 ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + +Server is Community Edition. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | +id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | ====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | offline | any | 2021-06-01 14:07:27.078 | status msg timeout | - 4 | tdengine-3.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 14:07:48.362 | status msg timeout | -Query OK, 4 row(s) in set (0.001236s) - - +1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | +2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | +5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | +6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | +Query OK, 4 row(s) in set (0.001323s) ``` -但 `drop dnode` 的行为按不会按照预期进行，且下次集群重启后，所有的 dnode 节点将无法启动 dropping 状态无法退出。 - ### 错误二 TDengine 集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： From d5e3fb7341fb53c9fce83aad0c4d657fcdea7ed6 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 10 Aug 2022 15:02:12 +0800 Subject: [PATCH 20/20] doc: remove opentsdb to tdengine temporarily --- docs/zh/25-application/{03-immigrate.md => _03-immigrate.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/25-application/{03-immigrate.md => _03-immigrate.md} (100%) diff --git a/docs/zh/25-application/03-immigrate.md b/docs/zh/25-application/_03-immigrate.md similarity index 100% rename from docs/zh/25-application/03-immigrate.md rename to docs/zh/25-application/_03-immigrate.md