From 8a095004299bf705dfa212895f8f9edbcaee3406 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 17:17:22 +0800 Subject: [PATCH 01/36] add telemetry timer --- include/common/tmsgdef.h | 5 ++-- source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mnode.c | 41 +++++++++++++++++++------- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1a63ea73a5..f43641c6f5 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -135,8 +135,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) @@ -145,7 +146,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) // Requests handled by VNODE diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 97cd6caf04..ccb523d11e 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -81,6 +81,7 @@ typedef struct SMnode { tmr_h timer; tmr_h transTimer; tmr_h mqTimer; + tmr_h telemTimer; char *path; SMnodeCfg cfg; int64_t checkTime; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7d684cd542..a009e01a52 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTransActionUpdate, .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 699ccab92c..d771dd6e17 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -36,6 +36,10 @@ #include "mndUser.h" #include "mndVgroup.h" +#define MQ_TIMER_MS 3000 +#define TRNAS_TIMER_MS 6000 +#define TELEM_TIMER_MS 86400000 + int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; @@ -73,16 +77,16 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndTransReExecute(void *param, void *tmrId) { +static void mndExecuteTransaction(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndExecuteTransaction, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -94,25 +98,39 @@ static void mndCalMqRebalance(void *param, void *tmrId) { pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); +} + +static void mndExecuteTelemetry(void *param, void *tmrId) { + SMnode *pMnode = param; + if (mndIsMaster(pMnode)) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + } + + taosTmrReset(mndExecuteTelemetry, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { - if (pMnode->timer == NULL) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); - } - + pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); if (pMnode->timer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndExecuteTransaction, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (taosTmrReset(mndCalMqRebalance, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -404,7 +422,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); From 12d1bf03ac133088b1cf32f7004c0d4d5ff9ce2d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 17:29:38 +0800 Subject: [PATCH 02/36] pull telemetry by timer --- source/dnode/mnode/impl/inc/mndInt.h | 9 ++-- source/dnode/mnode/impl/src/mndTelem.c | 70 ++++---------------------- source/dnode/mnode/impl/src/mnode.c | 12 ++--- 3 files changed, 20 insertions(+), 71 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index ccb523d11e..6c7d4be581 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -56,12 +56,9 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; + int8_t enable; + SRWLatch lock; + char email[TSDB_FQDN_LEN]; } STelemMgmt; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5beb1b10e3..a9cf8f9fab 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -234,32 +234,15 @@ static void mndSendTelemetryReport(SMnode* pMnode) { taosCloseSocket(fd); } -static void* mndTelemThreadFp(void* param) { - SMnode* pMnode = param; +static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { + SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return 0; - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("mnd-telem"); - - while (!pMgmt->exit) { - int32_t r = 0; - struct timespec ts = end; - pthread_mutex_lock(&pMgmt->lock); - r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); - pthread_mutex_unlock(&pMgmt->lock); - if (r == 0) break; - if (r != ETIMEDOUT) continue; - - if (mndIsMaster(pMnode)) { - mndSendTelemetryReport(pMnode); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; + taosWLockLatch(&pMgmt->lock); + mndSendTelemetryReport(pMnode); + taosWUnLockLatch(&pMgmt->lock); + return 0; } static void mndGetEmail(SMnode* pMnode, char* filepath) { @@ -280,43 +263,12 @@ static void mndGetEmail(SMnode* pMnode, char* filepath) { int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; pMgmt->enable = pMnode->cfg.enableTelem; - - if (!pMgmt->enable) return 0; - - pMgmt->exit = 0; - pthread_mutex_init(&pMgmt->lock, NULL); - pthread_cond_init(&pMgmt->cond, NULL); - pMgmt->email[0] = 0; - + taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); - pthread_attr_destroy(&attr); - if (code != 0) { - mDebug("failed to create telemetry thread since :%s", strerror(code)); - } - - mInfo("mnd telemetry is initialized"); + mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } -void mndCleanupTelem(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return; - - if (taosCheckPthreadValid(pMgmt->thread)) { - pthread_mutex_lock(&pMgmt->lock); - pMgmt->exit = 1; - pthread_cond_signal(&pMgmt->cond); - pthread_mutex_unlock(&pMgmt->lock); - - pthread_join(pMgmt->thread, NULL); - } - - pthread_mutex_destroy(&pMgmt->lock); - pthread_cond_destroy(&pMgmt->cond); -} +void mndCleanupTelem(SMnode* pMnode) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index d771dd6e17..b9085c0600 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -77,7 +77,7 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndExecuteTransaction(void *param, void *tmrId) { +static void mndPullupTrans(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; @@ -86,7 +86,7 @@ static void mndExecuteTransaction(void *param, void *tmrId) { pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndExecuteTransaction, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -101,7 +101,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); } -static void mndExecuteTelemetry(void *param, void *tmrId) { +static void mndPullupTelem(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; @@ -110,7 +110,7 @@ static void mndExecuteTelemetry(void *param, void *tmrId) { pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndExecuteTelemetry, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); + taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { @@ -120,7 +120,7 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndExecuteTransaction, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -130,7 +130,7 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndCalMqRebalance, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer)) { + if (taosTmrReset(mndPullupTelem, 1000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } From 28dd4f7eafeb44153bbcc769279d1d8ad4695318 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 21:24:45 +0800 Subject: [PATCH 03/36] adjust telemetry code --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndTelem.c | 95 +++++++++++++----------- source/dnode/mnode/impl/src/mnode.c | 2 + 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a41ccd9896..3410a386da 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { SSdb *pSdb = pMnode->pSdb; SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); - if (pCluster = NULL) { + if (pCluster == NULL) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index a9cf8f9fab..922910b658 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -20,9 +20,10 @@ #include "tbuffer.h" #include "tversion.h" -#define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +// #define TELEMETRY_SERVER "telemetry.taosdata.com" +#define TELEMETRY_SERVER "localhost" +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } @@ -33,25 +34,8 @@ static void mndCloseObject(SBufferWriter* bw) { } else { tbufWriteChar(bw, '}'); } - tbufWriteChar(bw, ','); } -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); -} -#endif - static void mndWriteString(SBufferWriter* bw, const char* str) { tbufWriteChar(bw, '"'); tbufWrite(bw, str, strlen(str)); @@ -61,7 +45,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) { static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { mndWriteString(bw, k); tbufWriteChar(bw, ':'); - char buf[32]; + char buf[32] = {0}; sprintf(buf, "%" PRId64, v); tbufWrite(bw, buf, strlen(buf)); tbufWriteChar(bw, ','); @@ -184,24 +168,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { } static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char buf[128] = {0}; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } + STelemMgmt* pMgmt = &pMnode->telemMgmt; + SBufferWriter bw = tbufInitWriter(NULL, false); + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; char clusterName[64] = {0}; - mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { + goto SEND_OVER; + } - SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); @@ -212,32 +189,64 @@ static void mndSendTelemetryReport(SMnode* pMnode) { mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + goto SEND_OVER; + } + const char* header = "POST /report HTTP/1.1\n" "Host: " TELEMETRY_SERVER "\n" "Content-Type: application/json\n" "Content-Length: "; + if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + goto SEND_OVER; + } - taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + int32_t contLen = (int32_t)(tbufTell(&bw)); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); + if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + goto SEND_OVER; + } + + const char* pCont = tbufGetData(&bw, false); + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + goto SEND_OVER; + } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mDebug("failed to receive response since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to receive response since %s", terrstr()); + goto SEND_OVER; } + mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); + code = 0; + +SEND_OVER: + tbufCloseWriter(&bw); taosCloseSocket(fd); + + if (code != 0) { + mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + } } static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return 0; + // if (!pMgmt->enable) return 0; taosWLockLatch(&pMgmt->lock); mndSendTelemetryReport(pMnode); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b9085c0600..27b4651b8c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -144,6 +144,8 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->transTimer = NULL; taosTmrStop(pMnode->mqTimer); pMnode->mqTimer = NULL; + taosTmrStop(pMnode->telemTimer); + pMnode->telemTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } From 90dd7746d6544c01ae866ce030d529f1a31009ee Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 21:26:50 +0800 Subject: [PATCH 04/36] minor changes --- source/dnode/mnode/impl/src/mndTelem.c | 3 +-- source/dnode/mnode/impl/src/mnode.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 922910b658..147735f356 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -20,8 +20,7 @@ #include "tbuffer.h" #include "tversion.h" -// #define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_SERVER "localhost" +#define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 #define REPORT_INTERVAL 86400 diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 27b4651b8c..4a2b61898c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -130,7 +130,7 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTelem, 1000, pMnode, pMnode->timer, &pMnode->telemTimer)) { + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } From aa6a0cb47f9fc206c7daa6fe65a94217d6735979 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 21:28:03 +0800 Subject: [PATCH 05/36] config interface --- include/libs/config/config.h | 106 ++++++++++++++++++++++++ source/dnode/mgmt/daemon/CMakeLists.txt | 1 + source/libs/CMakeLists.txt | 3 +- source/libs/config/CMakeLists.txt | 16 ++++ source/libs/config/inc/cfgInt.h | 44 ++++++++++ source/libs/config/src/cfgApolloUrl.c | 22 +++++ source/libs/config/src/cfgDotEnv.c | 22 +++++ source/libs/config/src/cfgEnvVar.c | 22 +++++ source/libs/config/src/cfgTaosFile.c | 22 +++++ source/libs/config/src/config.c | 45 ++++++++++ source/libs/config/test/CMakeLists.txt | 14 ++++ source/libs/config/test/cfgTest.cpp | 31 +++++++ 12 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 include/libs/config/config.h create mode 100644 source/libs/config/CMakeLists.txt create mode 100644 source/libs/config/inc/cfgInt.h create mode 100644 source/libs/config/src/cfgApolloUrl.c create mode 100644 source/libs/config/src/cfgDotEnv.c create mode 100644 source/libs/config/src/cfgEnvVar.c create mode 100644 source/libs/config/src/cfgTaosFile.c create mode 100644 source/libs/config/src/config.c create mode 100644 source/libs/config/test/CMakeLists.txt create mode 100644 source/libs/config/test/cfgTest.cpp diff --git a/include/libs/config/config.h b/include/libs/config/config.h new file mode 100644 index 0000000000..b25016a324 --- /dev/null +++ b/include/libs/config/config.h @@ -0,0 +1,106 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CONFIG_H_ +#define _TD_CONFIG_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { CFG_TYPE_NONE, CFG_TYPE_TAOS_CFG, CFG_TYPE_DOT_ENV, CFG_TYPE_ENV_VAR, CFG_TYPE_APOLLO_URL } ECfgType; + +typedef enum { + CFG_DYPE_NONE, + CFG_DYPE_BOOL, + CFG_DTYPE_INT8, + CFG_DTYPE_UINT8, + CFG_DTYPE_INT16, + CFG_DTYPE_UINT16, + CFG_DTYPE_INT32, + CFG_DTYPE_UINT32, + CFG_DTYPE_INT64, + CFG_DTYPE_UINT64, + CFG_DTYPE_FLOAT, + CFG_DTYPE_DOUBLE, + CFG_DTYPE_STRING, + CFG_DTYPE_FQDN, + CFG_DTYPE_IPSTR, + CFG_DTYPE_DIR, + CFG_DTYPE_FILE +} ECfgDataType; + +typedef enum { + CFG_UTYPE_NONE, + CFG_UTYPE_PERCENT, + CFG_UTYPE_GB, + CFG_UTYPE_MB, + CFG_UTYPE_BYTE, + CFG_UTYPE_SECOND, + CFG_UTYPE_MS +} ECfgUnitType; + +typedef struct SConfig SConfig; + +SConfig *cfgInit(); +int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr); +void cfgCleanup(SConfig *pConfig); + +int32_t cfgGetSize(SConfig *pConfig); +void *cfgIterate(SConfig *pConfig, void *p); +void cfgCancelIterate(SConfig *pConfig, void *p); + +void cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); +void cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype); +void cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype); +void cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype); +void cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype); +void cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype); +void cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype); +void cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype); +void cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype); +void cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype); +void cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype); +void cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +void cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +void cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +void cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +void cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); + +bool cfgGetBool(SConfig *pConfig, const char *name); +int8_t cfgGetInt8(SConfig *pConfig, const char *name); +uint8_t cfgGetUInt8(SConfig *pConfig, const char *name); +int16_t cfgGetInt16(SConfig *pConfig, const char *name); +uint16_t cfgGetUInt16(SConfig *pConfig, const char *name); +int32_t cfgGetInt32(SConfig *pConfig, const char *name); +uint32_t cfgGetUInt32(SConfig *pConfig, const char *name); +int64_t cfgGetInt64(SConfig *pConfig, const char *name); +uint64_t cfgGetUInt64(SConfig *pConfig, const char *name); +float cfgGetFloat(SConfig *pConfig, const char *name); +double cfgGetDouble(SConfig *pConfig, const char *name); +const char *cfgGetString(SConfig *pConfig, const char *name); +const char *cfgGetFqdn(SConfig *pConfig, const char *name); +const char *cfgGetIpStr(SConfig *pConfig, const char *name); +const char *cfgGetDir(SConfig *pConfig, const char *name); +const char *cfgGetFile(SConfig *pConfig, const char *name); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CONFIG_H_*/ diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index f1ce726d85..7f9a7c4d21 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(taosd ${DAEMON_SRC}) target_link_libraries( taosd PUBLIC dnode + PUBLIC config PUBLIC util PUBLIC os ) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 049b69991f..f515dda4bb 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,4 +13,5 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(nodes) +add_subdirectory(config) \ No newline at end of file diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt new file mode 100644 index 0000000000..9d899d2ff9 --- /dev/null +++ b/source/libs/config/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src CONFIG_SRC) +add_library(config ${CONFIG_SRC}) +target_include_directories( + config + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + config + PRIVATE os util +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h new file mode 100644 index 0000000000..46ba14ed03 --- /dev/null +++ b/source/libs/config/inc/cfgInt.h @@ -0,0 +1,44 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CFG_INT_H_ +#define _TD_CFG_INT_H_ + +#include "config.h" +#include "thash.h" +#include "tlockfree.h" +#include "ulog.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SConfig { + ECfgType loadType; + SRWLatch lock; + SHashObj *hash; +} SConfig; + +int32_t cfgLoadFromTaosFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromDotEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromGlobalEnvVariable(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CFG_INT_H_*/ diff --git a/source/libs/config/src/cfgApolloUrl.c b/source/libs/config/src/cfgApolloUrl.c new file mode 100644 index 0000000000..f35eca70c3 --- /dev/null +++ b/source/libs/config/src/cfgApolloUrl.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgDotEnv.c b/source/libs/config/src/cfgDotEnv.c new file mode 100644 index 0000000000..e65dddae7e --- /dev/null +++ b/source/libs/config/src/cfgDotEnv.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromDotEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from .env file %s", filepath); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvVar.c b/source/libs/config/src/cfgEnvVar.c new file mode 100644 index 0000000000..caa5a88110 --- /dev/null +++ b/source/libs/config/src/cfgEnvVar.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromGlobalEnvVariable(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgTaosFile.c b/source/libs/config/src/cfgTaosFile.c new file mode 100644 index 0000000000..7d5b4e4743 --- /dev/null +++ b/source/libs/config/src/cfgTaosFile.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromTaosFile(SConfig *pConfig, const char *filepath) { + uInfo("load from .cfg file %s", filepath); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c new file mode 100644 index 0000000000..23fbfc1dc0 --- /dev/null +++ b/source/libs/config/src/config.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +SConfig *cfgInit() { + SConfig *pConfig = calloc(1, sizeof(SConfig)); + return pConfig; +} + +int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr) { + switch (cfgType) { + case CFG_TYPE_TAOS_CFG: + return cfgLoadFromTaosFile(pConfig, sourceStr); + case CFG_TYPE_DOT_ENV: + return cfgLoadFromDotEnvFile(pConfig, sourceStr); + case CFG_TYPE_ENV_VAR: + return cfgLoadFromGlobalEnvVariable(pConfig); + case CFG_TYPE_APOLLO_URL: + return cfgLoadFromApollUrl(pConfig, sourceStr); + default: + return -1; + } +} + +void cfgCleanup(SConfig *pConfig) { free(pConfig); } + +int32_t cfgGetSize(SConfig *pConfig) { return 0; } + +void *cfgIterate(SConfig *pConfig, void *p) { return NULL; } + +void cfgCancelIterate(SConfig *pConfig, void *p); diff --git a/source/libs/config/test/CMakeLists.txt b/source/libs/config/test/CMakeLists.txt new file mode 100644 index 0000000000..1e63d5025d --- /dev/null +++ b/source/libs/config/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. CFG_TEST_SRC) +add_executable(cfg_test ${CFG_TEST_SRC}) +target_link_libraries( + cfg_test + PUBLIC config + PUBLIC gtest_main +) + +add_test( + NAME cfg_test + COMMAND cfg_test +) diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp new file mode 100644 index 0000000000..ba58a5b977 --- /dev/null +++ b/source/libs/config/test/cfgTest.cpp @@ -0,0 +1,31 @@ +/** + * @file cfgTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief config module tests + * @version 1.0 + * @date 2022-02-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "os.h" + +#include "tfs.h" + +class CfgTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + + static const char *pConfig; +}; + +const char *TfsTest::pConfig; + +TEST_F(CfgTest, 01_Taos_File) {} From 58bbd6f28fb43d7d4dcb1530a04cd5ca94da7978 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 21:31:38 +0800 Subject: [PATCH 06/36] config compile error --- source/libs/config/CMakeLists.txt | 5 +---- source/libs/config/test/cfgTest.cpp | 6 ++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt index 9d899d2ff9..2726aec98b 100644 --- a/source/libs/config/CMakeLists.txt +++ b/source/libs/config/CMakeLists.txt @@ -6,10 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - config - PRIVATE os util -) +target_link_libraries(config os util) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index ba58a5b977..674d3d2196 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -10,9 +10,7 @@ */ #include -#include "os.h" - -#include "tfs.h" +#include "config.h" class CfgTest : public ::testing::Test { protected: @@ -26,6 +24,6 @@ class CfgTest : public ::testing::Test { static const char *pConfig; }; -const char *TfsTest::pConfig; +const char *CfgTest::pConfig; TEST_F(CfgTest, 01_Taos_File) {} From f22d662b88ede89b3ea06cbfcba0e26cd6ec9537 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Feb 2022 21:37:05 +0800 Subject: [PATCH 07/36] adjust interface --- include/libs/config/config.h | 17 +++++++++++++---- tests | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index b25016a324..fb484ff379 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -23,7 +23,14 @@ extern "C" { #endif -typedef enum { CFG_TYPE_NONE, CFG_TYPE_TAOS_CFG, CFG_TYPE_DOT_ENV, CFG_TYPE_ENV_VAR, CFG_TYPE_APOLLO_URL } ECfgType; +typedef enum { + CFG_TYPE_NONE, + CFG_TYPE_TAOS_CFG, + CFG_TYPE_DOT_ENV, + CFG_TYPE_ENV_VAR, + CFG_TYPE_APOLLO_URL, + CFG_TYPE_CONSOLE_PARA +} ECfgType; typedef enum { CFG_DYPE_NONE, @@ -61,9 +68,11 @@ SConfig *cfgInit(); int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr); void cfgCleanup(SConfig *pConfig); -int32_t cfgGetSize(SConfig *pConfig); -void *cfgIterate(SConfig *pConfig, void *p); -void cfgCancelIterate(SConfig *pConfig, void *p); +int32_t cfgGetSize(SConfig *pConfig); +void *cfgIterate(SConfig *pConfig, void *p); +void cfgCancelIterate(SConfig *pConfig, void *p); +ECfgUnitType cfgGetUtype(SConfig *pConfig, const char *name); +ECfgDataType cfgGetDtype(SConfig *pConfig, const char *name); void cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); void cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype); diff --git a/tests b/tests index 4da4bb00c5..498e17e225 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 4da4bb00c5615f0b1bea04340abef5d9d15a96b4 +Subproject commit 498e17e225c454f2b45ff16eaa6d3eb595b107cf From 863ad191d53e6da8e9f4c38e34dd1d7ea0c1c935 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Feb 2022 15:09:10 +0800 Subject: [PATCH 08/36] config interface --- include/libs/config/config.h | 87 +++++++++++--------- source/libs/config/inc/cfgInt.h | 5 +- source/libs/config/src/config.c | 138 +++++++++++++++++++++++++++++++- 3 files changed, 183 insertions(+), 47 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index fb484ff379..a04ebad981 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -24,7 +24,7 @@ extern "C" { #endif typedef enum { - CFG_TYPE_NONE, + CFG_TYPE_DEFAULT, CFG_TYPE_TAOS_CFG, CFG_TYPE_DOT_ENV, CFG_TYPE_ENV_VAR, @@ -33,8 +33,8 @@ typedef enum { } ECfgType; typedef enum { - CFG_DYPE_NONE, - CFG_DYPE_BOOL, + CFG_DTYPE_NONE, + CFG_DTYPE_BOOL, CFG_DTYPE_INT8, CFG_DTYPE_UINT8, CFG_DTYPE_INT16, @@ -62,6 +62,31 @@ typedef enum { CFG_UTYPE_MS } ECfgUnitType; +typedef struct SConfigItem { + ECfgType stype; + ECfgUnitType utype; + ECfgDataType dtype; + const char *name; + union { + bool boolVal; + uint8_t uint8Val; + int8_t int8Val; + uint16_t uint16Val; + int16_t int16Val; + uint32_t uint32Val; + int32_t int32Val; + uint64_t uint64Val; + int64_t int64Val; + float floatVal; + double doubleVal; + char *strVal; + char *fqdnVal; + char *ipstrVal; + char *dirVal; + char *fileVal; + }; +} SConfigItem; + typedef struct SConfig SConfig; SConfig *cfgInit(); @@ -69,44 +94,26 @@ int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr); void cfgCleanup(SConfig *pConfig); int32_t cfgGetSize(SConfig *pConfig); -void *cfgIterate(SConfig *pConfig, void *p); -void cfgCancelIterate(SConfig *pConfig, void *p); -ECfgUnitType cfgGetUtype(SConfig *pConfig, const char *name); -ECfgDataType cfgGetDtype(SConfig *pConfig, const char *name); +SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter); +void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter); +SConfigItem *cfgGetItem(SConfig *pConfig, const char *name); -void cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); -void cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype); -void cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype); -void cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype); -void cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype); -void cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype); -void cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype); -void cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype); -void cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype); -void cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype); -void cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype); -void cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -void cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -void cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -void cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -void cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); - -bool cfgGetBool(SConfig *pConfig, const char *name); -int8_t cfgGetInt8(SConfig *pConfig, const char *name); -uint8_t cfgGetUInt8(SConfig *pConfig, const char *name); -int16_t cfgGetInt16(SConfig *pConfig, const char *name); -uint16_t cfgGetUInt16(SConfig *pConfig, const char *name); -int32_t cfgGetInt32(SConfig *pConfig, const char *name); -uint32_t cfgGetUInt32(SConfig *pConfig, const char *name); -int64_t cfgGetInt64(SConfig *pConfig, const char *name); -uint64_t cfgGetUInt64(SConfig *pConfig, const char *name); -float cfgGetFloat(SConfig *pConfig, const char *name); -double cfgGetDouble(SConfig *pConfig, const char *name); -const char *cfgGetString(SConfig *pConfig, const char *name); -const char *cfgGetFqdn(SConfig *pConfig, const char *name); -const char *cfgGetIpStr(SConfig *pConfig, const char *name); -const char *cfgGetDir(SConfig *pConfig, const char *name); -const char *cfgGetFile(SConfig *pConfig, const char *name); +int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); +int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype); +int32_t cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype); +int32_t cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype); +int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype); +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype); +int32_t cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype); +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype); +int32_t cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype); +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype); +int32_t cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype); +int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); #ifdef __cplusplus } diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 46ba14ed03..2c4b05af7f 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -18,8 +18,8 @@ #define _TD_CFG_INT_H_ #include "config.h" +#include "taoserror.h" #include "thash.h" -#include "tlockfree.h" #include "ulog.h" #ifdef __cplusplus @@ -27,8 +27,7 @@ extern "C" { #endif typedef struct SConfig { - ECfgType loadType; - SRWLatch lock; + ECfgType stype; SHashObj *hash; } SConfig; diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 23fbfc1dc0..acebfee32b 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -36,10 +36,140 @@ int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr) { } } -void cfgCleanup(SConfig *pConfig) { free(pConfig); } +void cfgCleanup(SConfig *pConfig) { + if (pConfig == NULL) return; + if (pConfig->hash != NULL) { + taosHashCleanup(pConfig->hash); + pConfig->hash == NULL; + } +} -int32_t cfgGetSize(SConfig *pConfig) { return 0; } +int32_t cfgGetSize(SConfig *pConfig) { return taosHashGetSize(pConfig->hash); } -void *cfgIterate(SConfig *pConfig, void *p) { return NULL; } +SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashIterate(pConfig->hash, pIter); } -void cfgCancelIterate(SConfig *pConfig, void *p); +void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashCancelIterate(pConfig->hash, pIter); } + +SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { taosHashGet(pConfig->hash, name, strlen(name) + 1); } + +static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name, ECfgUnitType utype) { + pItem->stype = CFG_TYPE_DEFAULT; + pItem->utype = utype; + pItem->name = strdup(name); + if (pItem->name != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (taosHashPut(pConfig->hash, name, strlen(name) + 1, pItem, sizeof(SConfigItem)) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .boolVal = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_INT8, .int8Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_UINT8, .uint8Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_INT16, .int16Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_UINT16, .uint16Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .int32Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_UINT32, .uint32Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .int64Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_UINT64, .uint64Val = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .floatVal = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_DOUBLE, .doubleVal = defaultVal}; + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING}; + item.strVal = strdup(defaultVal); + if (item.strVal != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_FQDN}; + item.fqdnVal = strdup(defaultVal); + if (item.fqdnVal != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; + item.ipstrVal = strdup(defaultVal); + if (item.ipstrVal != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR}; + item.dirVal = strdup(defaultVal); + if (item.dirVal != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); +} + +int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_FILE}; + item.fileVal = strdup(defaultVal); + if (item.fileVal != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); +} From 48a2d415d12424963fc77371d745ed2d807ebefb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Feb 2022 16:04:37 +0800 Subject: [PATCH 09/36] cfgtest --- include/libs/config/config.h | 2 +- source/libs/config/CMakeLists.txt | 2 +- source/libs/config/src/config.c | 25 ++++ source/libs/config/test/cfgTest.cpp | 200 +++++++++++++++++++++++++++- 4 files changed, 226 insertions(+), 3 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index a04ebad981..e80faf88bc 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -66,7 +66,7 @@ typedef struct SConfigItem { ECfgType stype; ECfgUnitType utype; ECfgDataType dtype; - const char *name; + char *name; union { bool boolVal; uint8_t uint8Val; diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt index 2726aec98b..0bd467198d 100644 --- a/source/libs/config/CMakeLists.txt +++ b/source/libs/config/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src CONFIG_SRC) -add_library(config ${CONFIG_SRC}) +add_library(config STATIC ${CONFIG_SRC}) target_include_directories( config PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config" diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index acebfee32b..f3635ef36d 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -18,6 +18,18 @@ SConfig *cfgInit() { SConfig *pConfig = calloc(1, sizeof(SConfig)); + if (pConfig == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pConfig->hash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if (pConfig->hash == NULL) { + free(pConfig); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + return pConfig; } @@ -62,6 +74,19 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name } if (taosHashPut(pConfig->hash, name, strlen(name) + 1, pItem, sizeof(SConfigItem)) != 0) { + if (pItem->dtype == CFG_DTYPE_STRING) { + free(pItem->strVal); + } else if (pItem->dtype == CFG_DTYPE_FQDN) { + free(pItem->fqdnVal); + } else if (pItem->dtype == CFG_DTYPE_IPSTR) { + free(pItem->ipstrVal); + } else if (pItem->dtype == CFG_DTYPE_DIR) { + free(pItem->dirVal); + } else if (pItem->dtype == CFG_DTYPE_FILE) { + free(pItem->fileVal); + } else { + } + free(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index 674d3d2196..1375754443 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -21,9 +21,207 @@ class CfgTest : public ::testing::Test { void SetUp() override {} void TearDown() override {} + void InitializeConfig(SConfig *pConfig); + static const char *pConfig; }; const char *CfgTest::pConfig; -TEST_F(CfgTest, 01_Taos_File) {} +TEST_F(CfgTest, 01_Basic) { + SConfig *pConfig = cfgInit(); + cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE); + cfgAddInt8(pConfig, "test_int8", 1, CFG_UTYPE_GB); + cfgAddUInt8(pConfig, "test_uint8", 2, CFG_UTYPE_MB); + cfgAddInt16(pConfig, "test_int16", 3, CFG_UTYPE_BYTE); + cfgAddUInt16(pConfig, "test_uint16", 4, CFG_UTYPE_SECOND); + cfgAddInt32(pConfig, "test_int32", 5, CFG_UTYPE_MS); + cfgAddUInt32(pConfig, "test_uint32", 6, CFG_UTYPE_PERCENT); + cfgAddInt64(pConfig, "test_int64", 7, CFG_UTYPE_NONE); + cfgAddUInt64(pConfig, "test_uint64", 8, CFG_UTYPE_NONE); + cfgAddFloat(pConfig, "test_float", 9, CFG_UTYPE_NONE); + cfgAddDouble(pConfig, "test_double", 10, CFG_UTYPE_NONE); + cfgAddString(pConfig, "test_string", "11", CFG_UTYPE_NONE); + cfgAddFqdn(pConfig, "test_fqdn", "localhost", CFG_UTYPE_NONE); + cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE); + cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE); + cfgAddFile(pConfig, "test_file", "/tmp/file1", CFG_UTYPE_NONE); + + EXPECT_EQ(cfgGetSize(pConfig), 16); + + int32_t size = 0; + SConfigItem *pItem = cfgIterate(pConfig, NULL); + while (pItem != NULL) { + pItem = cfgIterate(pConfig, pItem); + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("cfg:%s, value:%d\n", pItem->name, pItem->boolVal); + break; + CFG_DTYPE_INT8: + printf("cfg:%s, value:%d\n", pItem->name, pItem->uint8Val); + break; + CFG_DTYPE_UINT8: + printf("cfg:%s, value:%d\n", pItem->name, pItem->int8Val); + break; + CFG_DTYPE_INT16: + printf("cfg:%s, value:%d\n", pItem->name, pItem->uint16Val); + break; + CFG_DTYPE_UINT16: + printf("cfg:%s, value:%d\n", pItem->name, pItem->int16Val); + break; + CFG_DTYPE_INT32: + printf("cfg:%s, value:%d\n", pItem->name, pItem->uint32Val); + break; + CFG_DTYPE_UINT32: + printf("cfg:%s, value:%d\n", pItem->name, pItem->int32Val); + break; + CFG_DTYPE_INT64: + printf("cfg:%s, value:%" PRIu64, pItem->name, pItem->uint64Val); + break; + CFG_DTYPE_UINT64: + printf("cfg:%s, value:%" PRId64, pItem->name, pItem->int64Val); + break; + CFG_DTYPE_FLOAT: + printf("cfg:%s, value:%f\n", pItem->name, pItem->floatVal); + break; + CFG_DTYPE_DOUBLE: + printf("cfg:%s, value:%f\n", pItem->name, pItem->doubleVal); + break; + CFG_DTYPE_STRING: + printf("cfg:%s, value:%s\n", pItem->name, pItem->strVal); + break; + CFG_DTYPE_FQDN: + printf("cfg:%s, value:%s\n", pItem->name, pItem->fqdnVal); + break; + CFG_DTYPE_IPSTR: + printf("cfg:%s, value:%s\n", pItem->name, pItem->ipstrVal); + break; + CFG_DTYPE_DIR: + printf("cfg:%s, value:%s\n", pItem->name, pItem->dirVal); + break; + CFG_DTYPE_FILE: + printf("cfg:%s, value:%s\n", pItem->name, pItem->fileVal); + break; + default: + break; + } + size++; + } + cfgCancelIterate(pConfig, pItem); + + EXPECT_EQ(cfgGetSize(pConfig), 16); + + pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->boolVal, 0); + + pItem = cfgGetItem(pConfig, "test_int8"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_GB); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT8); + EXPECT_STREQ(pItem->name, "test_int8"); + EXPECT_EQ(pItem->int8Val, 1); + + pItem = cfgGetItem(pConfig, "test_uint8"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_MB); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT8); + EXPECT_STREQ(pItem->name, "test_uint8"); + EXPECT_EQ(pItem->uint8Val, 2); + + pItem = cfgGetItem(pConfig, "test_int16"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_BYTE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT16); + EXPECT_STREQ(pItem->name, "test_int16"); + EXPECT_EQ(pItem->int16Val, 3); + + pItem = cfgGetItem(pConfig, "test_uint16"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT16); + EXPECT_STREQ(pItem->name, "test_uint16"); + EXPECT_EQ(pItem->uint16Val, 4); + + pItem = cfgGetItem(pConfig, "test_int32"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_MS); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); + EXPECT_STREQ(pItem->name, "test_int32"); + EXPECT_EQ(pItem->int32Val, 5); + + pItem = cfgGetItem(pConfig, "test_uint32"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_PERCENT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT32); + EXPECT_STREQ(pItem->name, "test_uint32"); + EXPECT_EQ(pItem->uint32Val, 6); + + pItem = cfgGetItem(pConfig, "test_int64"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); + EXPECT_EQ(pItem->dtype, CFG_UTYPE_NONE); + EXPECT_STREQ(pItem->name, "test_int64"); + EXPECT_EQ(pItem->int64Val, 7); + + pItem = cfgGetItem(pConfig, "test_uint64"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT64); + EXPECT_STREQ(pItem->name, "test_uint64"); + EXPECT_EQ(pItem->uint64Val, 8); + + pItem = cfgGetItem(pConfig, "test_float"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); + EXPECT_STREQ(pItem->name, "test_float"); + EXPECT_EQ(pItem->floatVal, 9); + + pItem = cfgGetItem(pConfig, "test_double"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DOUBLE); + EXPECT_STREQ(pItem->name, "test_double"); + EXPECT_EQ(pItem->doubleVal, 10); + + pItem = cfgGetItem(pConfig, "test_string"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); + EXPECT_STREQ(pItem->name, "test_string"); + EXPECT_STREQ(pItem->strVal, "11"); + + pItem = cfgGetItem(pConfig, "test_fqdn"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FQDN); + EXPECT_STREQ(pItem->name, "test_fqdn"); + EXPECT_STREQ(pItem->strVal, "localhost"); + + pItem = cfgGetItem(pConfig, "test_ipstr"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); + EXPECT_STREQ(pItem->name, "test_ipstr"); + EXPECT_STREQ(pItem->ipstrVal, "192.168.0.1"); + + pItem = cfgGetItem(pConfig, "test_dir"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); + EXPECT_STREQ(pItem->name, "test_dir"); + EXPECT_STREQ(pItem->dirVal, "/tmp"); + + pItem = cfgGetItem(pConfig, "test_file"); + EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FILE); + EXPECT_STREQ(pItem->name, "test_file"); + EXPECT_STREQ(pItem->fileVal, "/tmp/file1"); + + cfgCleanup(pConfig); +} From 2f97bd24ce5154af4483bbf5ec050b7bfc3fe17a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Feb 2022 17:33:37 +0800 Subject: [PATCH 10/36] unitest for config --- include/libs/config/config.h | 12 ++- source/libs/config/inc/cfgInt.h | 2 +- source/libs/config/src/config.c | 105 +++++++++++++++++--- source/libs/config/test/cfgTest.cpp | 142 ++++++++++++++++++---------- 4 files changed, 193 insertions(+), 68 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index e80faf88bc..c785c5d797 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -25,12 +25,12 @@ extern "C" { typedef enum { CFG_TYPE_DEFAULT, - CFG_TYPE_TAOS_CFG, + CFG_TYPE_CFG_FILE, CFG_TYPE_DOT_ENV, CFG_TYPE_ENV_VAR, CFG_TYPE_APOLLO_URL, CFG_TYPE_CONSOLE_PARA -} ECfgType; +} ECfgSrcType; typedef enum { CFG_DTYPE_NONE, @@ -63,7 +63,7 @@ typedef enum { } ECfgUnitType; typedef struct SConfigItem { - ECfgType stype; + ECfgSrcType stype; ECfgUnitType utype; ECfgDataType dtype; char *name; @@ -90,7 +90,7 @@ typedef struct SConfigItem { typedef struct SConfig SConfig; SConfig *cfgInit(); -int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr); +int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr); void cfgCleanup(SConfig *pConfig); int32_t cfgGetSize(SConfig *pConfig); @@ -115,6 +115,10 @@ int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +const char *cfgStypeStr(ECfgSrcType type); +const char *cfgDtypeStr(ECfgDataType type); +const char *cfgUtypeStr(ECfgUnitType type); + #ifdef __cplusplus } #endif diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 2c4b05af7f..e441e401fb 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -27,7 +27,7 @@ extern "C" { #endif typedef struct SConfig { - ECfgType stype; + ECfgSrcType stype; SHashObj *hash; } SConfig; diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index f3635ef36d..77cde8221e 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -33,9 +33,9 @@ SConfig *cfgInit() { return pConfig; } -int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr) { +int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr) { switch (cfgType) { - case CFG_TYPE_TAOS_CFG: + case CFG_TYPE_CFG_FILE: return cfgLoadFromTaosFile(pConfig, sourceStr); case CFG_TYPE_DOT_ENV: return cfgLoadFromDotEnvFile(pConfig, sourceStr); @@ -49,10 +49,12 @@ int32_t cfgLoad(SConfig *pConfig, ECfgType cfgType, const char *sourceStr) { } void cfgCleanup(SConfig *pConfig) { - if (pConfig == NULL) return; - if (pConfig->hash != NULL) { - taosHashCleanup(pConfig->hash); - pConfig->hash == NULL; + if (pConfig != NULL) { + if (pConfig->hash != NULL) { + taosHashCleanup(pConfig->hash); + pConfig->hash == NULL; + } + free(pConfig); } } @@ -68,7 +70,7 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name pItem->stype = CFG_TYPE_DEFAULT; pItem->utype = utype; pItem->name = strdup(name); - if (pItem->name != NULL) { + if (pItem->name == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -152,7 +154,7 @@ int32_t cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfg int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { SConfigItem item = {.dtype = CFG_DTYPE_STRING}; item.strVal = strdup(defaultVal); - if (item.strVal != NULL) { + if (item.strVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -162,7 +164,7 @@ int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { SConfigItem item = {.dtype = CFG_DTYPE_FQDN}; item.fqdnVal = strdup(defaultVal); - if (item.fqdnVal != NULL) { + if (item.fqdnVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -172,7 +174,7 @@ int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, E int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; item.ipstrVal = strdup(defaultVal); - if (item.ipstrVal != NULL) { + if (item.ipstrVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -182,7 +184,7 @@ int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { SConfigItem item = {.dtype = CFG_DTYPE_DIR}; item.dirVal = strdup(defaultVal); - if (item.dirVal != NULL) { + if (item.dirVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -192,9 +194,88 @@ int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, EC int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { SConfigItem item = {.dtype = CFG_DTYPE_FILE}; item.fileVal = strdup(defaultVal); - if (item.fileVal != NULL) { + if (item.fileVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } return cfgAddItem(pConfig, &item, name, utype); } + +const char *cfgStypeStr(ECfgSrcType type) { + switch (type) { + case CFG_TYPE_DEFAULT: + return "default"; + case CFG_TYPE_CFG_FILE: + return "cfg"; + case CFG_TYPE_DOT_ENV: + return ".env"; + case CFG_TYPE_ENV_VAR: + return "env"; + case CFG_TYPE_APOLLO_URL: + return "apollo"; + default: + return "invalid"; + } +} + +const char *cfgDtypeStr(ECfgDataType type) { + switch (type) { + case CFG_DTYPE_NONE: + return "none"; + case CFG_DTYPE_BOOL: + return "bool"; + case CFG_DTYPE_INT8: + return "int8"; + case CFG_DTYPE_UINT8: + return "uint8"; + case CFG_DTYPE_INT16: + return "int16"; + case CFG_DTYPE_UINT16: + return "uint16"; + case CFG_DTYPE_INT32: + return "int32"; + case CFG_DTYPE_UINT32: + return "uint32"; + case CFG_DTYPE_INT64: + return "int64"; + case CFG_DTYPE_UINT64: + return "uint64"; + case CFG_DTYPE_FLOAT: + return "float"; + case CFG_DTYPE_DOUBLE: + return "double"; + case CFG_DTYPE_STRING: + return "string"; + case CFG_DTYPE_FQDN: + return "fqdn"; + case CFG_DTYPE_IPSTR: + return "ipstr"; + case CFG_DTYPE_DIR: + return "dir"; + case CFG_DTYPE_FILE: + return "file"; + default: + return "invalid"; + } +} + +const char *cfgUtypeStr(ECfgUnitType type) { + switch (type) { + case CFG_UTYPE_NONE: + return ""; + case CFG_UTYPE_PERCENT: + return "(%)"; + case CFG_UTYPE_GB: + return "(GB)"; + case CFG_UTYPE_MB: + return "(Mb)"; + case CFG_UTYPE_BYTE: + return "(byte)"; + case CFG_UTYPE_SECOND: + return "(s)"; + case CFG_UTYPE_MS: + return "(ms)"; + default: + return "invalid"; + } +} \ No newline at end of file diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index 1375754443..d198d0bfe1 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -28,84 +28,124 @@ class CfgTest : public ::testing::Test { const char *CfgTest::pConfig; -TEST_F(CfgTest, 01_Basic) { +TEST_F(CfgTest, 02_Str) { + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_CFG_FILE), "cfg"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_DOT_ENV), ".env"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_VAR), "env"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_APOLLO_URL), "apollo"); + EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); + + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT8), "int8"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT8), "uint8"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT16), "int16"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT16), "uint16"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT32), "uint32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT64), "uint64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DOUBLE), "double"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FQDN), "fqdn"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FILE), "file"); + EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); + + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_NONE), ""); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_PERCENT), "(%)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_GB), "(GB)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MB), "(Mb)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_BYTE), "(byte)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_SECOND), "(s)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MS), "(ms)"); + EXPECT_STREQ(cfgUtypeStr(ECfgUnitType(1024)), "invalid"); +} + +TEST_F(CfgTest, 02_Basic) { SConfig *pConfig = cfgInit(); - cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE); - cfgAddInt8(pConfig, "test_int8", 1, CFG_UTYPE_GB); - cfgAddUInt8(pConfig, "test_uint8", 2, CFG_UTYPE_MB); - cfgAddInt16(pConfig, "test_int16", 3, CFG_UTYPE_BYTE); - cfgAddUInt16(pConfig, "test_uint16", 4, CFG_UTYPE_SECOND); - cfgAddInt32(pConfig, "test_int32", 5, CFG_UTYPE_MS); - cfgAddUInt32(pConfig, "test_uint32", 6, CFG_UTYPE_PERCENT); - cfgAddInt64(pConfig, "test_int64", 7, CFG_UTYPE_NONE); - cfgAddUInt64(pConfig, "test_uint64", 8, CFG_UTYPE_NONE); - cfgAddFloat(pConfig, "test_float", 9, CFG_UTYPE_NONE); - cfgAddDouble(pConfig, "test_double", 10, CFG_UTYPE_NONE); - cfgAddString(pConfig, "test_string", "11", CFG_UTYPE_NONE); - cfgAddFqdn(pConfig, "test_fqdn", "localhost", CFG_UTYPE_NONE); - cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE); - cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE); - cfgAddFile(pConfig, "test_file", "/tmp/file1", CFG_UTYPE_NONE); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddInt8(pConfig, "test_int8", 1, CFG_UTYPE_GB), 0); + EXPECT_EQ(cfgAddUInt8(pConfig, "test_uint8", 2, CFG_UTYPE_MB), 0); + EXPECT_EQ(cfgAddInt16(pConfig, "test_int16", 3, CFG_UTYPE_BYTE), 0); + EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 4, CFG_UTYPE_SECOND), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 5, CFG_UTYPE_MS), 0); + EXPECT_EQ(cfgAddUInt32(pConfig, "test_uint32", 6, CFG_UTYPE_PERCENT), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 7, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddUInt64(pConfig, "test_uint64", 8, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 9, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddDouble(pConfig, "test_double", 10, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "11", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddFqdn(pConfig, "test_fqdn", "localhost", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddFile(pConfig, "test_file", "/tmp/file1", CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgGetSize(pConfig), 16); int32_t size = 0; SConfigItem *pItem = cfgIterate(pConfig, NULL); while (pItem != NULL) { - pItem = cfgIterate(pConfig, pItem); switch (pItem->dtype) { case CFG_DTYPE_BOOL: - printf("cfg:%s, value:%d\n", pItem->name, pItem->boolVal); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->boolVal); break; - CFG_DTYPE_INT8: - printf("cfg:%s, value:%d\n", pItem->name, pItem->uint8Val); + case CFG_DTYPE_INT8: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint8Val); break; - CFG_DTYPE_UINT8: - printf("cfg:%s, value:%d\n", pItem->name, pItem->int8Val); + case CFG_DTYPE_UINT8: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int8Val); break; - CFG_DTYPE_INT16: - printf("cfg:%s, value:%d\n", pItem->name, pItem->uint16Val); + case CFG_DTYPE_INT16: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint16Val); break; - CFG_DTYPE_UINT16: - printf("cfg:%s, value:%d\n", pItem->name, pItem->int16Val); + case CFG_DTYPE_UINT16: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int16Val); break; - CFG_DTYPE_INT32: - printf("cfg:%s, value:%d\n", pItem->name, pItem->uint32Val); + case CFG_DTYPE_INT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint32Val); break; - CFG_DTYPE_UINT32: - printf("cfg:%s, value:%d\n", pItem->name, pItem->int32Val); + case CFG_DTYPE_UINT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int32Val); break; - CFG_DTYPE_INT64: - printf("cfg:%s, value:%" PRIu64, pItem->name, pItem->uint64Val); + case CFG_DTYPE_INT64: + printf("index:%d, cfg:%s value:%" PRIu64 "\n", size, pItem->name, pItem->uint64Val); break; - CFG_DTYPE_UINT64: - printf("cfg:%s, value:%" PRId64, pItem->name, pItem->int64Val); + case CFG_DTYPE_UINT64: + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->int64Val); break; - CFG_DTYPE_FLOAT: - printf("cfg:%s, value:%f\n", pItem->name, pItem->floatVal); + case CFG_DTYPE_FLOAT: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->floatVal); break; - CFG_DTYPE_DOUBLE: - printf("cfg:%s, value:%f\n", pItem->name, pItem->doubleVal); + case CFG_DTYPE_DOUBLE: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->doubleVal); break; - CFG_DTYPE_STRING: - printf("cfg:%s, value:%s\n", pItem->name, pItem->strVal); + case CFG_DTYPE_STRING: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); break; - CFG_DTYPE_FQDN: - printf("cfg:%s, value:%s\n", pItem->name, pItem->fqdnVal); + case CFG_DTYPE_FQDN: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->fqdnVal); break; - CFG_DTYPE_IPSTR: - printf("cfg:%s, value:%s\n", pItem->name, pItem->ipstrVal); + case CFG_DTYPE_IPSTR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->ipstrVal); break; - CFG_DTYPE_DIR: - printf("cfg:%s, value:%s\n", pItem->name, pItem->dirVal); + case CFG_DTYPE_DIR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->dirVal); break; - CFG_DTYPE_FILE: - printf("cfg:%s, value:%s\n", pItem->name, pItem->fileVal); + case CFG_DTYPE_FILE: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->fileVal); break; default: + printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); break; } size++; + pItem = cfgIterate(pConfig, pItem); } cfgCancelIterate(pConfig, pItem); @@ -162,8 +202,8 @@ TEST_F(CfgTest, 01_Basic) { pItem = cfgGetItem(pConfig, "test_int64"); EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); - EXPECT_EQ(pItem->dtype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); EXPECT_STREQ(pItem->name, "test_int64"); EXPECT_EQ(pItem->int64Val, 7); From 7d4ae77521f705499b16ecc1a46f1bf77da8ee24 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Feb 2022 17:47:03 +0800 Subject: [PATCH 11/36] rename var --- source/libs/config/inc/cfgInt.h | 2 +- source/libs/config/src/{cfgTaosFile.c => cfgCfgFile.c} | 2 +- source/libs/config/src/config.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename source/libs/config/src/{cfgTaosFile.c => cfgCfgFile.c} (91%) diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index e441e401fb..56cc58e658 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -31,7 +31,7 @@ typedef struct SConfig { SHashObj *hash; } SConfig; -int32_t cfgLoadFromTaosFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromDotEnvFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromGlobalEnvVariable(SConfig *pConfig); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); diff --git a/source/libs/config/src/cfgTaosFile.c b/source/libs/config/src/cfgCfgFile.c similarity index 91% rename from source/libs/config/src/cfgTaosFile.c rename to source/libs/config/src/cfgCfgFile.c index 7d5b4e4743..9cf36997eb 100644 --- a/source/libs/config/src/cfgTaosFile.c +++ b/source/libs/config/src/cfgCfgFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" -int32_t cfgLoadFromTaosFile(SConfig *pConfig, const char *filepath) { +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { uInfo("load from .cfg file %s", filepath); return 0; } \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 77cde8221e..9650489655 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -36,7 +36,7 @@ SConfig *cfgInit() { int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr) { switch (cfgType) { case CFG_TYPE_CFG_FILE: - return cfgLoadFromTaosFile(pConfig, sourceStr); + return cfgLoadFromCfgFile(pConfig, sourceStr); case CFG_TYPE_DOT_ENV: return cfgLoadFromDotEnvFile(pConfig, sourceStr); case CFG_TYPE_ENV_VAR: From 1244bbe49a0ea7c3a0eec5c4ef3a02e87d894245 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Feb 2022 18:04:22 +0800 Subject: [PATCH 12/36] integrate cfg --- include/libs/config/config.h | 2 +- source/dnode/mgmt/daemon/CMakeLists.txt | 10 ++-- source/dnode/mgmt/daemon/inc/dmnInt.h | 38 ++++++++++++++ source/dnode/mgmt/daemon/src/daemon.c | 38 ++------------ source/dnode/mgmt/daemon/src/dmnCfg.c | 50 +++++++++++++++++++ source/libs/config/inc/cfgInt.h | 4 +- .../config/src/{cfgDotEnv.c => cfgEnvFile.c} | 4 +- source/libs/config/src/cfgEnvVar.c | 2 +- source/libs/config/src/config.c | 16 +++--- source/libs/config/test/cfgTest.cpp | 8 +-- 10 files changed, 116 insertions(+), 56 deletions(-) create mode 100644 source/dnode/mgmt/daemon/inc/dmnInt.h create mode 100644 source/dnode/mgmt/daemon/src/dmnCfg.c rename source/libs/config/src/{cfgDotEnv.c => cfgEnvFile.c} (85%) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index c785c5d797..3806d9f605 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -26,7 +26,7 @@ extern "C" { typedef enum { CFG_TYPE_DEFAULT, CFG_TYPE_CFG_FILE, - CFG_TYPE_DOT_ENV, + CFG_TYPE_ENV_FILE, CFG_TYPE_ENV_VAR, CFG_TYPE_APOLLO_URL, CFG_TYPE_CONSOLE_PARA diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index 7f9a7c4d21..a577344761 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -1,9 +1,9 @@ aux_source_directory(src DAEMON_SRC) add_executable(taosd ${DAEMON_SRC}) -target_link_libraries( + +target_include_directories( taosd - PUBLIC dnode - PUBLIC config - PUBLIC util - PUBLIC os + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +target_link_libraries(taosd dnode config util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h new file mode 100644 index 0000000000..1bba29bbd2 --- /dev/null +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -0,0 +1,38 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DMN_INT_H_ +#define _TD_DMN_INT_H_ + +#include "config.h" +#include "dnode.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "ulog.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dnmInitCfg(SDnodeEnvCfg *pEnvCfg, SDnodeObjCfg *pObjCfg, const char *configFile, const char *envFile, + const char *apolloUrl); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DMN_INT_H_*/ diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index ce672bf7b8..a40867d11d 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -15,11 +15,7 @@ #define _DEFAULT_SOURCE #include "dnode.h" -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "ulog.h" +#include "dmnInt.h" static struct { bool stop; @@ -136,38 +132,14 @@ void dmnWaitSignal() { } } -void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); -} - -void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); -} - int dmnRunDnode() { SDnodeEnvCfg envCfg = {0}; SDnodeObjCfg objCfg = {0}; - dnmInitEnvCfg(&envCfg); - dmnInitObjCfg(&objCfg); + if (dnmInitCfg(&envCfg, &objCfg, "", "", "") != 0) { + uInfo("Failed to start TDengine since load config error"); + return -1; + } if (dndInit(&envCfg) != 0) { uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c new file mode 100644 index 0000000000..c941f5959c --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static void dmnInitEnvCfg(SDnodeEnvCfg *pCfg) { + pCfg->sver = 30000000; // 3.0.0.0 + pCfg->numOfCores = tsNumOfCores; + pCfg->numOfCommitThreads = tsNumOfCommitThreads; + pCfg->enableTelem = 0; + tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); + tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pCfg->buildinfo, buildinfo, 64); + tstrncpy(pCfg->gitinfo, gitinfo, 48); +} + +static void dmnInitObjCfg(SDnodeObjCfg *pCfg) { + pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; + pCfg->statusInterval = tsStatusInterval; + pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; + pCfg->ratioOfQueryCores = tsRatioOfQueryCores; + pCfg->maxShellConns = tsMaxShellConns; + pCfg->shellActivityTimer = tsShellActivityTimer; + pCfg->serverPort = tsServerPort; + tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); + tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); + tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); + tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); +} + +int32_t dnmInitCfg(SDnodeEnvCfg *pEnvCfg, SDnodeObjCfg *pObjCfg, const char *configFile, const char *envFile, + const char *apolloUrl) { + dmnInitEnvCfg(pEnvCfg); + dmnInitObjCfg(pObjCfg); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 56cc58e658..821e920610 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -32,8 +32,8 @@ typedef struct SConfig { } SConfig; int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); -int32_t cfgLoadFromDotEnvFile(SConfig *pConfig, const char *filepath); -int32_t cfgLoadFromGlobalEnvVariable(SConfig *pConfig); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); #ifdef __cplusplus diff --git a/source/libs/config/src/cfgDotEnv.c b/source/libs/config/src/cfgEnvFile.c similarity index 85% rename from source/libs/config/src/cfgDotEnv.c rename to source/libs/config/src/cfgEnvFile.c index e65dddae7e..a12d1cd1d5 100644 --- a/source/libs/config/src/cfgDotEnv.c +++ b/source/libs/config/src/cfgEnvFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" -int32_t cfgLoadFromDotEnvFile(SConfig *pConfig, const char *filepath) { - uInfo("load from .env file %s", filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); return 0; } \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvVar.c b/source/libs/config/src/cfgEnvVar.c index caa5a88110..c3b3df4047 100644 --- a/source/libs/config/src/cfgEnvVar.c +++ b/source/libs/config/src/cfgEnvVar.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" -int32_t cfgLoadFromGlobalEnvVariable(SConfig *pConfig) { +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { uInfo("load from global env variables"); return 0; } \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 9650489655..98d3ad234c 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -37,10 +37,10 @@ int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr) { switch (cfgType) { case CFG_TYPE_CFG_FILE: return cfgLoadFromCfgFile(pConfig, sourceStr); - case CFG_TYPE_DOT_ENV: - return cfgLoadFromDotEnvFile(pConfig, sourceStr); + case CFG_TYPE_ENV_FILE: + return cfgLoadFromEnvFile(pConfig, sourceStr); case CFG_TYPE_ENV_VAR: - return cfgLoadFromGlobalEnvVariable(pConfig); + return cfgLoadFromEnvVar(pConfig); case CFG_TYPE_APOLLO_URL: return cfgLoadFromApollUrl(pConfig, sourceStr); default: @@ -206,13 +206,13 @@ const char *cfgStypeStr(ECfgSrcType type) { case CFG_TYPE_DEFAULT: return "default"; case CFG_TYPE_CFG_FILE: - return "cfg"; - case CFG_TYPE_DOT_ENV: - return ".env"; + return "cfg_file"; + case CFG_TYPE_ENV_FILE: + return "env_file"; case CFG_TYPE_ENV_VAR: - return "env"; + return "env_var"; case CFG_TYPE_APOLLO_URL: - return "apollo"; + return "apollo_url"; default: return "invalid"; } diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index d198d0bfe1..8e031e8a89 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -30,10 +30,10 @@ const char *CfgTest::pConfig; TEST_F(CfgTest, 02_Str) { EXPECT_STREQ(cfgStypeStr(CFG_TYPE_DEFAULT), "default"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_CFG_FILE), "cfg"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_DOT_ENV), ".env"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_VAR), "env"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_APOLLO_URL), "apollo"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_TYPE_APOLLO_URL), "apollo_url"); EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); From 18634533a50b79a93d731e1ebb8bcfebc402e44a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Feb 2022 13:36:26 +0800 Subject: [PATCH 13/36] config --- include/libs/config/config.h | 60 ++- include/util/taoserror.h | 2 + include/util/tconfig.h | 97 ----- source/client/src/clientEnv.c | 1 - source/common/src/tglobal.c | 15 +- source/common/src/tlocale.c | 1 - source/common/src/ttimezone.c | 1 - source/dnode/mgmt/daemon/inc/dmnInt.h | 1 - source/dnode/mgmt/daemon/src/dmnCfg.c | 170 ++++++++ source/dnode/mgmt/daemon/src/dmnLog.c | 0 source/dnode/mgmt/impl/test/sut/inc/sut.h | 1 - source/libs/config/inc/cfgInt.h | 6 +- source/libs/config/src/cfgCfgFile.c | 50 ++- source/libs/config/src/config.c | 347 +++++++++++---- source/libs/config/test/cfgTest.cpp | 137 ++---- source/util/src/tconfig.c | 507 ---------------------- source/util/src/terror.c | 2 + tools/shell/src/shellMain.c | 1 - 18 files changed, 556 insertions(+), 843 deletions(-) delete mode 100644 include/util/tconfig.h create mode 100644 source/dnode/mgmt/daemon/src/dmnLog.c delete mode 100644 source/util/src/tconfig.c diff --git a/include/libs/config/config.h b/include/libs/config/config.h index 3806d9f605..51fd83c49f 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -24,32 +24,26 @@ extern "C" { #endif typedef enum { - CFG_TYPE_DEFAULT, - CFG_TYPE_CFG_FILE, - CFG_TYPE_ENV_FILE, - CFG_TYPE_ENV_VAR, - CFG_TYPE_APOLLO_URL, - CFG_TYPE_CONSOLE_PARA + CFG_STYPE_DEFAULT, + CFG_STYPE_CFG_FILE, + CFG_STYPE_ENV_FILE, + CFG_STYPE_ENV_VAR, + CFG_STYPE_APOLLO_URL, + CFG_STYPE_ARG_LIST, + CFG_STYPE_API_OPTION } ECfgSrcType; typedef enum { CFG_DTYPE_NONE, CFG_DTYPE_BOOL, CFG_DTYPE_INT8, - CFG_DTYPE_UINT8, - CFG_DTYPE_INT16, CFG_DTYPE_UINT16, CFG_DTYPE_INT32, - CFG_DTYPE_UINT32, CFG_DTYPE_INT64, - CFG_DTYPE_UINT64, CFG_DTYPE_FLOAT, - CFG_DTYPE_DOUBLE, CFG_DTYPE_STRING, - CFG_DTYPE_FQDN, CFG_DTYPE_IPSTR, CFG_DTYPE_DIR, - CFG_DTYPE_FILE } ECfgDataType; typedef enum { @@ -63,28 +57,30 @@ typedef enum { } ECfgUnitType; typedef struct SConfigItem { - ECfgSrcType stype; + ECfgSrcType stype; ECfgUnitType utype; ECfgDataType dtype; char *name; union { bool boolVal; - uint8_t uint8Val; int8_t int8Val; uint16_t uint16Val; - int16_t int16Val; - uint32_t uint32Val; int32_t int32Val; - uint64_t uint64Val; int64_t int64Val; float floatVal; - double doubleVal; char *strVal; - char *fqdnVal; char *ipstrVal; char *dirVal; - char *fileVal; }; + union { + int64_t minIntVal; + double minFloatVal; + }; + union { + int64_t maxIntVal; + double maxFloatVal; + }; + } SConfigItem; typedef struct SConfig SConfig; @@ -99,21 +95,19 @@ void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter); SConfigItem *cfgGetItem(SConfig *pConfig, const char *name); int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); -int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype); -int32_t cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype); -int32_t cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype); -int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype); -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype); -int32_t cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype); -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype); -int32_t cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype); -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype); -int32_t cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype); +int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype); +int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype); +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype); +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype); +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval, + ECfgUnitType utype); int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 92028b0837..3f6db72c80 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -71,6 +71,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) diff --git a/include/util/tconfig.h b/include/util/tconfig.h deleted file mode 100644 index 7dee01247b..0000000000 --- a/include/util/tconfig.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_CONFIG_H -#define _TD_UTIL_CONFIG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_CFG_MAX_NUM 115 -#define TSDB_CFG_PRINT_LEN 23 -#define TSDB_CFG_OPTION_LEN 24 -#define TSDB_CFG_VALUE_LEN 41 - -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password - -#define MAX_FLOAT 100000 -#define MIN_FLOAT 0 - -enum { - TAOS_CFG_CSTATUS_NONE, // not configured - TAOS_CFG_CSTATUS_DEFAULT, // use system default value - TAOS_CFG_CSTATUS_FILE, // configured from file - TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function - TAOS_CFG_CSTATUS_ARG, // configured by program argument -}; - -enum { - TAOS_CFG_VTYPE_INT8, - TAOS_CFG_VTYPE_INT16, - TAOS_CFG_VTYPE_INT32, - TAOS_CFG_VTYPE_UINT16, - TAOS_CFG_VTYPE_FLOAT, - TAOS_CFG_VTYPE_STRING, - TAOS_CFG_VTYPE_IPSTR, - TAOS_CFG_VTYPE_DIRECTORY, - TAOS_CFG_VTYPE_DATA_DIRCTORY, - TAOS_CFG_VTYPE_DOUBLE, -}; - -enum { - TAOS_CFG_UTYPE_NONE, - TAOS_CFG_UTYPE_PERCENT, - TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_MB, - TAOS_CFG_UTYPE_BYTE, - TAOS_CFG_UTYPE_SECOND, - TAOS_CFG_UTYPE_MS -}; - -typedef struct { - char * option; - void * ptr; - float minValue; - float maxValue; - int8_t cfgType; - int8_t cfgStatus; - int8_t unitType; - int8_t valType; - int32_t ptrLength; -} SGlobalCfg; - -extern SGlobalCfg tsGlobalConfig[]; -extern int32_t tsGlobalConfigNum; -extern char * tsCfgStatusStr[]; - -void taosReadGlobalLogCfg(); -int32_t taosReadCfgFromFile(); -void taosPrintCfg(); -void taosDumpGlobalCfg(); - -void taosAddConfigOption(SGlobalCfg cfg); -SGlobalCfg *taosGetConfigOption(const char *option); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_CONFIG_H*/ diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa..1fa6446ec7 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -21,7 +21,6 @@ #include "scheduler.h" #include "tmsg.h" #include "tcache.h" -#include "tconfig.h" #include "tglobal.h" #include "tnote.h" #include "tref.h" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index afba1190ba..364820ba07 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -19,7 +19,6 @@ #include "taosdef.h" #include "taoserror.h" #include "tcompare.h" -#include "tconfig.h" #include "tep.h" #include "tglobal.h" #include "tlocale.h" @@ -305,7 +304,7 @@ static void taosCheckDataDirCfg() { static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); - +#if 0 SGlobalCfg cfg = {0}; // ip address @@ -1025,7 +1024,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; @@ -1035,8 +1034,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &dPrecision; cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.minValue = 100000; + cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); @@ -1064,11 +1063,15 @@ static void doInitGlobalConfig(void) { #else // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif + +#endif } void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckAndPrintCfg() { +#if 0 + SEp ep = {0}; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); @@ -1121,7 +1124,7 @@ int32_t taosCheckAndPrintCfg() { uInfo(" check global cfg completed"); uInfo("=================================="); taosPrintCfg(); - +#endif return 0; } diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c index a98a46b28a..1f2b9cd4b1 100644 --- a/source/common/src/tlocale.c +++ b/source/common/src/tlocale.c @@ -17,7 +17,6 @@ #include "os.h" #include "ulog.h" #include "tglobal.h" -#include "tconfig.h" #include "tutil.h" /** diff --git a/source/common/src/ttimezone.c b/source/common/src/ttimezone.c index c45e39c20d..0ee5702242 100644 --- a/source/common/src/ttimezone.c +++ b/source/common/src/ttimezone.c @@ -17,7 +17,6 @@ #include "os.h" #include "ulog.h" #include "tglobal.h" -#include "tconfig.h" #include "tutil.h" // TODO refactor to set the tz value through parameter diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h index 1bba29bbd2..3a28288bed 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -19,7 +19,6 @@ #include "config.h" #include "dnode.h" -#include "tconfig.h" #include "tglobal.h" #include "tnote.h" #include "ulog.h" diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index c941f5959c..1bd8476f94 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -42,6 +42,176 @@ static void dmnInitObjCfg(SDnodeObjCfg *pCfg) { tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); } +#if 0 +void taosReadGlobalLogCfg() { + FILE * fp; + char * line, *option, *value; + int olen, vlen; + char fileName[PATH_MAX] = {0}; + + taosExpandDir(configDir, configDir, PATH_MAX); + taosReadLogOption("logDir", tsLogDir); + + sprintf(fileName, "%s/taos.cfg", configDir); + fp = fopen(fileName, "r"); + if (fp == NULL) { + printf("\nconfig file:%s not found, all variables are set to default\n", fileName); + return; + } + + ssize_t _bytes = 0; + size_t len = 1024; + line = calloc(1, len); + + while (!feof(fp)) { + memset(line, 0, len); + + option = value = NULL; + olen = vlen = 0; + + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) + { + break; + } + + line[len - 1] = 0; + + paGetToken(line, &option, &olen); + if (olen == 0) continue; + option[olen] = 0; + + paGetToken(option + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + taosReadLogOption(option, value); + } + + tfree(line); + fclose(fp); +} + + +void taosPrintCfg() { + uInfo(" taos config & system info:"); + uInfo("=================================="); + + for (int i = 0; i < tsGlobalConfigNum; ++i) { + SGlobalCfg *cfg = tsGlobalConfig + i; + if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; + + int optionLen = (int)strlen(cfg->option); + int blankLen = TSDB_CFG_PRINT_LEN - optionLen; + blankLen = blankLen < 0 ? 0 : blankLen; + + char blank[TSDB_CFG_PRINT_LEN]; + memset(blank, ' ', TSDB_CFG_PRINT_LEN); + blank[blankLen] = 0; + + switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT16: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT32: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_UINT16: + uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_FLOAT: + uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_DOUBLE: + uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_STRING: + case TAOS_CFG_VTYPE_IPSTR: + case TAOS_CFG_VTYPE_DIRECTORY: + uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); + break; + default: + break; + } + } + + taosPrintOsInfo(); + uInfo("=================================="); +} + +#if 0 +static void taosDumpCfg(SGlobalCfg *cfg) { + int optionLen = (int)strlen(cfg->option); + int blankLen = TSDB_CFG_PRINT_LEN - optionLen; + blankLen = blankLen < 0 ? 0 : blankLen; + + char blank[TSDB_CFG_PRINT_LEN]; + memset(blank, ' ', TSDB_CFG_PRINT_LEN); + blank[blankLen] = 0; + + switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT32: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_UINT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_FLOAT: + printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_STRING: + case TAOS_CFG_VTYPE_IPSTR: + case TAOS_CFG_VTYPE_DIRECTORY: + printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); + break; + default: + break; + } +} + +void taosDumpGlobalCfg() { + printf("taos global config:\n"); + printf("==================================\n"); + for (int i = 0; i < tsGlobalConfigNum; ++i) { + SGlobalCfg *cfg = tsGlobalConfig + i; + if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; + if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; + + taosDumpCfg(cfg); + } + + printf("\ntaos local config:\n"); + printf("==================================\n"); + + for (int i = 0; i < tsGlobalConfigNum; ++i) { + SGlobalCfg *cfg = tsGlobalConfig + i; + if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; + + taosDumpCfg(cfg); + } +} + +#endif + +#endif + +static int32_t dmnInitLog() { + +} + int32_t dnmInitCfg(SDnodeEnvCfg *pEnvCfg, SDnodeObjCfg *pObjCfg, const char *configFile, const char *envFile, const char *apolloUrl) { dmnInitEnvCfg(pEnvCfg); diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 23913b0531..c1f52844a4 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -21,7 +21,6 @@ #include "dnode.h" #include "tmsg.h" -#include "tconfig.h" #include "tdataformat.h" #include "tglobal.h" #include "tnote.h" diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 821e920610..6f78b97435 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -27,8 +27,8 @@ extern "C" { #endif typedef struct SConfig { - ECfgSrcType stype; - SHashObj *hash; + ECfgSrcType stype; + SHashObj *hash; } SConfig; int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); @@ -36,6 +36,8 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); + #ifdef __cplusplus } #endif diff --git a/source/libs/config/src/cfgCfgFile.c b/source/libs/config/src/cfgCfgFile.c index 9cf36997eb..4eb835be7f 100644 --- a/source/libs/config/src/cfgCfgFile.c +++ b/source/libs/config/src/cfgCfgFile.c @@ -17,6 +17,54 @@ #include "cfgInt.h" int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { - uInfo("load from .cfg file %s", filepath); + char *line, *name, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + size_t len = 1024; + + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + line = malloc(len); + + while (!feof(fp)) { + memset(line, 0, len); + + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) { + break; + } + + line[len - 1] = 0; + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + // taosReadConfigOption(name, value, value2, value3); + } + + fclose(fp); + tfree(line); + + uInfo("load from cfg file %s success", filepath); return 0; } \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 98d3ad234c..74c03bfb3d 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -35,13 +35,13 @@ SConfig *cfgInit() { int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr) { switch (cfgType) { - case CFG_TYPE_CFG_FILE: + case CFG_STYPE_CFG_FILE: return cfgLoadFromCfgFile(pConfig, sourceStr); - case CFG_TYPE_ENV_FILE: + case CFG_STYPE_ENV_FILE: return cfgLoadFromEnvFile(pConfig, sourceStr); - case CFG_TYPE_ENV_VAR: + case CFG_STYPE_ENV_VAR: return cfgLoadFromEnvVar(pConfig); - case CFG_TYPE_APOLLO_URL: + case CFG_STYPE_APOLLO_URL: return cfgLoadFromApollUrl(pConfig, sourceStr); default: return -1; @@ -64,10 +64,180 @@ SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashI void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashCancelIterate(pConfig->hash, pIter); } -SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { taosHashGet(pConfig->hash, name, strlen(name) + 1); } + +int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + pItem->boolVal = tmp; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetInt8(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int8_t ival = (int8_t)atoi(value); + if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, + cfgStypeStr(pItem->stype), pItem->int8Val); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + pItem->int8Val = ival; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetUInt16(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + uint16_t ival = (uint16_t)atoi(value); + if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, + cfgStypeStr(pItem->stype), pItem->uint16Val); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + pItem->uint16Val = ival; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int32_t ival = (int32_t)atoi(value); + if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, + cfgStypeStr(pItem->stype), pItem->int32Val); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + pItem->int32Val = ival; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int64_t ival = (int64_t)atoi(value); + if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, + cfgStypeStr(pItem->stype), pItem->int64Val); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + pItem->int64Val = ival; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + float fval = (float)atof(value); + if (fval < pItem->minFloatVal || fval > pItem->maxFloatVal) { + uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->minFloatVal, pItem->maxFloatVal, + cfgStypeStr(pItem->stype), pItem->floatVal); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + pItem->floatVal = fval; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + return -1; + } + free(pItem->strVal); + pItem->strVal = tmp; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + return -1; + } + free(pItem->strVal); + pItem->strVal = tmp; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + return -1; + } + free(pItem->strVal); + pItem->strVal = tmp; + pItem->stype = stype; + return 0; +} + +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pConfig, name); + if (pItem == NULL) { + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + return cfgSetBool(pItem, value, stype); + case CFG_DTYPE_INT8: + return cfgSetInt8(pItem, value, stype); + case CFG_DTYPE_UINT16: + return cfgSetUInt16(pItem, value, stype); + case CFG_DTYPE_INT32: + return cfgSetInt32(pItem, value, stype); + case CFG_DTYPE_INT64: + return cfgSetInt64(pItem, value, stype); + case CFG_DTYPE_FLOAT: + return cfgSetFloat(pItem, value, stype); + case CFG_DTYPE_STRING: + return cfgSetString(pItem, value, stype); + case CFG_DTYPE_IPSTR: + return cfgSetIpStr(pItem, value, stype); + case CFG_DTYPE_DIR: + return cfgSetFqdn(pItem, value, stype); + case CFG_DTYPE_NONE: + default: + break; + } + + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { + char lowcaseName[128] = 0; + memcpy(lowcaseName, name, 127); + + SConfigItem *pItem = taosHashGet(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1); + if (pItem == NULL) { + terrno = TSDB_CODE_CFG_NOT_FOUND; + } + + return pItem; +} static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name, ECfgUnitType utype) { - pItem->stype = CFG_TYPE_DEFAULT; + pItem->stype = CFG_STYPE_DEFAULT; pItem->utype = utype; pItem->name = strdup(name); if (pItem->name == NULL) { @@ -75,18 +245,15 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name return -1; } - if (taosHashPut(pConfig->hash, name, strlen(name) + 1, pItem, sizeof(SConfigItem)) != 0) { + char lowcaseName[128] = 0; + memcpy(lowcaseName, name, 127); + if (taosHashPut(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { if (pItem->dtype == CFG_DTYPE_STRING) { free(pItem->strVal); - } else if (pItem->dtype == CFG_DTYPE_FQDN) { - free(pItem->fqdnVal); } else if (pItem->dtype == CFG_DTYPE_IPSTR) { free(pItem->ipstrVal); } else if (pItem->dtype == CFG_DTYPE_DIR) { free(pItem->dirVal); - } else if (pItem->dtype == CFG_DTYPE_FILE) { - free(pItem->fileVal); - } else { } free(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -101,57 +268,67 @@ int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnit return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_INT8, .int8Val = defaultVal}; +int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT8, .int8Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddUInt8(SConfig *pConfig, const char *name, uint8_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_UINT8, .uint8Val = defaultVal}; +int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_UINT16, .uint16Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddInt16(SConfig *pConfig, const char *name, int16_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_INT16, .int16Val = defaultVal}; +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .int32Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_UINT16, .uint16Val = defaultVal}; +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, + ECfgUnitType utype) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .int64Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .int32Val = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); -} +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval, + ECfgUnitType utype) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } -int32_t cfgAddUInt32(SConfig *pConfig, const char *name, uint32_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_UINT32, .uint32Val = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .int64Val = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddUInt64(SConfig *pConfig, const char *name, uint64_t defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_UINT64, .uint64Val = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .floatVal = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddDouble(SConfig *pConfig, const char *name, double defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_DOUBLE, .doubleVal = defaultVal}; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .floatVal = defaultVal, .minFloatVal = minval, .maxFloatVal = maxval}; return cfgAddItem(pConfig, &item, name, utype); } int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + if (defaultVal == NULL) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + SConfigItem item = {.dtype = CFG_DTYPE_STRING}; item.strVal = strdup(defaultVal); if (item.strVal == NULL) { @@ -161,17 +338,22 @@ int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddFqdn(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_FQDN}; - item.fqdnVal = strdup(defaultVal); - if (item.fqdnVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +static int32_t cfgCheckIpStr(const char *ip) { + uint32_t value = taosInetAddr(ip); + if (value == INADDR_NONE) { + uError("ip:%s is not a valid ip address", ip); return -1; } - return cfgAddItem(pConfig, &item, name, utype); + + return 0; } int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + if (cfgCheckIpStr(defaultVal) != 0) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; item.ipstrVal = strdup(defaultVal); if (item.ipstrVal == NULL) { @@ -181,21 +363,38 @@ int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, return cfgAddItem(pConfig, &item, name, utype); } -int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR}; - item.dirVal = strdup(defaultVal); - if (item.dirVal == NULL) { +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + tfree(pItem->dirVal); + pItem->dirVal = strdup(fullDir); + if (pItem->dirVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - return cfgAddItem(pConfig, &item, name, utype); + + return 0; } -int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_FILE}; - item.fileVal = strdup(defaultVal); - if (item.fileVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR}; + if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } return cfgAddItem(pConfig, &item, name, utype); @@ -203,15 +402,15 @@ int32_t cfgAddFile(SConfig *pConfig, const char *name, const char *defaultVal, E const char *cfgStypeStr(ECfgSrcType type) { switch (type) { - case CFG_TYPE_DEFAULT: + case CFG_STYPE_DEFAULT: return "default"; - case CFG_TYPE_CFG_FILE: + case CFG_STYPE_CFG_FILE: return "cfg_file"; - case CFG_TYPE_ENV_FILE: + case CFG_STYPE_ENV_FILE: return "env_file"; - case CFG_TYPE_ENV_VAR: + case CFG_STYPE_ENV_VAR: return "env_var"; - case CFG_TYPE_APOLLO_URL: + case CFG_STYPE_APOLLO_URL: return "apollo_url"; default: return "invalid"; @@ -226,34 +425,20 @@ const char *cfgDtypeStr(ECfgDataType type) { return "bool"; case CFG_DTYPE_INT8: return "int8"; - case CFG_DTYPE_UINT8: - return "uint8"; - case CFG_DTYPE_INT16: - return "int16"; case CFG_DTYPE_UINT16: return "uint16"; case CFG_DTYPE_INT32: return "int32"; - case CFG_DTYPE_UINT32: - return "uint32"; case CFG_DTYPE_INT64: return "int64"; - case CFG_DTYPE_UINT64: - return "uint64"; case CFG_DTYPE_FLOAT: return "float"; - case CFG_DTYPE_DOUBLE: - return "double"; case CFG_DTYPE_STRING: return "string"; - case CFG_DTYPE_FQDN: - return "fqdn"; case CFG_DTYPE_IPSTR: return "ipstr"; case CFG_DTYPE_DIR: return "dir"; - case CFG_DTYPE_FILE: - return "file"; default: return "invalid"; } diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index 8e031e8a89..b14c0bb201 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -29,30 +29,24 @@ class CfgTest : public ::testing::Test { const char *CfgTest::pConfig; TEST_F(CfgTest, 02_Str) { - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_DEFAULT), "default"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_CFG_FILE), "cfg_file"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_FILE), "env_file"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_ENV_VAR), "env_var"); - EXPECT_STREQ(cfgStypeStr(CFG_TYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT8), "int8"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT8), "uint8"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT16), "int16"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT16), "uint16"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT32), "uint32"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT64), "uint64"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DOUBLE), "double"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FQDN), "fqdn"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FILE), "file"); EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_NONE), ""); @@ -71,22 +65,15 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgAddInt8(pConfig, "test_int8", 1, CFG_UTYPE_GB), 0); - EXPECT_EQ(cfgAddUInt8(pConfig, "test_uint8", 2, CFG_UTYPE_MB), 0); - EXPECT_EQ(cfgAddInt16(pConfig, "test_int16", 3, CFG_UTYPE_BYTE), 0); - EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 4, CFG_UTYPE_SECOND), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 5, CFG_UTYPE_MS), 0); - EXPECT_EQ(cfgAddUInt32(pConfig, "test_uint32", 6, CFG_UTYPE_PERCENT), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 7, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddUInt64(pConfig, "test_uint64", 8, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 9, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddDouble(pConfig, "test_double", 10, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "11", CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddFqdn(pConfig, "test_fqdn", "localhost", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 2, CFG_UTYPE_SECOND), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 3, CFG_UTYPE_MS), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 4, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 5, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "6", CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddFile(pConfig, "test_file", "/tmp/file1", CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgGetSize(pConfig), 16); + EXPECT_EQ(cfgGetSize(pConfig), 9); int32_t size = 0; SConfigItem *pItem = cfgIterate(pConfig, NULL); @@ -98,48 +85,27 @@ TEST_F(CfgTest, 02_Basic) { case CFG_DTYPE_INT8: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint8Val); break; - case CFG_DTYPE_UINT8: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int8Val); - break; - case CFG_DTYPE_INT16: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint16Val); - break; case CFG_DTYPE_UINT16: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int16Val); break; case CFG_DTYPE_INT32: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint32Val); break; - case CFG_DTYPE_UINT32: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int32Val); - break; case CFG_DTYPE_INT64: printf("index:%d, cfg:%s value:%" PRIu64 "\n", size, pItem->name, pItem->uint64Val); break; - case CFG_DTYPE_UINT64: - printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->int64Val); - break; case CFG_DTYPE_FLOAT: printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->floatVal); break; - case CFG_DTYPE_DOUBLE: - printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->doubleVal); - break; case CFG_DTYPE_STRING: printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); break; - case CFG_DTYPE_FQDN: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->fqdnVal); - break; case CFG_DTYPE_IPSTR: printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->ipstrVal); break; case CFG_DTYPE_DIR: printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->dirVal); break; - case CFG_DTYPE_FILE: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->fileVal); - break; default: printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); break; @@ -149,119 +115,70 @@ TEST_F(CfgTest, 02_Basic) { } cfgCancelIterate(pConfig, pItem); - EXPECT_EQ(cfgGetSize(pConfig), 16); + EXPECT_EQ(cfgGetSize(pConfig), 9); pItem = cfgGetItem(pConfig, "test_bool"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); EXPECT_STREQ(pItem->name, "test_bool"); EXPECT_EQ(pItem->boolVal, 0); pItem = cfgGetItem(pConfig, "test_int8"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_GB); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT8); EXPECT_STREQ(pItem->name, "test_int8"); EXPECT_EQ(pItem->int8Val, 1); - pItem = cfgGetItem(pConfig, "test_uint8"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_MB); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT8); - EXPECT_STREQ(pItem->name, "test_uint8"); - EXPECT_EQ(pItem->uint8Val, 2); - - pItem = cfgGetItem(pConfig, "test_int16"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_BYTE); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT16); - EXPECT_STREQ(pItem->name, "test_int16"); - EXPECT_EQ(pItem->int16Val, 3); - pItem = cfgGetItem(pConfig, "test_uint16"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT16); EXPECT_STREQ(pItem->name, "test_uint16"); - EXPECT_EQ(pItem->uint16Val, 4); + EXPECT_EQ(pItem->uint16Val, 2); pItem = cfgGetItem(pConfig, "test_int32"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_MS); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); EXPECT_STREQ(pItem->name, "test_int32"); - EXPECT_EQ(pItem->int32Val, 5); - - pItem = cfgGetItem(pConfig, "test_uint32"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_PERCENT); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT32); - EXPECT_STREQ(pItem->name, "test_uint32"); - EXPECT_EQ(pItem->uint32Val, 6); + EXPECT_EQ(pItem->int32Val, 3); pItem = cfgGetItem(pConfig, "test_int64"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); EXPECT_STREQ(pItem->name, "test_int64"); - EXPECT_EQ(pItem->int64Val, 7); - - pItem = cfgGetItem(pConfig, "test_uint64"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT64); - EXPECT_STREQ(pItem->name, "test_uint64"); - EXPECT_EQ(pItem->uint64Val, 8); + EXPECT_EQ(pItem->int64Val, 4); pItem = cfgGetItem(pConfig, "test_float"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); EXPECT_STREQ(pItem->name, "test_float"); - EXPECT_EQ(pItem->floatVal, 9); - - pItem = cfgGetItem(pConfig, "test_double"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_DOUBLE); - EXPECT_STREQ(pItem->name, "test_double"); - EXPECT_EQ(pItem->doubleVal, 10); + EXPECT_EQ(pItem->floatVal, 5); pItem = cfgGetItem(pConfig, "test_string"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); EXPECT_STREQ(pItem->name, "test_string"); - EXPECT_STREQ(pItem->strVal, "11"); - - pItem = cfgGetItem(pConfig, "test_fqdn"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_FQDN); - EXPECT_STREQ(pItem->name, "test_fqdn"); - EXPECT_STREQ(pItem->strVal, "localhost"); + EXPECT_STREQ(pItem->strVal, "6"); pItem = cfgGetItem(pConfig, "test_ipstr"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); EXPECT_STREQ(pItem->name, "test_ipstr"); EXPECT_STREQ(pItem->ipstrVal, "192.168.0.1"); pItem = cfgGetItem(pConfig, "test_dir"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); EXPECT_STREQ(pItem->name, "test_dir"); EXPECT_STREQ(pItem->dirVal, "/tmp"); - pItem = cfgGetItem(pConfig, "test_file"); - EXPECT_EQ(pItem->stype, CFG_TYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_FILE); - EXPECT_STREQ(pItem->name, "test_file"); - EXPECT_STREQ(pItem->fileVal, "/tmp/file1"); - cfgCleanup(pConfig); } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c deleted file mode 100644 index 469da11d93..0000000000 --- a/source/util/src/tconfig.c +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tutil.h" -#include "ulog.h" - -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; -int32_t tsGlobalConfigNum = 0; - -static char *tsGlobalUnit[] = { - " ", - "(%)", - "(GB)", - "(Mb)", - "(byte)", - "(s)", - "(ms)" -}; - -char *tsCfgStatusStr[] = { - "none", - "system default", - "config file", - "taos_options", - "program argument list" -}; - -static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { - float value = (float)atof(input_value); - float *option = (float *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { - double value = atof(input_value); - double *option = (double *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int32_t *option = (int32_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int16_t *option = (int16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - uint16_t *option = (uint16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (uint16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int8_t *option = (int8_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int8_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int)strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, - input_value, cfg->ptrLength, option); - return false; - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - taosExpandDir(input_value, option, cfg->ptrLength); - taosRealPath(option, cfg->ptrLength); - - if (taosMkDir(option) != 0) { - uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, - strerror(errno)); - return false; - } - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } - - return true; -} - -static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = taosInetAddr(input_value); - char * option = (char *)cfg->ptr; - if (value == INADDR_NONE) { - uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", - cfg->option, input_value, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int) strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadLogOption(char *option, char *value) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - // if (strcasecmp(cfg->option, "debugFlag") == 0) { - // taosSetAllDebugFlag(); - // } - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - default: - break; - } - break; - } -} - -SGlobalCfg *taosGetConfigOption(const char *option) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (strcasecmp(cfg->option, option) != 0) continue; - return cfg; - } - return NULL; -} - -static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - taosReadInt8Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT16: - taosReadInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - break; - case TAOS_CFG_VTYPE_UINT16: - taosReadUInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_FLOAT: - taosReadFloatConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DOUBLE: - taosReadDoubleConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_STRING: - taosReadStringConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_IPSTR: - taosReadIpStrConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DATA_DIRCTORY: - if (taosReadDirectoryConfig(cfg, value)) { - // taosReadDataDirCfg(value, value2, value3); - } - break; - default: - uError("config option:%s, input value:%s, can't be recognized", option, value); - break; - } - break; - } -} - -void taosAddConfigOption(SGlobalCfg cfg) { - tsGlobalConfig[tsGlobalConfigNum++] = cfg; -} - -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; - - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - taosReadLogOption(option, value); - } - - tfree(line); - fclose(fp); -} - -int32_t taosReadCfgFromFile() { - char * line, *option, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - char fileName[PATH_MAX] = {0}; - - sprintf(fileName, "%s/taos.cfg", configDir); - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - fp = fopen(configDir, "r"); - if (fp == NULL) { - return -1; - } - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = value2 = value3 = NULL; - olen = vlen = vlen2 = vlen3 = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; - } - - taosReadConfigOption(option, value, value2, value3); - } - - fclose(fp); - - tfree(line); - - // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - // taosSetAllDebugFlag(); - // } - - return 0; -} - -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } - } - - taosPrintOsInfo(); - uInfo("=================================="); -} - -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } -} - -void taosDumpGlobalCfg() { - printf("taos global config:\n"); - printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; - - taosDumpCfg(cfg); - } - - printf("\ntaos local config:\n"); - printf("==================================\n"); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; - - taosDumpCfg(cfg); - } -} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2b53a769ff..e6bef5f8b9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -81,6 +81,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 607dc79257..8a1763c4fc 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -15,7 +15,6 @@ #include "os.h" #include "shell.h" -#include "tconfig.h" #include "tglobal.h" pthread_t pid; From 392d3df98fc347b2fc62dbb9d77420fb0d2841cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Feb 2022 13:59:51 +0800 Subject: [PATCH 14/36] config --- include/libs/config/config.h | 6 ++-- include/os/osDir.h | 2 +- include/os/osSocket.h | 2 +- source/libs/config/inc/cfgInt.h | 1 + source/libs/config/src/config.c | 54 +++++++++++++++++++---------- source/libs/config/test/cfgTest.cpp | 31 +++++++++-------- source/os/src/osDir.c | 2 +- source/os/src/osSocket.c | 4 +-- source/util/src/tlog.c | 2 +- 9 files changed, 62 insertions(+), 42 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index 51fd83c49f..d603595cbf 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -23,6 +23,8 @@ extern "C" { #endif +#define CFG_NAME_MAX_LEN 128 + typedef enum { CFG_STYPE_DEFAULT, CFG_STYPE_CFG_FILE, @@ -48,12 +50,12 @@ typedef enum { typedef enum { CFG_UTYPE_NONE, - CFG_UTYPE_PERCENT, CFG_UTYPE_GB, CFG_UTYPE_MB, CFG_UTYPE_BYTE, CFG_UTYPE_SECOND, - CFG_UTYPE_MS + CFG_UTYPE_MS, + CFG_UTYPE_PERCENT } ECfgUnitType; typedef struct SConfigItem { diff --git a/include/os/osDir.h b/include/os/osDir.h index 35c294af39..01ec24f235 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -24,7 +24,7 @@ void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen); +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus diff --git a/include/os/osSocket.h b/include/os/osSocket.h index d76b286d65..af99e4b474 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 6f78b97435..0b62018222 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -20,6 +20,7 @@ #include "config.h" #include "taoserror.h" #include "thash.h" +#include "tutil.h" #include "ulog.h" #ifdef __cplusplus diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 74c03bfb3d..41d59d20a5 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -64,8 +64,7 @@ SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashI void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashCancelIterate(pConfig->hash, pIter); } - -int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { bool tmp = false; if (strcasecmp(value, "true") == 0) { tmp = true; @@ -73,12 +72,13 @@ int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { if (atoi(value) > 0) { tmp = true; } + pItem->boolVal = tmp; pItem->stype = stype; return 0; } -int32_t cfgSetInt8(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetInt8(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int8_t ival = (int8_t)atoi(value); if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", @@ -87,12 +87,13 @@ int32_t cfgSetInt8(SConfigItem *pItem, const char *value, ECfgSrcType stype) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } + pItem->int8Val = ival; pItem->stype = stype; return 0; } -int32_t cfgSetUInt16(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetUInt16(SConfigItem *pItem, const char *value, ECfgSrcType stype) { uint16_t ival = (uint16_t)atoi(value); if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", @@ -101,12 +102,13 @@ int32_t cfgSetUInt16(SConfigItem *pItem, const char *value, ECfgSrcType stype) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } + pItem->uint16Val = ival; pItem->stype = stype; return 0; } -int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int32_t ival = (int32_t)atoi(value); if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", @@ -115,26 +117,29 @@ int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } + pItem->int32Val = ival; pItem->stype = stype; return 0; } -int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int64_t ival = (int64_t)atoi(value); if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { - uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 + "], use last src:%s value:%" PRId64, pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, cfgStypeStr(pItem->stype), pItem->int64Val); terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } + pItem->int64Val = ival; pItem->stype = stype; return 0; } -int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { float fval = (float)atof(value); if (fval < pItem->minFloatVal || fval > pItem->maxFloatVal) { uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, @@ -143,47 +148,51 @@ int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } + pItem->floatVal = fval; pItem->stype = stype; return 0; } -int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); return -1; } + free(pItem->strVal); pItem->strVal = tmp; pItem->stype = stype; return 0; } -int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); return -1; } + free(pItem->strVal); pItem->strVal = tmp; pItem->stype = stype; return 0; } -int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { +static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), terrstr(), cfgStypeStr(pItem->stype), pItem->floatVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); return -1; } + free(pItem->strVal); pItem->strVal = tmp; pItem->stype = stype; @@ -214,7 +223,7 @@ int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSr case CFG_DTYPE_IPSTR: return cfgSetIpStr(pItem, value, stype); case CFG_DTYPE_DIR: - return cfgSetFqdn(pItem, value, stype); + return cfgSetDir(pItem, value, stype); case CFG_DTYPE_NONE: default: break; @@ -225,8 +234,9 @@ int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSr } SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { - char lowcaseName[128] = 0; - memcpy(lowcaseName, name, 127); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); SConfigItem *pItem = taosHashGet(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1); if (pItem == NULL) { @@ -245,8 +255,10 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name return -1; } - char lowcaseName[128] = 0; - memcpy(lowcaseName, name, 127); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + if (taosHashPut(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { if (pItem->dtype == CFG_DTYPE_STRING) { free(pItem->strVal); @@ -412,6 +424,10 @@ const char *cfgStypeStr(ECfgSrcType type) { return "env_var"; case CFG_STYPE_APOLLO_URL: return "apollo_url"; + case CFG_STYPE_ARG_LIST: + return "arg_list"; + case CFG_STYPE_API_OPTION: + return "api_option"; default: return "invalid"; } diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index b14c0bb201..8610fa3643 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -34,12 +34,13 @@ TEST_F(CfgTest, 02_Str) { EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_API_OPTION), "api_option"); EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT8), "int8"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT16), "uint16"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); @@ -50,12 +51,12 @@ TEST_F(CfgTest, 02_Str) { EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_NONE), ""); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_PERCENT), "(%)"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_GB), "(GB)"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MB), "(Mb)"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_BYTE), "(byte)"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_SECOND), "(s)"); EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MS), "(ms)"); + EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_PERCENT), "(%)"); EXPECT_STREQ(cfgUtypeStr(ECfgUnitType(1024)), "invalid"); } @@ -64,11 +65,11 @@ TEST_F(CfgTest, 02_Basic) { ASSERT_NE(pConfig, nullptr); EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddInt8(pConfig, "test_int8", 1, CFG_UTYPE_GB), 0); - EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 2, CFG_UTYPE_SECOND), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 3, CFG_UTYPE_MS), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 4, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 5, CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddInt8(pConfig, "test_int8", 1, 0, 16, CFG_UTYPE_GB), 0); + EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 2, 0, 16, CFG_UTYPE_MB), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 3, 0, 16, CFG_UTYPE_BYTE), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 4, 0, 16, CFG_UTYPE_SECOND), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 5, 0, 16, CFG_UTYPE_MS), 0); EXPECT_EQ(cfgAddString(pConfig, "test_string", "6", CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE), 0); EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE), 0); @@ -83,16 +84,16 @@ TEST_F(CfgTest, 02_Basic) { printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->boolVal); break; case CFG_DTYPE_INT8: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint8Val); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int8Val); break; case CFG_DTYPE_UINT16: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int16Val); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint16Val); break; case CFG_DTYPE_INT32: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint32Val); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int32Val); break; case CFG_DTYPE_INT64: - printf("index:%d, cfg:%s value:%" PRIu64 "\n", size, pItem->name, pItem->uint64Val); + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->int64Val); break; case CFG_DTYPE_FLOAT: printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->floatVal); @@ -133,28 +134,28 @@ TEST_F(CfgTest, 02_Basic) { pItem = cfgGetItem(pConfig, "test_uint16"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); + EXPECT_EQ(pItem->utype, CFG_UTYPE_MB); EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT16); EXPECT_STREQ(pItem->name, "test_uint16"); EXPECT_EQ(pItem->uint16Val, 2); pItem = cfgGetItem(pConfig, "test_int32"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_MS); + EXPECT_EQ(pItem->utype, CFG_UTYPE_BYTE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); EXPECT_STREQ(pItem->name, "test_int32"); EXPECT_EQ(pItem->int32Val, 3); pItem = cfgGetItem(pConfig, "test_int64"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); EXPECT_STREQ(pItem->name, "test_int64"); EXPECT_EQ(pItem->int64Val, 4); pItem = cfgGetItem(pConfig, "test_float"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); + EXPECT_EQ(pItem->utype, CFG_UTYPE_MS); EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); EXPECT_STREQ(pItem->name, "test_float"); EXPECT_EQ(pItem->floatVal, 5); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8999646f6a..8fee6a6f7d 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { rmdir(dirname); } -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) { +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; if (0 != wordexp(dirname, &full_path, 0)) { //printf("failed to expand path:%s since %s", dirname, strerror(errno)); diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e12f9493b3..8afb4ae8b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -150,7 +150,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) -uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } +uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } @@ -240,7 +240,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #ifdef _MSC_VER //#if _MSC_VER >= 1900 -uint32_t taosInetAddr(char *ipAddr) { +uint32_t taosInetAddr(const char *ipAddr) { uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 12aa77214f..2579082022 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -217,7 +217,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintCfg(); + // taosPrintCfg(); taosKeepOldLog(keepName); return NULL; From 22cb2ada73c759e78991f3940a399eeb91191627 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Feb 2022 14:22:56 +0800 Subject: [PATCH 15/36] config --- include/libs/config/config.h | 9 ++-- source/common/src/tglobal.c | 2 + source/libs/config/src/config.c | 81 +++++++++++++++++++++++++---- source/libs/config/test/cfgTest.cpp | 8 +-- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index d603595cbf..cf8daae72b 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -46,6 +46,9 @@ typedef enum { CFG_DTYPE_STRING, CFG_DTYPE_IPSTR, CFG_DTYPE_DIR, + CFG_DTYPE_LOCALE, + CFG_DTYPE_CHARSET, + CFG_DTYPE_TIMEZONE } ECfgDataType; typedef enum { @@ -71,8 +74,6 @@ typedef struct SConfigItem { int64_t int64Val; float floatVal; char *strVal; - char *ipstrVal; - char *dirVal; }; union { int64_t minIntVal; @@ -82,7 +83,6 @@ typedef struct SConfigItem { int64_t maxIntVal; double maxFloatVal; }; - } SConfigItem; typedef struct SConfig SConfig; @@ -110,6 +110,9 @@ int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 364820ba07..398ded034c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -196,6 +196,7 @@ void taosSetAllDebugFlag() { } int32_t taosCfgDynamicOptions(char *msg) { + #if 0 char *option, *value; int32_t olen, vlen; int32_t vint = 0; @@ -265,6 +266,7 @@ int32_t taosCfgDynamicOptions(char *msg) { } } +#endif return false; } diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 41d59d20a5..a40783eaef 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -262,10 +262,6 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name if (taosHashPut(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { if (pItem->dtype == CFG_DTYPE_STRING) { free(pItem->strVal); - } else if (pItem->dtype == CFG_DTYPE_IPSTR) { - free(pItem->ipstrVal); - } else if (pItem->dtype == CFG_DTYPE_DIR) { - free(pItem->dirVal); } free(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -367,8 +363,8 @@ int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, } SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; - item.ipstrVal = strdup(defaultVal); - if (item.ipstrVal == NULL) { + item.strVal = strdup(defaultVal); + if (item.strVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -394,9 +390,9 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - tfree(pItem->dirVal); - pItem->dirVal = strdup(fullDir); - if (pItem->dirVal == NULL) { + tfree(pItem->strVal); + pItem->strVal = strdup(fullDir); + if (pItem->strVal == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -409,6 +405,67 @@ int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, EC if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } + + return cfgAddItem(pConfig, &item, name, utype); +} + +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + tfree(pItem->strVal); + pItem->strVal = strdup(locale); + if (pItem->strVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; + if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pConfig, &item, name, utype); +} + +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + tfree(pItem->strVal); + pItem->strVal = strdup(charset); + if (pItem->strVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; + if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pConfig, &item, name, utype); +} + +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + tfree(pItem->strVal); + pItem->strVal = strdup(timezone); + if (pItem->strVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; + if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { + return -1; + } + return cfgAddItem(pConfig, &item, name, utype); } @@ -455,6 +512,12 @@ const char *cfgDtypeStr(ECfgDataType type) { return "ipstr"; case CFG_DTYPE_DIR: return "dir"; + case CFG_DTYPE_LOCALE: + return "locale"; + case CFG_DTYPE_CHARSET: + return "charset"; + case CFG_DTYPE_TIMEZONE: + return "timezone"; default: return "invalid"; } diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index 8610fa3643..62ed5764ac 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -102,10 +102,10 @@ TEST_F(CfgTest, 02_Basic) { printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); break; case CFG_DTYPE_IPSTR: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->ipstrVal); + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); break; case CFG_DTYPE_DIR: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->dirVal); + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); break; default: printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); @@ -172,14 +172,14 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); EXPECT_STREQ(pItem->name, "test_ipstr"); - EXPECT_STREQ(pItem->ipstrVal, "192.168.0.1"); + EXPECT_STREQ(pItem->strVal, "192.168.0.1"); pItem = cfgGetItem(pConfig, "test_dir"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); EXPECT_STREQ(pItem->name, "test_dir"); - EXPECT_STREQ(pItem->dirVal, "/tmp"); + EXPECT_STREQ(pItem->strVal, "/tmp"); cfgCleanup(pConfig); } From 06a7ba7b80734d987aba45bb027f6c82117996f7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Feb 2022 19:49:11 +0800 Subject: [PATCH 16/36] config integrate with taosd --- include/dnode/mgmt/dnode.h | 20 +- include/dnode/mnode/mnode.h | 3 +- include/libs/config/config.h | 61 +- include/util/tlog.h | 3 + source/client/src/clientEnv.c | 8 +- source/common/src/tglobal.c | 340 ------------ source/dnode/mgmt/daemon/inc/dmnInt.h | 16 +- source/dnode/mgmt/daemon/src/daemon.c | 131 ++--- source/dnode/mgmt/daemon/src/dmnCfg.c | 318 +++++------ source/dnode/mgmt/daemon/src/dmnLog.c | 67 +++ source/dnode/mgmt/daemon/src/dmnUtil.c | 35 ++ source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/libs/config/CMakeLists.txt | 2 +- source/libs/config/inc/cfgInt.h | 1 + source/libs/config/src/config.c | 521 ++++++++++-------- source/libs/config/test/cfgTest.cpp | 94 +--- .../src/ttimezone.c => os/src/osTimeZone.c} | 9 +- source/util/src/tlog.c | 9 + 18 files changed, 676 insertions(+), 964 deletions(-) create mode 100644 source/dnode/mgmt/daemon/src/dmnUtil.c rename source/{common/src/ttimezone.c => os/src/osTimeZone.c} (95%) diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e64d358de4..99fe2457a8 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -27,15 +27,16 @@ typedef struct SDnode SDnode; /* ------------------------ Environment ------------------ */ typedef struct { - int32_t sver; - int32_t numOfCores; - int16_t numOfCommitThreads; - int8_t enableTelem; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; + int32_t sver; + int32_t numOfCores; + uint16_t numOfCommitThreads; + bool enableTelem; + bool printAuth; + char timezone[TSDB_TIMEZONE_LEN]; + char locale[TSDB_LOCALE_LEN]; + char charset[TSDB_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; } SDnodeEnvCfg; /** @@ -65,6 +66,7 @@ typedef struct { char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; + char secondEp[TSDB_EP_LEN]; } SDnodeObjCfg; /** diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e1306800d3..01855f6d92 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -46,7 +46,8 @@ typedef struct SMnodeLoad { typedef struct SMnodeCfg { int32_t sver; - int8_t enableTelem; + bool enableTelem; + bool printAuth; int32_t statusInterval; int32_t shellActivityTimer; char *timezone; diff --git a/include/libs/config/config.h b/include/libs/config/config.h index cf8daae72b..c3e026e20b 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -38,8 +38,6 @@ typedef enum { typedef enum { CFG_DTYPE_NONE, CFG_DTYPE_BOOL, - CFG_DTYPE_INT8, - CFG_DTYPE_UINT16, CFG_DTYPE_INT32, CFG_DTYPE_INT64, CFG_DTYPE_FLOAT, @@ -51,37 +49,24 @@ typedef enum { CFG_DTYPE_TIMEZONE } ECfgDataType; -typedef enum { - CFG_UTYPE_NONE, - CFG_UTYPE_GB, - CFG_UTYPE_MB, - CFG_UTYPE_BYTE, - CFG_UTYPE_SECOND, - CFG_UTYPE_MS, - CFG_UTYPE_PERCENT -} ECfgUnitType; - typedef struct SConfigItem { ECfgSrcType stype; - ECfgUnitType utype; ECfgDataType dtype; char *name; union { - bool boolVal; - int8_t int8Val; - uint16_t uint16Val; - int32_t int32Val; - int64_t int64Val; - float floatVal; - char *strVal; + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; }; union { - int64_t minIntVal; - double minFloatVal; + int64_t imin; + double fmin; }; union { - int64_t maxIntVal; - double maxFloatVal; + int64_t imax; + double fmax; }; } SConfigItem; @@ -96,27 +81,19 @@ SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter); void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter); SConfigItem *cfgGetItem(SConfig *pConfig, const char *name); -int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype); -int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype); -int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype); -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype); -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype); -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval, - ECfgUnitType utype); -int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); -int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype); +int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal); +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval); +int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal); +int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVa); +int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal); +int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); -const char *cfgUtypeStr(ECfgUnitType type); #ifdef __cplusplus } diff --git a/include/util/tlog.h b/include/util/tlog.h index 26a5417320..d5a23f9e96 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -74,6 +74,9 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . void taosDumpData(unsigned char *msg, int32_t len); + +void taosSetDebugFlag(int32_t flag); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1fa6446ec7..dba4aff3a7 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -73,6 +73,7 @@ static void deregisterRequest(SRequestObj* pRequest) { } static void tscInitLogFile() { + #if 0 taosReadGlobalLogCfg(); if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { printf("failed to create log dir:%s\n", tsLogDir); @@ -86,6 +87,7 @@ static void tscInitLogFile() { if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } + #endif } // todo close the transporter properly @@ -224,9 +226,10 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); +#if 0 taosInitGlobalCfg(); taosReadCfgFromFile(); - +#endif tscInitLogFile(); if (taosCheckAndPrintCfg()) { tscInitRes = -1; @@ -268,6 +271,7 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#if 0 SGlobalCfg *cfg = NULL; switch (option) { @@ -418,7 +422,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("Invalid option %d", option); return -1; } - +#endif return 0; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 398ded034c..35ded42991 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -340,47 +340,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // port - cfg.option = "serverPort"; - cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_UINT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65056; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "supportVnodes"; - cfg.ptr = &tsNumOfSupportVnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // directory - cfg.option = "configDir"; - cfg.ptr = configDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logDir"; - cfg.ptr = tsLogDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; @@ -402,36 +361,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // dnode configs - cfg.option = "numOfThreadsPerCore"; - cfg.ptr = &tsNumOfThreadsPerCore; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "numOfCommitThreads"; - cfg.ptr = &tsNumOfCommitThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "ratioOfQueryCores"; - cfg.ptr = &tsRatioOfQueryCores; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.0f; - cfg.maxValue = 2.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -494,25 +423,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosAddConfigOption(cfg); - cfg.option = "statusInterval"; - cfg.ptr = &tsStatusInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "shellActivityTimer"; - cfg.ptr = &tsShellActivityTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 120; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; @@ -654,47 +564,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // locale & charset - cfg.option = "timezone"; - cfg.ptr = tsTimezone; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_TIMEZONE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "locale"; - cfg.ptr = tsLocale; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "charset"; - cfg.ptr = tsCharset; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // connect configs - cfg.option = "maxShellConns"; - cfg.ptr = &tsMaxShellConns; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 50000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxConnections"; cfg.ptr = &tsMaxConnections; @@ -746,176 +615,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // debug flag - cfg.option = "numOfLogLines"; - cfg.ptr = &tsNumOfLogLines; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1000; - cfg.maxValue = 2000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "logKeepDays"; - cfg.ptr = &tsLogKeepDays; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = -365000; - cfg.maxValue = 365000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "asyncLog"; - cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "debugFlag"; - cfg.ptr = &debugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mDebugFlag"; - cfg.ptr = &mDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dDebugFlag"; - cfg.ptr = &dDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "sDebugFlag"; - cfg.ptr = &sDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "wDebugFlag"; - cfg.ptr = &wDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcDebugFlag"; - cfg.ptr = &rpcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tmrDebugFlag"; - cfg.ptr = &tmrDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cDebugFlag"; - cfg.ptr = &cDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "jniDebugFlag"; - cfg.ptr = &jniDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "uDebugFlag"; - cfg.ptr = &uDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "qDebugFlag"; - cfg.ptr = &qDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vDebugFlag"; - cfg.ptr = &vDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbDebugFlag"; - cfg.ptr = &tsdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cqDebugFlag"; - cfg.ptr = &cqDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -937,46 +637,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // version info - cfg.option = "gitinfo"; - cfg.ptr = gitinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "gitinfoOfInternal"; - cfg.ptr = gitinfoOfInternal; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "buildinfo"; - cfg.ptr = buildinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "version"; - cfg.ptr = version; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h index 3a28288bed..dd264b982d 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -20,15 +20,25 @@ #include "config.h" #include "dnode.h" #include "tglobal.h" -#include "tnote.h" #include "ulog.h" +#include "taoserror.h" #ifdef __cplusplus extern "C" { #endif -int32_t dnmInitCfg(SDnodeEnvCfg *pEnvCfg, SDnodeObjCfg *pObjCfg, const char *configFile, const char *envFile, - const char *apolloUrl); +int32_t dmnInitLogCfg(SConfig *pCfg); +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); + +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); + +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg); +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg); + +void dmnDumpCfg(SConfig *pCfg); +void dmnPrintVersion(SConfig *pCfg); +void dmnGenerateGrant(); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index a40867d11d..a91662a316 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include "dnode.h" #include "dmnInt.h" static struct { @@ -24,14 +23,16 @@ static struct { bool printAuth; bool printVersion; char configDir[PATH_MAX]; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; } dmn = {0}; -void dmnSigintHandle(int signum, void *info, void *ctx) { +static void dmnSigintHandle(int signum, void *info, void *ctx) { uInfo("singal:%d is received", signum); dmn.stop = true; } -void dmnSetSignalHandle() { +static void dmnSetSignalHandle() { taosSetSignal(SIGTERM, dmnSigintHandle); taosSetSignal(SIGHUP, dmnSigintHandle); taosSetSignal(SIGINT, dmnSigintHandle); @@ -39,10 +40,17 @@ void dmnSetSignalHandle() { taosSetSignal(SIGBREAK, dmnSigintHandle); } -int dmnParseOption(int argc, char const *argv[]) { +static void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!dmn.stop) { + taosMsleep(100); + } +} + +static int32_t dmnParseOption(int32_t argc, char const *argv[]) { tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - for (int i = 1; i < argc; ++i) { + for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { if (strlen(argv[++i]) >= PATH_MAX) { @@ -58,8 +66,6 @@ int dmnParseOption(int argc, char const *argv[]) { dmn.dumpConfig = true; } else if (strcmp(argv[i], "-k") == 0) { dmn.generateGrant = true; - } else if (strcmp(argv[i], "-A") == 0) { - dmn.printAuth = true; } else if (strcmp(argv[i], "-V") == 0) { dmn.printVersion = true; } else { @@ -69,86 +75,17 @@ int dmnParseOption(int argc, char const *argv[]) { return 0; } -void dmnGenerateGrant() { -#if 0 - grantParseParameter(); -#endif -} - -void dmnPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("builuInfo: %s\n", buildinfo); -} - -int dmnReadConfig(const char *path) { - tstrncpy(configDir, dmn.configDir, PATH_MAX); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (taosMkDir(tsLogDir) != 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosInitNotes() != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosReadCfgFromFile() != 0) { - uError("failed to read config"); - return -1; - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); - return -1; - } - - taosSetCoreDump(tsEnableCoreFile); - return 0; -} - -void dmnDumpConfig() { taosDumpGlobalCfg(); } - -void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -int dmnRunDnode() { - SDnodeEnvCfg envCfg = {0}; - SDnodeObjCfg objCfg = {0}; - - if (dnmInitCfg(&envCfg, &objCfg, "", "", "") != 0) { - uInfo("Failed to start TDengine since load config error"); - return -1; - } - +int32_t dmnRunDnode(SConfig *pCfg) { + SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg); if (dndInit(&envCfg) != 0) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); + uInfo("Failed to start TDengine, please check the log"); return -1; } + SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg); SDnode *pDnode = dndCreate(&objCfg); if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); + uInfo("Failed to start TDengine, please check the log"); return -1; } @@ -168,23 +105,33 @@ int main(int argc, char const *argv[]) { } if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; + dmnGenerateGrant(); + return 0; } - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (dmnReadConfig(dmn.configDir) != 0) { + if (dmnInitLog(dmn.configDir, dmn.envFile, dmn.apolloUrl) != 0) { return -1; } - if (dmn.dumpConfig) { - dmnDumpConfig(); + SConfig *pCfg = dmnReadCfg(dmn.configDir, dmn.envFile, dmn.apolloUrl); + if (pCfg == NULL) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (dmn.printVersion) { + dmnPrintVersion(pCfg); + cfgCleanup(pCfg); return 0; } - return dmnRunDnode(); + if (dmn.dumpConfig) { + dmnDumpCfg(pCfg); + cfgCleanup(pCfg); + return 0; + } + + int32_t code = dmnRunDnode(pCfg); + cfgCleanup(pCfg); + return code; } diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 1bd8476f94..e4414d8dad 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -16,205 +16,157 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" -static void dmnInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); +static int32_t dmnInitDnodeCfg(SConfig *pConfig) { + if (cfgAddString(pConfig, "version", version) != 0) return -1; + if (cfgAddString(pConfig, "buildinfo", buildinfo) != 0) return -1; + if (cfgAddString(pConfig, "gitinfo", gitinfo) != 0) return -1; + if (cfgAddTimezone(pConfig, "timezone", "") != 0) return -1; + if (cfgAddLocale(pConfig, "locale", "") != 0) return -1; + if (cfgAddCharset(pConfig, "charset", "") != 0) return -1; + if (cfgAddInt32(pConfig, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pConfig, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + if (cfgAddBool(pConfig, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pConfig, "enableCoreFile", 0) != 0) return -1; + if (cfgAddInt32(pConfig, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pConfig, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pConfig, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pConfig, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pConfig, "maxShellConns", 50000, 10, 50000000) != 0) return -1; + if (cfgAddInt32(pConfig, "shellActivityTimer", 3, 1, 120) != 0) return -1; + if (cfgAddInt32(pConfig, "serverPort", 6030, 1, 65056) != 0) return -1; + return 0; } -static void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char configDir[PATH_MAX] = {0}; + char configFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, configDir, PATH_MAX); + snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { + uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; } -#if 0 -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pConfig = cfgInit(); + if (pConfig == NULL) return NULL; + + if (dmnInitLogCfg(pConfig) != 0) { + uError("failed to init log cfg since %s", terrstr()); + cfgCleanup(pConfig); + return NULL; + } + + if (dmnInitDnodeCfg(pConfig) != 0) { + uError("failed to init dnode cfg since %s", terrstr()); + cfgCleanup(pConfig); + return NULL; + } + + if (dmnLoadCfg(pConfig, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pConfig); + return NULL; + } + + bool enableCore = cfgGetItem(pConfig, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + if (taosCheckAndPrintCfg() != 0) { + uError("failed to check config"); + return NULL; + } - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - taosReadLogOption(option, value); - } - - tfree(line); - fclose(fp); + return pConfig; } - -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } - } - - taosPrintOsInfo(); - uInfo("=================================="); -} - -#if 0 -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } -} - -void taosDumpGlobalCfg() { +void dmnDumpCfg(SConfig *pCfg) { printf("taos global config:\n"); printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; - taosDumpCfg(cfg); - } - - printf("\ntaos local config:\n"); - printf("==================================\n"); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; - - taosDumpCfg(cfg); + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT32: + printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT64: + printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_FLOAT: + printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + break; + } + pItem = cfgIterate(pCfg, pItem); } } -#endif +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) { + SDnodeEnvCfg envCfg = {0}; -#endif - -static int32_t dmnInitLog() { + const char *vstr = cfgGetItem(pCfg, "version")->str; + envCfg.sver = 30000000; + tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo)); + tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo)); + tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone)); + tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale)); + tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset)); + envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32; + envCfg.numOfCommitThreads = (uint16_t) cfgGetItem(pCfg, "numOfCommitThreads")->i32; + envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + return envCfg; } -int32_t dnmInitCfg(SDnodeEnvCfg *pEnvCfg, SDnodeObjCfg *pObjCfg, const char *configFile, const char *envFile, - const char *apolloUrl) { - dmnInitEnvCfg(pEnvCfg); - dmnInitObjCfg(pObjCfg); - return 0; +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { + SDnodeObjCfg objCfg = {0}; + + objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; + objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; + objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); + tstrncpy(objCfg.localEp, cfgGetItem(pCfg, "localEp")->str, sizeof(objCfg.localEp)); + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "localFqdn")->str, sizeof(objCfg.localFqdn, cfgGetItem)); + tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); + + return objCfg; } \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c index e69de29bb2..8cce1bb01e 100644 --- a/source/dnode/mgmt/daemon/src/dmnLog.c +++ b/source/dnode/mgmt/daemon/src/dmnLog.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +int32_t dmnInitLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cqDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (dmnInitLogCfg(pCfg) != 0) { + uError("failed to init log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + char temp[PATH_MAX] = {0}; + snprintf(temp, PATH_MAX, "%s" TD_DIRSEP "taosdlog", cfgGetItem(pCfg, "logDir")->str); + if (taosInitLog(temp, cfgGetItem(pCfg, "numOfLogLines")->i32, 1) != 0) { + uError("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgCleanup(pCfg); + return 0; +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c new file mode 100644 index 0000000000..aac27c4c33 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +void dmnGenerateGrant() { +#if 0 + grantParseParameter(); +#endif +} + +void dmnPrintVersion(SConfig *pCfg) { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 6c7d4be581..5c32da966b 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -56,7 +56,7 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; + bool enable; SRWLatch lock; char email[TSDB_FQDN_LEN]; } STelemMgmt; diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt index 0bd467198d..596cc0b433 100644 --- a/source/libs/config/CMakeLists.txt +++ b/source/libs/config/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(config os util) +target_link_libraries(config os util common) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h index 0b62018222..6497393792 100644 --- a/source/libs/config/inc/cfgInt.h +++ b/source/libs/config/inc/cfgInt.h @@ -22,6 +22,7 @@ #include "thash.h" #include "tutil.h" #include "ulog.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index a40783eaef..dbc4767aee 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -15,6 +15,10 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" +#include "tmsg.h" +#include "tep.h" +#include "tlocale.h" +#include "ttimezone.h" SConfig *cfgInit() { SConfig *pConfig = calloc(1, sizeof(SConfig)); @@ -23,7 +27,7 @@ SConfig *cfgInit() { return NULL; } - pConfig->hash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + pConfig->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); if (pConfig->hash == NULL) { free(pConfig); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -64,6 +68,85 @@ SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashI void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashCancelIterate(pConfig->hash, pIter); } +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + tfree(pItem->str); + pItem->str = strdup(timezone); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + tfree(pItem->str); + pItem->str = strdup(charset); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + tfree(pItem->str); + pItem->str = strdup(locale); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(fullDir); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetIpStr(SConfigItem *pItem, const char *ip) { + uint32_t value = taosInetAddr(ip); + if (value == INADDR_NONE) { + uError("ip:%s is not a valid ip address", ip); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(ip); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { bool tmp = false; if (strcasecmp(value, "true") == 0) { @@ -73,83 +156,53 @@ static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType sty tmp = true; } - pItem->boolVal = tmp; - pItem->stype = stype; - return 0; -} - -static int32_t cfgSetInt8(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int8_t ival = (int8_t)atoi(value); - if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { - uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", - pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, - cfgStypeStr(pItem->stype), pItem->int8Val); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - pItem->int8Val = ival; - pItem->stype = stype; - return 0; -} - -static int32_t cfgSetUInt16(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - uint16_t ival = (uint16_t)atoi(value); - if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { - uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", - pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, - cfgStypeStr(pItem->stype), pItem->uint16Val); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - pItem->uint16Val = ival; + pItem->bval = tmp; pItem->stype = stype; return 0; } static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int32_t ival = (int32_t)atoi(value); - if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", - pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, - cfgStypeStr(pItem->stype), pItem->int32Val); + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i32); terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - pItem->int32Val = ival; + pItem->i32 = ival; pItem->stype = stype; return 0; } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int64_t ival = (int64_t)atoi(value); - if (ival < pItem->minIntVal || ival > pItem->maxIntVal) { + if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%" PRId64, - pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->minIntVal, pItem->maxIntVal, - cfgStypeStr(pItem->stype), pItem->int64Val); + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i64); terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - pItem->int64Val = ival; + pItem->i64 = ival; pItem->stype = stype; return 0; } static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { float fval = (float)atof(value); - if (fval < pItem->minFloatVal || fval > pItem->maxFloatVal) { + if (fval < pItem->fmin || fval > pItem->fmax) { uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->minFloatVal, pItem->maxFloatVal, - cfgStypeStr(pItem->stype), pItem->floatVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype), + pItem->fval); terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - pItem->floatVal = fval; + pItem->fval = fval; pItem->stype = stype; return 0; } @@ -159,42 +212,82 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); return -1; } - free(pItem->strVal); - pItem->strVal = tmp; + free(pItem->str); + pItem->str = tmp; pItem->stype = stype; return 0; } static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); - if (tmp == NULL) { + if (tmp == NULL || cfgCheckAndSetIpStr(pItem, value) != 0) { + free(tmp); terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); return -1; } - free(pItem->strVal); - pItem->strVal = tmp; pItem->stype = stype; return 0; } static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); - if (tmp == NULL) { + if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) { + free(tmp); terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->strVal); + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetLocale(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetCharset(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetTimezone(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); return -1; } - free(pItem->strVal); - pItem->strVal = tmp; pItem->stype = stype; return 0; } @@ -208,10 +301,6 @@ int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSr switch (pItem->dtype) { case CFG_DTYPE_BOOL: return cfgSetBool(pItem, value, stype); - case CFG_DTYPE_INT8: - return cfgSetInt8(pItem, value, stype); - case CFG_DTYPE_UINT16: - return cfgSetUInt16(pItem, value, stype); case CFG_DTYPE_INT32: return cfgSetInt32(pItem, value, stype); case CFG_DTYPE_INT64: @@ -224,6 +313,12 @@ int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSr return cfgSetIpStr(pItem, value, stype); case CFG_DTYPE_DIR: return cfgSetDir(pItem, value, stype); + case CFG_DTYPE_TIMEZONE: + return cfgSetTimezone(pItem, value, stype); + case CFG_DTYPE_CHARSET: + return cfgSetCharset(pItem, value, stype); + case CFG_DTYPE_LOCALE: + return cfgSetLocale(pItem, value, stype); case CFG_DTYPE_NONE: default: break; @@ -246,9 +341,8 @@ SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { return pItem; } -static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name, ECfgUnitType utype) { +static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; - pItem->utype = utype; pItem->name = strdup(name); if (pItem->name == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -261,7 +355,7 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name if (taosHashPut(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { if (pItem->dtype == CFG_DTYPE_STRING) { - free(pItem->strVal); + free(pItem->str); } free(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -271,202 +365,94 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name return 0; } -int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal, ECfgUnitType utype) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .boolVal = defaultVal}; - return cfgAddItem(pConfig, &item, name, utype); +int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal}; + return cfgAddItem(pConfig, &item, name); } -int32_t cfgAddInt8(SConfig *pConfig, const char *name, int8_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype) { +int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT8, .int8Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; - return cfgAddItem(pConfig, &item, name, utype); + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pConfig, &item, name); } -int32_t cfgAddUInt16(SConfig *pConfig, const char *name, uint16_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype) { +int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_UINT16, .uint16Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; - return cfgAddItem(pConfig, &item, name, utype); + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pConfig, &item, name); } -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype) { +int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .int32Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; - return cfgAddItem(pConfig, &item, name, utype); + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval}; + return cfgAddItem(pConfig, &item, name); } -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, - ECfgUnitType utype) { - if (defaultVal < minval || defaultVal > maxval) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .int64Val = defaultVal, .minIntVal = minval, .maxIntVal = maxval}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval, - ECfgUnitType utype) { - if (defaultVal < minval || defaultVal > maxval) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .floatVal = defaultVal, .minFloatVal = minval, .maxFloatVal = maxval}; - return cfgAddItem(pConfig, &item, name, utype); -} - -int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { - if (defaultVal == NULL) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - +int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_STRING}; - item.strVal = strdup(defaultVal); - if (item.strVal == NULL) { + item.str = strdup(defaultVal); + if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - return cfgAddItem(pConfig, &item, name, utype); + return cfgAddItem(pConfig, &item, name); } -static int32_t cfgCheckIpStr(const char *ip) { - uint32_t value = taosInetAddr(ip); - if (value == INADDR_NONE) { - uError("ip:%s is not a valid ip address", ip); - return -1; - } - - return 0; -} - -int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { - if (cfgCheckIpStr(defaultVal) != 0) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - +int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; - item.strVal = strdup(defaultVal); - if (item.strVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name, utype); + + return cfgAddItem(pConfig, &item, name); } -static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { - char fullDir[PATH_MAX] = {0}; - if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to expand dir:%s since %s", inputDir, terrstr()); - return -1; - } - - if (taosRealPath(fullDir, PATH_MAX) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); - return -1; - } - - if (taosMkDir(fullDir) != 0) { - uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); - return -1; - } - - tfree(pItem->strVal); - pItem->strVal = strdup(fullDir); - if (pItem->strVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { +int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_DIR}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name, utype); + return cfgAddItem(pConfig, &item, name); } -static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { - tfree(pItem->strVal); - pItem->strVal = strdup(locale); - if (pItem->strVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { +int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name, utype); + return cfgAddItem(pConfig, &item, name); } -static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { - tfree(pItem->strVal); - pItem->strVal = strdup(charset); - if (pItem->strVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { +int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name, utype); + return cfgAddItem(pConfig, &item, name); } -static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { - tfree(pItem->strVal); - pItem->strVal = strdup(timezone); - if (pItem->strVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal, ECfgUnitType utype) { +int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name, utype); + return cfgAddItem(pConfig, &item, name); } const char *cfgStypeStr(ECfgSrcType type) { @@ -496,10 +482,6 @@ const char *cfgDtypeStr(ECfgDataType type) { return "none"; case CFG_DTYPE_BOOL: return "bool"; - case CFG_DTYPE_INT8: - return "int8"; - case CFG_DTYPE_UINT16: - return "uint16"; case CFG_DTYPE_INT32: return "int32"; case CFG_DTYPE_INT64: @@ -523,23 +505,120 @@ const char *cfgDtypeStr(ECfgDataType type) { } } -const char *cfgUtypeStr(ECfgUnitType type) { - switch (type) { - case CFG_UTYPE_NONE: - return ""; - case CFG_UTYPE_PERCENT: - return "(%)"; - case CFG_UTYPE_GB: - return "(GB)"; - case CFG_UTYPE_MB: - return "(Mb)"; - case CFG_UTYPE_BYTE: - return "(byte)"; - case CFG_UTYPE_SECOND: - return "(s)"; - case CFG_UTYPE_MS: - return "(ms)"; - default: - return "invalid"; +void cfgPrintCfg(SConfig *pCfg) { + uInfo("taos global config"); + uInfo("=================================="); + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + uInfo("cfg:%s, value:%u src:%s", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT32: + uInfo("cfg:%s, value:%d src:%s", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT64: + uInfo("cfg:%s, value:%" PRId64 " src:%s", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_FLOAT: + uInfo("cfg:%s, value:%f src:%s", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uInfo("cfg:%s, value:%s src:%s", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + break; + } + pItem = cfgIterate(pCfg, pItem); } -} \ No newline at end of file +} + +int32_t cfgCheck(SConfig *pConfig) { + SConfigItem *pItem = cfgGetItem(pConfig, "debugFlag"); + if (pItem != NULL) { + taosSetDebugFlag(pItem->i32); + } + + pItem = cfgGetItem(pConfig, "localFqdn"); + if (pItem != NULL) { + tstrncpy(tsLocalFqdn, pItem->str, TSDB_FQDN_LEN); + } + + pItem = cfgGetItem(pConfig, "serverPort"); + if (pItem != NULL) { + tsServerPort = (uint16_t)pItem->i32; + } + + pItem = cfgGetItem(pConfig, "firstEp"); + if (pItem != NULL) { + tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); + } + + snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + uInfo("localEp is: %s", tsLocalEp); + + SEp ep = {0}; + if (tsFirst[0] == 0) { + strcpy(tsFirst, tsLocalEp); + } else { + taosGetFqdnPortFromEp(tsFirst, &ep); + snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); + } + + pItem = cfgGetItem(pConfig, "secondEp"); + if (pItem != NULL) { + tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); + } + + if (tsSecond[0] == 0) { + strcpy(tsSecond, tsLocalEp); + } else { + taosGetFqdnPortFromEp(tsSecond, &ep); + snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); + } + + pItem = cfgGetItem(pConfig, "dataDir"); + if (pItem != NULL) { + tstrncpy(tsDataDir, pItem->str, PATH_MAX); + } + + if (tsDiskCfgNum <= 0) { + taosAddDataDir(0, tsDataDir, 0, 1); + tsDiskCfgNum = 1; + uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); + } + + if (taosDirExist(tsTempDir) != 0) { + return -1; + } + + taosGetSystemInfo(); + + tsSetLocale(); + + // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); + // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { + tsSetTimeZone(); + // } + + pItem = cfgGetItem(pConfig, "numOfCores"); + if (pItem != NULL) { + tsNumOfCores = pItem->i32; + } + + if (tsNumOfCores <= 0) { + tsNumOfCores = 1; + } + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + + cfgPrintCfg(pConfig); + + return 0; +} diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp index 62ed5764ac..7c70e1a8f8 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/libs/config/test/cfgTest.cpp @@ -20,15 +20,14 @@ class CfgTest : public ::testing::Test { public: void SetUp() override {} void TearDown() override {} - - void InitializeConfig(SConfig *pConfig); + void InitCfg(SConfig *pConfig); static const char *pConfig; }; const char *CfgTest::pConfig; -TEST_F(CfgTest, 02_Str) { +TEST_F(CfgTest, 01_Str) { EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); @@ -40,72 +39,56 @@ TEST_F(CfgTest, 02_Str) { EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT8), "int8"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_UINT16), "uint16"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); - - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_NONE), ""); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_GB), "(GB)"); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MB), "(Mb)"); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_BYTE), "(byte)"); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_SECOND), "(s)"); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_MS), "(ms)"); - EXPECT_STREQ(cfgUtypeStr(CFG_UTYPE_PERCENT), "(%)"); - EXPECT_STREQ(cfgUtypeStr(ECfgUnitType(1024)), "invalid"); } TEST_F(CfgTest, 02_Basic) { SConfig *pConfig = cfgInit(); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddInt8(pConfig, "test_int8", 1, 0, 16, CFG_UTYPE_GB), 0); - EXPECT_EQ(cfgAddUInt16(pConfig, "test_uint16", 2, 0, 16, CFG_UTYPE_MB), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 3, 0, 16, CFG_UTYPE_BYTE), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 4, 0, 16, CFG_UTYPE_SECOND), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 5, 0, 16, CFG_UTYPE_MS), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "6", CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1", CFG_UTYPE_NONE), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", CFG_UTYPE_NONE), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4"), 0); + EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1"), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp"), 0); - EXPECT_EQ(cfgGetSize(pConfig), 9); + EXPECT_EQ(cfgGetSize(pConfig), 7); int32_t size = 0; SConfigItem *pItem = cfgIterate(pConfig, NULL); while (pItem != NULL) { switch (pItem->dtype) { case CFG_DTYPE_BOOL: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->boolVal); - break; - case CFG_DTYPE_INT8: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int8Val); - break; - case CFG_DTYPE_UINT16: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->uint16Val); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); break; case CFG_DTYPE_INT32: - printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->int32Val); + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32); break; case CFG_DTYPE_INT64: - printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->int64Val); + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64); break; case CFG_DTYPE_FLOAT: - printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->floatVal); + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval); break; case CFG_DTYPE_STRING: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); break; case CFG_DTYPE_IPSTR: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); break; case CFG_DTYPE_DIR: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->strVal); + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); break; default: printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); @@ -116,70 +99,49 @@ TEST_F(CfgTest, 02_Basic) { } cfgCancelIterate(pConfig, pItem); - EXPECT_EQ(cfgGetSize(pConfig), 9); + EXPECT_EQ(cfgGetSize(pConfig), 7); pItem = cfgGetItem(pConfig, "test_bool"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); EXPECT_STREQ(pItem->name, "test_bool"); - EXPECT_EQ(pItem->boolVal, 0); - - pItem = cfgGetItem(pConfig, "test_int8"); - EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_GB); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT8); - EXPECT_STREQ(pItem->name, "test_int8"); - EXPECT_EQ(pItem->int8Val, 1); - - pItem = cfgGetItem(pConfig, "test_uint16"); - EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_MB); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_UINT16); - EXPECT_STREQ(pItem->name, "test_uint16"); - EXPECT_EQ(pItem->uint16Val, 2); + EXPECT_EQ(pItem->bval, 0); pItem = cfgGetItem(pConfig, "test_int32"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_BYTE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); EXPECT_STREQ(pItem->name, "test_int32"); - EXPECT_EQ(pItem->int32Val, 3); + EXPECT_EQ(pItem->i32, 1); pItem = cfgGetItem(pConfig, "test_int64"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_SECOND); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); EXPECT_STREQ(pItem->name, "test_int64"); - EXPECT_EQ(pItem->int64Val, 4); + EXPECT_EQ(pItem->i64, 2); pItem = cfgGetItem(pConfig, "test_float"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_MS); EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); EXPECT_STREQ(pItem->name, "test_float"); - EXPECT_EQ(pItem->floatVal, 5); + EXPECT_EQ(pItem->fval, 3); pItem = cfgGetItem(pConfig, "test_string"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); EXPECT_STREQ(pItem->name, "test_string"); - EXPECT_STREQ(pItem->strVal, "6"); + EXPECT_STREQ(pItem->str, "4"); pItem = cfgGetItem(pConfig, "test_ipstr"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); EXPECT_STREQ(pItem->name, "test_ipstr"); - EXPECT_STREQ(pItem->strVal, "192.168.0.1"); + EXPECT_STREQ(pItem->str, "192.168.0.1"); pItem = cfgGetItem(pConfig, "test_dir"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->utype, CFG_UTYPE_NONE); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); EXPECT_STREQ(pItem->name, "test_dir"); - EXPECT_STREQ(pItem->strVal, "/tmp"); + EXPECT_STREQ(pItem->str, "/tmp"); cfgCleanup(pConfig); } diff --git a/source/common/src/ttimezone.c b/source/os/src/osTimeZone.c similarity index 95% rename from source/common/src/ttimezone.c rename to source/os/src/osTimeZone.c index 0ee5702242..da40e14389 100644 --- a/source/common/src/ttimezone.c +++ b/source/os/src/osTimeZone.c @@ -15,12 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tutil.h" +// #include "ulog.h" +// #include "tglobal.h" +// #include "tutil.h" // TODO refactor to set the tz value through parameter void tsSetTimeZone() { +#if 0 SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); if (cfg_timezone != NULL) { uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); @@ -63,4 +64,6 @@ void tsSetTimeZone() { tsDaylight = daylight; uInfo("timezone format changed to %s", tsTimezone); + +#endif } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2579082022..233b000c61 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -773,3 +773,12 @@ void taosPrintOsInfo() { uInfo(" os version: %s", info.version); uInfo(" os machine: %s", info.machine); } + +void taosSetDebugFlag(int32_t flag) { + if (!(debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP)) return; + + debugFlag = flag; + mDebugFlag = flag; + dDebugFlag = flag; + cDebugFlag = flag; +} \ No newline at end of file From 460f78903648311e51c056ddc83682c781e5c149 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 10:35:53 +0800 Subject: [PATCH 17/36] log variables --- include/util/tlog.h | 12 +-- source/client/src/clientEnv.c | 4 +- source/common/src/tglobal.c | 16 --- source/dnode/mgmt/daemon/inc/dmnInt.h | 11 +-- source/dnode/mgmt/daemon/src/daemon.c | 32 +++--- source/dnode/mgmt/daemon/src/dmnCfg.c | 99 ++++++++----------- source/dnode/mgmt/daemon/src/dmnLog.c | 95 +++++++++++++++--- source/dnode/mgmt/daemon/src/dmnUtil.c | 2 +- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 6 +- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/config/src/config.c | 5 +- source/libs/qworker/test/qworkerTests.cpp | 4 +- source/libs/scheduler/test/schedulerTests.cpp | 4 +- source/libs/transport/test/pushClient.c | 2 +- source/libs/transport/test/pushServer.c | 2 +- source/libs/transport/test/rclient.c | 2 +- source/libs/transport/test/rsclient.c | 2 +- source/libs/transport/test/rserver.c | 2 +- source/libs/transport/test/syncClient.c | 2 +- source/util/src/tlog.c | 39 +++++--- source/util/src/ttimer.c | 1 - source/util/test/trefTest.c | 2 +- 22 files changed, 182 insertions(+), 166 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index d5a23f9e96..7499eaa7b4 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -23,7 +23,7 @@ extern "C" { #endif // log -extern int8_t tsAsyncLog; +extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; @@ -32,9 +32,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; @@ -42,8 +39,7 @@ extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; +extern int32_t fsDebugFlag; #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL @@ -56,7 +52,7 @@ extern int32_t debugFlag; #define DEBUG_SCREEN 64U #define DEBUG_FILE 128U -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); @@ -75,7 +71,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . void taosDumpData(unsigned char *msg, int32_t len); -void taosSetDebugFlag(int32_t flag); +void taosSetAllDebugFlag(int32_t flag); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index dba4aff3a7..93fbdeb2ac 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -82,9 +82,7 @@ static void tscInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 35ded42991..655d62a148 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -178,22 +178,6 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; -void taosSetAllDebugFlag() { - if (debugFlag != 0) { - mDebugFlag = debugFlag; - dDebugFlag = debugFlag; - vDebugFlag = debugFlag; - jniDebugFlag = debugFlag; - qDebugFlag = debugFlag; - rpcDebugFlag = debugFlag; - uDebugFlag = debugFlag; - sDebugFlag = debugFlag; - wDebugFlag = debugFlag; - tsdbDebugFlag = debugFlag; - cqDebugFlag = debugFlag; - uInfo("all debug flag are set to %d", debugFlag); - } -} int32_t taosCfgDynamicOptions(char *msg) { #if 0 diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h index dd264b982d..7ffb0bdeb7 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -19,25 +19,24 @@ #include "config.h" #include "dnode.h" +#include "taoserror.h" #include "tglobal.h" #include "ulog.h" -#include "taoserror.h" #ifdef __cplusplus extern "C" { #endif -int32_t dmnInitLogCfg(SConfig *pCfg); +int32_t dmnAddLogCfg(SConfig *pCfg); int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); -int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); -SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); - +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg); SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg); void dmnDumpCfg(SConfig *pCfg); -void dmnPrintVersion(SConfig *pCfg); +void dmnPrintVersion(); void dmnGenerateGrant(); #ifdef __cplusplus diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index a91662a316..1f1d2d8d05 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -22,7 +22,6 @@ static struct { bool generateGrant; bool printAuth; bool printVersion; - char configDir[PATH_MAX]; char envFile[PATH_MAX]; char apolloUrl[PATH_MAX]; } dmn = {0}; @@ -48,8 +47,6 @@ static void dmnWaitSignal() { } static int32_t dmnParseOption(int32_t argc, char const *argv[]) { - tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { @@ -57,7 +54,7 @@ static int32_t dmnParseOption(int32_t argc, char const *argv[]) { printf("config file path overflow"); return -1; } - tstrncpy(dmn.configDir, argv[i], PATH_MAX); + tstrncpy(configDir, argv[i], PATH_MAX); } else { printf("'-c' requires a parameter, default is %s\n", configDir); return -1; @@ -105,26 +102,25 @@ int main(int argc, char const *argv[]) { } if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmnInitLog(dmn.configDir, dmn.envFile, dmn.apolloUrl) != 0) { - return -1; - } - - SConfig *pCfg = dmnReadCfg(dmn.configDir, dmn.envFile, dmn.apolloUrl); - if (pCfg == NULL) { - uInfo("Failed to start TDengine since read config error"); - return -1; + dmnGenerateGrant(); + return 0; } if (dmn.printVersion) { - dmnPrintVersion(pCfg); - cfgCleanup(pCfg); + dmnPrintVersion(); return 0; } + if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) { + return -1; + } + + SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl); + if (pCfg == NULL) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + if (dmn.dumpConfig) { dmnDumpCfg(pCfg); cfgCleanup(pCfg); diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index e4414d8dad..9d9082a76b 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -16,91 +16,72 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" -static int32_t dmnInitDnodeCfg(SConfig *pConfig) { - if (cfgAddString(pConfig, "version", version) != 0) return -1; - if (cfgAddString(pConfig, "buildinfo", buildinfo) != 0) return -1; - if (cfgAddString(pConfig, "gitinfo", gitinfo) != 0) return -1; - if (cfgAddTimezone(pConfig, "timezone", "") != 0) return -1; - if (cfgAddLocale(pConfig, "locale", "") != 0) return -1; - if (cfgAddCharset(pConfig, "charset", "") != 0) return -1; - if (cfgAddInt32(pConfig, "numOfCores", 1, 1, 100000) != 0) return -1; - if (cfgAddInt32(pConfig, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; - if (cfgAddBool(pConfig, "telemetryReporting", 0) != 0) return -1; - if (cfgAddBool(pConfig, "enableCoreFile", 0) != 0) return -1; - if (cfgAddInt32(pConfig, "supportVnodes", 256, 0, 65536) != 0) return -1; - if (cfgAddInt32(pConfig, "statusInterval", 1, 1, 30) != 0) return -1; - if (cfgAddFloat(pConfig, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; - if (cfgAddFloat(pConfig, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; - if (cfgAddInt32(pConfig, "maxShellConns", 50000, 10, 50000000) != 0) return -1; - if (cfgAddInt32(pConfig, "shellActivityTimer", 3, 1, 120) != 0) return -1; - if (cfgAddInt32(pConfig, "serverPort", 6030, 1, 65056) != 0) return -1; +static int32_t dmnAddDnodeCfg(SConfig *pCfg) { + if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + if (cfgAddString(pCfg, "version", version) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "") != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "") != 0) return -1; + if (cfgAddString(pCfg, "fqdn", "") != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", 6030, 1, 65056) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; return 0; } -int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { - char configDir[PATH_MAX] = {0}; - char configFile[PATH_MAX + 100] = {0}; +int32_t dmnCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); - taosExpandDir(inputCfgDir, configDir, PATH_MAX); - snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); - - if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { - uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { - uError("failed to load from config file:%s since %s\n", configFile, terrstr()); - return -1; - } - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { - uError("failed to load from env file:%s since %s\n", envFile, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { - uError("failed to load from global env variables since %s\n", terrstr()); - return -1; - } + return 0; } SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { - SConfig *pConfig = cfgInit(); - if (pConfig == NULL) return NULL; + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; - if (dmnInitLogCfg(pConfig) != 0) { - uError("failed to init log cfg since %s", terrstr()); - cfgCleanup(pConfig); + if (dmnAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); return NULL; } - if (dmnInitDnodeCfg(pConfig) != 0) { + if (dmnAddDnodeCfg(pCfg) != 0) { uError("failed to init dnode cfg since %s", terrstr()); - cfgCleanup(pConfig); + cfgCleanup(pCfg); return NULL; } - if (dmnLoadCfg(pConfig, cfgDir, envFile, apolloUrl) != 0) { + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { uError("failed to load cfg since %s", terrstr()); - cfgCleanup(pConfig); + cfgCleanup(pCfg); return NULL; } - bool enableCore = cfgGetItem(pConfig, "enableCoreFile")->bval; - taosSetCoreDump(enableCore); + if (dmnCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + } if (taosCheckAndPrintCfg() != 0) { uError("failed to check config"); return NULL; } - - return pConfig; + return pCfg; } void dmnDumpCfg(SConfig *pCfg) { @@ -146,7 +127,7 @@ SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) { tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale)); tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset)); envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32; - envCfg.numOfCommitThreads = (uint16_t) cfgGetItem(pCfg, "numOfCommitThreads")->i32; + envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32; envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; return envCfg; diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c index 8cce1bb01e..3ec43cb7f3 100644 --- a/source/dnode/mgmt/daemon/src/dmnLog.c +++ b/source/dnode/mgmt/daemon/src/dmnLog.c @@ -16,25 +16,52 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" -int32_t dmnInitLogCfg(SConfig *pCfg) { +int32_t dmnAddLogCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "cqDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +int32_t dmnSetLogCfg(SConfig *pCfg) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + return 0; } @@ -42,22 +69,26 @@ int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUr SConfig *pCfg = cfgInit(); if (pCfg == NULL) return -1; - if (dmnInitLogCfg(pCfg) != 0) { - uError("failed to init log cfg since %s\n", terrstr()); + if (dmnAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); cfgCleanup(pCfg); return -1; } if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { - uError("failed to load log cfg since %s\n", terrstr()); + printf("failed to load log cfg since %s\n", terrstr()); cfgCleanup(pCfg); return -1; } - char temp[PATH_MAX] = {0}; - snprintf(temp, PATH_MAX, "%s" TD_DIRSEP "taosdlog", cfgGetItem(pCfg, "logDir")->str); - if (taosInitLog(temp, cfgGetItem(pCfg, "numOfLogLines")->i32, 1) != 0) { - uError("failed to init log file since %s\n", terrstr()); + if (dmnSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file since %s\n", terrstr()); cfgCleanup(pCfg); return -1; } @@ -65,3 +96,35 @@ int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUr cfgCleanup(pCfg); return 0; } + +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char configDir[PATH_MAX] = {0}; + char configFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, configDir, PATH_MAX); + snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { + uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c index aac27c4c33..608bc7f3e6 100644 --- a/source/dnode/mgmt/daemon/src/dmnUtil.c +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -22,7 +22,7 @@ void dmnGenerateGrant() { #endif } -void dmnPrintVersion(SConfig *pCfg) { +void dmnPrintVersion() { #ifdef TD_ENTERPRISE char *releaseName = "enterprise"; #else diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 09a738be3b..30b796290f 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,16 +28,12 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1c84f06a22..ebe20fbb7f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -131,9 +131,7 @@ void ctgTestInitLogFile() { ctgDbgEnableDebug("api"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index dbc4767aee..90e818dc1e 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -538,10 +538,7 @@ void cfgPrintCfg(SConfig *pCfg) { } int32_t cfgCheck(SConfig *pConfig) { - SConfigItem *pItem = cfgGetItem(pConfig, "debugFlag"); - if (pItem != NULL) { - taosSetDebugFlag(pItem->i32); - } + SConfigItem *pItem = NULL; pItem = cfgGetItem(pConfig, "localFqdn"); if (pItem != NULL) { diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 38890150ee..8ad5a76388 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -101,9 +101,7 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6bfff0197c..89d365a7e7 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -66,9 +66,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f1aadafacc..4842a0c800 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0bcc47383b..0ccfde16ce 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cc6a63d3cd..bcdf32bf6a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 65170d4abb..f9bcc3da96 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index d1a587f4e5..4337cd242c 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index c5d7f5664a..b7ef296b9d 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -170,7 +170,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 233b000c61..e17f221925 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -70,7 +70,7 @@ typedef struct { int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; -int8_t tsAsyncLog = 1; +bool tsAsyncLog = true; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 1.0f; @@ -79,20 +79,19 @@ int32_t writeInterval = DEFAULT_LOG_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; +int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t wDebugFlag = 135; +int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; -int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int64_t dbgEmptyW = 0; @@ -120,10 +119,13 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) { +int32_t taosInitLog(const char *logName, int maxFiles) { + char fullName[PATH_MAX] = {0}; + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; return 0; } @@ -774,11 +776,22 @@ void taosPrintOsInfo() { uInfo(" os machine: %s", info.machine); } -void taosSetDebugFlag(int32_t flag) { - if (!(debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP)) return; +void taosSetAllDebugFlag(int32_t flag) { + if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; - debugFlag = flag; - mDebugFlag = flag; dDebugFlag = flag; + vDebugFlag = flag; + mDebugFlag = flag; cDebugFlag = flag; + jniDebugFlag = flag; + uDebugFlag = flag; + rpcDebugFlag = flag; + qDebugFlag = flag; + wDebugFlag = flag; + sDebugFlag = flag; + tsdbDebugFlag = flag; + tqDebugFlag = flag; + fsDebugFlag = flag; + + uInfo("all debug flag are set to %d", flag); } \ No newline at end of file diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 1fdc2257d7..b989a3f56c 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -81,7 +81,6 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; uint32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 54aa6a5f34..1484ed2182 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 5000000, 10); + taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); From 0a0dfd5f7ab34d797094a50d8c11d2b9ae152bd3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 10:58:53 +0800 Subject: [PATCH 18/36] rename var --- include/libs/config/config.h | 34 +++--- source/dnode/mgmt/daemon/src/dmnCfg.c | 8 +- source/libs/config/src/config.c | 142 ++++++++++++++++---------- 3 files changed, 109 insertions(+), 75 deletions(-) diff --git a/include/libs/config/config.h b/include/libs/config/config.h index c3e026e20b..3c9ec1e3bf 100644 --- a/include/libs/config/config.h +++ b/include/libs/config/config.h @@ -73,28 +73,30 @@ typedef struct SConfigItem { typedef struct SConfig SConfig; SConfig *cfgInit(); -int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr); -void cfgCleanup(SConfig *pConfig); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +void cfgCleanup(SConfig *pCfg); -int32_t cfgGetSize(SConfig *pConfig); -SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter); -void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter); -SConfigItem *cfgGetItem(SConfig *pConfig, const char *name); +int32_t cfgGetSize(SConfig *pCfg); +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); -int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal); -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval); -int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal); -int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVa); -int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal); -int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal); -int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal); -int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); +void cfgDumpCfg(SConfig *pCfg); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 9d9082a76b..c77fcdb7d5 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -45,7 +45,7 @@ int32_t dmnCheckCfg(SConfig *pCfg) { bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetCoreDump(enableCore); - + return 0; } @@ -74,13 +74,11 @@ SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloU if (dmnCheckCfg(pCfg) != 0) { uError("failed to check cfg since %s", terrstr()); - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); + cfgCleanup(pCfg); return NULL; } + cfgDumpCfg(pCfg); return pCfg; } diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 90e818dc1e..0011a133e1 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -21,52 +21,52 @@ #include "ttimezone.h" SConfig *cfgInit() { - SConfig *pConfig = calloc(1, sizeof(SConfig)); - if (pConfig == NULL) { + SConfig *pCfg = calloc(1, sizeof(SConfig)); + if (pCfg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pConfig->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); - if (pConfig->hash == NULL) { - free(pConfig); + pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); + if (pCfg->hash == NULL) { + free(pCfg); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - return pConfig; + return pCfg; } -int32_t cfgLoad(SConfig *pConfig, ECfgSrcType cfgType, const char *sourceStr) { +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { switch (cfgType) { case CFG_STYPE_CFG_FILE: - return cfgLoadFromCfgFile(pConfig, sourceStr); + return cfgLoadFromCfgFile(pCfg, sourceStr); case CFG_STYPE_ENV_FILE: - return cfgLoadFromEnvFile(pConfig, sourceStr); + return cfgLoadFromEnvFile(pCfg, sourceStr); case CFG_STYPE_ENV_VAR: - return cfgLoadFromEnvVar(pConfig); + return cfgLoadFromEnvVar(pCfg); case CFG_STYPE_APOLLO_URL: - return cfgLoadFromApollUrl(pConfig, sourceStr); + return cfgLoadFromApollUrl(pCfg, sourceStr); default: return -1; } } -void cfgCleanup(SConfig *pConfig) { - if (pConfig != NULL) { - if (pConfig->hash != NULL) { - taosHashCleanup(pConfig->hash); - pConfig->hash == NULL; +void cfgCleanup(SConfig *pCfg) { + if (pCfg != NULL) { + if (pCfg->hash != NULL) { + taosHashCleanup(pCfg->hash); + pCfg->hash == NULL; } - free(pConfig); + free(pCfg); } } -int32_t cfgGetSize(SConfig *pConfig) { return taosHashGetSize(pConfig->hash); } +int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); } -SConfigItem *cfgIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashIterate(pConfig->hash, pIter); } +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); } -void cfgCancelIterate(SConfig *pConfig, SConfigItem *pIter) { return taosHashCancelIterate(pConfig->hash, pIter); } +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); } static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { tfree(pItem->str); @@ -292,8 +292,8 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType return 0; } -int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype) { - SConfigItem *pItem = cfgGetItem(pConfig, name); +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); if (pItem == NULL) { return -1; } @@ -328,12 +328,12 @@ int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSr return -1; } -SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); - SConfigItem *pItem = taosHashGet(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1); + SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1); if (pItem == NULL) { terrno = TSDB_CODE_CFG_NOT_FOUND; } @@ -341,7 +341,7 @@ SConfigItem *cfgGetItem(SConfig *pConfig, const char *name) { return pItem; } -static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name) { +static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; pItem->name = strdup(name); if (pItem->name == NULL) { @@ -353,7 +353,7 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); - if (taosHashPut(pConfig->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { + if (taosHashPut(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { if (pItem->dtype == CFG_DTYPE_STRING) { free(pItem->str); } @@ -365,94 +365,94 @@ static int32_t cfgAddItem(SConfig *pConfig, SConfigItem *pItem, const char *name return 0; } -int32_t cfgAddBool(SConfig *pConfig, const char *name, bool defaultVal) { +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal}; - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt32(SConfig *pConfig, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval}; - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt64(SConfig *pConfig, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval}; - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddFloat(SConfig *pConfig, const char *name, float defaultVal, double minval, double maxval) { +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval}; - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_STRING}; item.str = strdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddIpStr(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_DIR}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pConfig, const char *name, const char *defaultVal) { +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } - return cfgAddItem(pConfig, &item, name); + return cfgAddItem(pCfg, &item, name); } const char *cfgStypeStr(ECfgSrcType type) { @@ -537,20 +537,54 @@ void cfgPrintCfg(SConfig *pCfg) { } } -int32_t cfgCheck(SConfig *pConfig) { +void cfgDumpCfg(SConfig *pCfg) { + uInfo("global config"); + uInfo("=================================="); + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + uInfo("cfg:%s, value:%u src:%s", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT32: + uInfo("cfg:%s, value:%d src:%s", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT64: + uInfo("cfg:%s, value:%" PRId64 " src:%s", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_FLOAT: + uInfo("cfg:%s, value:%f src:%s", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uInfo("cfg:%s, value:%s src:%s", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + break; + } + pItem = cfgIterate(pCfg, pItem); + } + + uInfo("=================================="); +} + +int32_t cfgCheck(SConfig *pCfg) { SConfigItem *pItem = NULL; - pItem = cfgGetItem(pConfig, "localFqdn"); + pItem = cfgGetItem(pCfg, "localFqdn"); if (pItem != NULL) { tstrncpy(tsLocalFqdn, pItem->str, TSDB_FQDN_LEN); } - pItem = cfgGetItem(pConfig, "serverPort"); + pItem = cfgGetItem(pCfg, "serverPort"); if (pItem != NULL) { tsServerPort = (uint16_t)pItem->i32; } - pItem = cfgGetItem(pConfig, "firstEp"); + pItem = cfgGetItem(pCfg, "firstEp"); if (pItem != NULL) { tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); } @@ -566,7 +600,7 @@ int32_t cfgCheck(SConfig *pConfig) { snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); } - pItem = cfgGetItem(pConfig, "secondEp"); + pItem = cfgGetItem(pCfg, "secondEp"); if (pItem != NULL) { tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); } @@ -578,7 +612,7 @@ int32_t cfgCheck(SConfig *pConfig) { snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); } - pItem = cfgGetItem(pConfig, "dataDir"); + pItem = cfgGetItem(pCfg, "dataDir"); if (pItem != NULL) { tstrncpy(tsDataDir, pItem->str, PATH_MAX); } @@ -602,7 +636,7 @@ int32_t cfgCheck(SConfig *pConfig) { tsSetTimeZone(); // } - pItem = cfgGetItem(pConfig, "numOfCores"); + pItem = cfgGetItem(pCfg, "numOfCores"); if (pItem != NULL) { tsNumOfCores = pItem->i32; } @@ -615,7 +649,7 @@ int32_t cfgCheck(SConfig *pConfig) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } - cfgPrintCfg(pConfig); + cfgPrintCfg(pCfg); return 0; } From cefb07bc997fb2130f95a5e083c045dfbb449151 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 11:54:29 +0800 Subject: [PATCH 19/36] config for client --- include/common/tglobal.h | 5 - source/client/CMakeLists.txt | 2 +- source/client/inc/clientInt.h | 7 + source/client/src/clientCfg.c | 210 ++++++++++++++++++++++++++ source/client/src/clientEnv.c | 34 ++--- source/client/src/clientImpl.c | 4 +- source/client/src/clientMain.c | 2 +- source/client/test/CMakeLists.txt | 4 +- source/common/src/tep.c | 2 + source/common/src/tglobal.c | 57 ------- source/dnode/mgmt/daemon/src/dmnCfg.c | 31 +++- source/libs/config/src/config.c | 128 ++++++++-------- 12 files changed, 326 insertions(+), 160 deletions(-) create mode 100644 source/client/src/clientCfg.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4bc8c9fb86..94b2471ecc 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -24,11 +24,6 @@ extern "C" { #include "tcfg.h" // cluster -extern char tsFirst[]; -extern char tsSecond[]; -extern char tsLocalFqdn[]; -extern char tsLocalEp[]; -extern uint16_t tsServerPort; extern int32_t tsStatusInterval; extern int8_t tsEnableTelemetryReporting; extern int32_t tsNumOfSupportVnodes; diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 3210c0c6de..c793825556 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport parser planner catalog scheduler function qcom + PRIVATE os util common transport parser planner catalog scheduler function qcom config ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cd..dfff94eeb1 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -32,6 +32,8 @@ extern "C" { #include "query.h" #include "parser.h" +#include "config.h" + #define CHECK_CODE_GOTO(expr, label) \ do { \ int32_t code = expr; \ @@ -253,6 +255,11 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); +// config +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); + +extern SConfig *tscCfg; #ifdef __cplusplus } diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c new file mode 100644 index 0000000000..22eea6901e --- /dev/null +++ b/source/client/src/clientCfg.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "clientInt.h" +#include "ulog.h" + +// todo refact +SConfig *tscCfg; + +static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char configDir[PATH_MAX] = {0}; + char configFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, configDir, PATH_MAX); + snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { + uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} + +static int32_t tscAddLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +static int32_t tscSetLogCfg(SConfig *pCfg) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + return 0; +} + +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (tscAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + const int32_t maxLogFileNum = 10; + if (taosInitLog("taoslog", maxLogFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgCleanup(pCfg); + return 0; +} + +static int32_t tscAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + +static int32_t tscAddCfg(SConfig *pCfg) { + if (tscAddEpCfg(pCfg) != 0) return -1; + + + // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + // if (cfgAddString(pCfg, "version", version) != 0) return -1; + + // if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + // if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + return 0; +} + +int32_t tscCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + + + return 0; +} + +SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; + + if (tscAddLogCfg(pCfg) != 0) { + uError("failed to add log cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscAddCfg(pCfg) != 0) { + uError("failed to init tsc cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load tsc cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + cfgDumpCfg(pCfg); + return pCfg; +} + +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl); + if (tscCfg == NULL) return -1; + + return 0; +} \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 93fbdeb2ac..662a45a6f4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -72,22 +72,6 @@ static void deregisterRequest(SRequestObj* pRequest) { taosReleaseRef(clientConnRefPool, pTscObj->id); } -static void tscInitLogFile() { - #if 0 - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } - #endif -} - // todo close the transporter properly void closeTransporter(STscObj* pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { @@ -224,11 +208,17 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); -#if 0 - taosInitGlobalCfg(); - taosReadCfgFromFile(); -#endif - tscInitLogFile(); + + if (tscInitLog(configDir, NULL, NULL) != 0) { + tscInitRes = -1; + return; + } + + if (tscInitCfg(configDir, NULL, NULL) != 0) { + tscInitRes = -1; + return; + } + if (taosCheckAndPrintCfg()) { tscInitRes = -1; return; @@ -245,7 +235,7 @@ void taos_init_imp(void) { SSchedulerCfg scfg = {.maxJobNum = 100}; schedulerInit(&scfg); - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c39..dfbd725753 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -90,7 +90,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, epSet.epSet.eps[0].port = port; } } else { - if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); + SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); + if (initEpSetFromCfg(pFirst->str, pSecond->str, &epSet) < 0) { return NULL; } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b29813be03..35532f80ea 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -56,7 +56,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : tsServerPort; + int32_t p = (port != 0) ? port : cfgGetItem(tscCfg, "serverPort")->i32; tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); if (user == NULL) { diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index ee5109860e..7f6ce81788 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) ADD_EXECUTABLE(tmqTest tmqTest.cpp) TARGET_LINK_LIBRARIES( tmqTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f..643a313ec2 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -14,10 +14,12 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { pEp->port = atoi(temp+1); } +#if 0 if (pEp->port == 0) { pEp->port = tsServerPort; return -1; } +#endif return 0; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 655d62a148..8fd3802441 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -28,11 +28,6 @@ #include "ulog.h" // cluster -char tsFirst[TSDB_EP_LEN] = {0}; -char tsSecond[TSDB_EP_LEN] = {0}; -char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; -char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port -uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second int8_t tsEnableTelemetryReporting = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; @@ -293,38 +288,6 @@ static void doInitGlobalConfig(void) { #if 0 SGlobalCfg cfg = {0}; - // ip address - cfg.option = "firstEp"; - cfg.ptr = tsFirst; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "secondEp"; - cfg.ptr = tsSecond; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fqdn"; - cfg.ptr = tsLocalFqdn; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; @@ -723,26 +686,6 @@ int32_t taosCheckAndPrintCfg() { taosSetAllDebugFlag(); } - if (tsLocalFqdn[0] == 0) { - taosGetFqdn(tsLocalFqdn); - } - - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); - - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); - } - - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); - } taosCheckDataDirCfg(); diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index c77fcdb7d5..5c83f9fecd 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -16,14 +16,35 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" +static int32_t dmnAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + static int32_t dmnAddDnodeCfg(SConfig *pCfg) { + if (dmnAddEpCfg(pCfg) != 0) return -1; + + if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; if (cfgAddString(pCfg, "version", version) != 0) return -1; - if (cfgAddString(pCfg, "firstEp", "") != 0) return -1; - if (cfgAddString(pCfg, "secondEp", "") != 0) return -1; - if (cfgAddString(pCfg, "fqdn", "") != 0) return -1; - if (cfgAddInt32(pCfg, "serverPort", 6030, 1, 65056) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; @@ -55,7 +76,7 @@ SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloU if (pCfg == NULL) return NULL; if (dmnAddLogCfg(pCfg) != 0) { - printf("failed to add log cfg since %s\n", terrstr()); + uError("failed to add log cfg since %s", terrstr()); cfgCleanup(pCfg); return NULL; } diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 0011a133e1..25b65a0d1b 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -570,86 +570,82 @@ void cfgDumpCfg(SConfig *pCfg) { uInfo("=================================="); } +#if 0 +// int32_t cfgCheck(SConfig *pCfg) { +// SConfigItem *pItem = NULL; -int32_t cfgCheck(SConfig *pCfg) { - SConfigItem *pItem = NULL; +// pItem = cfgGetItem(pCfg, "serverPort"); +// if (pItem != NULL) { +// tsServerPort = (uint16_t)pItem->i32; +// } - pItem = cfgGetItem(pCfg, "localFqdn"); - if (pItem != NULL) { - tstrncpy(tsLocalFqdn, pItem->str, TSDB_FQDN_LEN); - } +// pItem = cfgGetItem(pCfg, "firstEp"); +// if (pItem != NULL) { +// tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); +// } - pItem = cfgGetItem(pCfg, "serverPort"); - if (pItem != NULL) { - tsServerPort = (uint16_t)pItem->i32; - } +// snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); +// uInfo("localEp is: %s", tsLocalEp); - pItem = cfgGetItem(pCfg, "firstEp"); - if (pItem != NULL) { - tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); - } +// SEp ep = {0}; +// if (tsFirst[0] == 0) { +// strcpy(tsFirst, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsFirst, &ep); +// snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } - snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); +// pItem = cfgGetItem(pCfg, "secondEp"); +// if (pItem != NULL) { +// tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); +// } - SEp ep = {0}; - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); - } +// if (tsSecond[0] == 0) { +// strcpy(tsSecond, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsSecond, &ep); +// snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } - pItem = cfgGetItem(pCfg, "secondEp"); - if (pItem != NULL) { - tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); - } +// pItem = cfgGetItem(pCfg, "dataDir"); +// if (pItem != NULL) { +// tstrncpy(tsDataDir, pItem->str, PATH_MAX); +// } - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); - } +// if (tsDiskCfgNum <= 0) { +// taosAddDataDir(0, tsDataDir, 0, 1); +// tsDiskCfgNum = 1; +// uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); +// } - pItem = cfgGetItem(pCfg, "dataDir"); - if (pItem != NULL) { - tstrncpy(tsDataDir, pItem->str, PATH_MAX); - } +// if (taosDirExist(tsTempDir) != 0) { +// return -1; +// } - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); - } +// taosGetSystemInfo(); - if (taosDirExist(tsTempDir) != 0) { - return -1; - } +// tsSetLocale(); - taosGetSystemInfo(); +// // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); +// // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { +// tsSetTimeZone(); +// // } - tsSetLocale(); +// pItem = cfgGetItem(pCfg, "numOfCores"); +// if (pItem != NULL) { +// tsNumOfCores = pItem->i32; +// } - // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); - // } +// if (tsNumOfCores <= 0) { +// tsNumOfCores = 1; +// } - pItem = cfgGetItem(pCfg, "numOfCores"); - if (pItem != NULL) { - tsNumOfCores = pItem->i32; - } +// if (tsQueryBufferSize >= 0) { +// tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; +// } - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; - } +// cfgPrintCfg(pCfg); - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } - - cfgPrintCfg(pCfg); - - return 0; -} +// return 0; +// } +#endif \ No newline at end of file From d9bc65e5be1a4d2a8c90b5038fce2cd047310990 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 12:03:25 +0800 Subject: [PATCH 20/36] config --- source/client/src/clientEnv.c | 5 --- source/common/src/tglobal.c | 47 +---------------------- source/dnode/mgmt/daemon/src/dmnCfg.c | 55 ++++++++++++++++++++++++--- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 662a45a6f4..c7640e90d8 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -219,11 +219,6 @@ void taos_init_imp(void) { return; } - if (taosCheckAndPrintCfg()) { - tscInitRes = -1; - return; - } - taosInitNotes(); initMsgHandleFp(); initQueryModuleMsgHandle(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8fd3802441..9129ccdcae 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -274,13 +274,7 @@ void taosPrintDataDirCfg() { } #endif -static void taosCheckDataDirCfg() { - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); - } -} + static void doInitGlobalConfig(void) { osInit(); @@ -678,45 +672,6 @@ static void doInitGlobalConfig(void) { void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckAndPrintCfg() { -#if 0 - - SEp ep = {0}; - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); - } - - - taosCheckDataDirCfg(); - - if (taosDirExist(tsTempDir) != 0) { - return -1; - } - - taosGetSystemInfo(); - - tsSetLocale(); - - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); - } - - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; - } - - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } - - uInfo(" check global cfg completed"); - uInfo("=================================="); - taosPrintCfg(); -#endif - return 0; -} - /* * alter dnode 1 balance "vnode:1-dnode:2" */ diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 5c83f9fecd..5f9a381273 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -15,6 +15,8 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" +#include "tlocale.h" +#include "ttimezone.h" static int32_t dmnAddEpCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; @@ -23,7 +25,7 @@ static int32_t dmnAddEpCfg(SConfig *pCfg) { return -1; } if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; - + int32_t defaultServerPort = 6030; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; @@ -37,15 +39,41 @@ static int32_t dmnAddEpCfg(SConfig *pCfg) { return 0; } -static int32_t dmnAddDnodeCfg(SConfig *pCfg) { - if (dmnAddEpCfg(pCfg) != 0) return -1; +static int32_t dmnAddDirCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + if (cfgAddDir(pCfg, "tmpDir", tsTempDir) != 0) return -1; + return 0; +} +static int32_t dmnCheckDirCfg(SConfig *pCfg) { + SConfigItem *pItem = NULL; + pItem = cfgGetItem(pCfg, "dataDir"); + if (tsDiskCfgNum <= 0) { + taosAddDataDir(0, pItem->str, 0, 1); + tsDiskCfgNum = 1; + uTrace("dataDir:%s, level:0 primary:1 is configured by default", pItem->str); + } + pItem = cfgGetItem(pCfg, "tmpDir"); + if (taosDirExist(pItem->str) != 0) { + return -1; + } + + return 0; +} + +static int32_t dmnAddVersionCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; if (cfgAddString(pCfg, "version", version) != 0) return -1; - - if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + return 0; +} + +static int32_t dmnAddDnodeCfg(SConfig *pCfg) { + if (dmnAddEpCfg(pCfg) != 0) return -1; + if (dmnAddDirCfg(pCfg) != 0) return -1; + if (dmnAddVersionCfg(pCfg) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; @@ -62,11 +90,26 @@ static int32_t dmnAddDnodeCfg(SConfig *pCfg) { return 0; } -int32_t dmnCheckCfg(SConfig *pCfg) { +static int32_t dmnCheckCfg(SConfig *pCfg) { bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetCoreDump(enableCore); + if (dmnCheckDirCfg(pCfg) != 0) { + return -1; + } + taosGetSystemInfo(); + + tsSetTimeZone(); + tsSetLocale(); + + if (tsNumOfCores <= 0) { + tsNumOfCores = 1; + } + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } return 0; } From 42c100df2d668be0713be589abfb7d8ed2a48633 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 15:02:17 +0800 Subject: [PATCH 21/36] sim config --- include/common/tep.h | 2 +- include/common/tglobal.h | 3 - source/client/src/clientCfg.c | 14 ++--- source/client/src/clientImpl.c | 62 +++++++++---------- source/client/src/clientMain.c | 6 +- source/common/src/tep.c | 6 +- source/common/src/tglobal.c | 30 +++++---- source/dnode/mgmt/daemon/src/dmnCfg.c | 15 ++--- .../mgmt/daemon/src/{daemon.c => dmnMain.c} | 0 source/dnode/mgmt/impl/src/dndMgmt.c | 2 +- source/util/src/tlog.c | 4 ++ 11 files changed, 63 insertions(+), 81 deletions(-) rename source/dnode/mgmt/daemon/src/{daemon.c => dmnMain.c} (100%) diff --git a/include/common/tep.h b/include/common/tep.h index 584b8a5a71..6ca180667b 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo { // bool hasNull; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 94b2471ecc..7458e5d837 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -100,13 +100,10 @@ extern uint32_t tsMaxRange; extern uint32_t tsCurRange; extern char tsCompressor[]; -extern int32_t tsDiskCfgNum; -extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c index 22eea6901e..352495178b 100644 --- a/source/client/src/clientCfg.c +++ b/source/client/src/clientCfg.c @@ -21,20 +21,20 @@ SConfig *tscCfg; static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { - char configDir[PATH_MAX] = {0}; - char configFile[PATH_MAX + 100] = {0}; + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; - taosExpandDir(inputCfgDir, configDir, PATH_MAX); - snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); return -1; } - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { - uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); return -1; } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dfbd725753..c98a3695d9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -12,7 +12,7 @@ #include "tpagedbuf.h" #include "tref.h" -static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); +static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); @@ -81,21 +81,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } SCorEpSet epSet = {0}; - if (ip) { - if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { - return NULL; - } - - if (port) { - epSet.epSet.eps[0].port = port; - } - } else { - SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); - SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); - if (initEpSetFromCfg(pFirst->str, pSecond->str, &epSet) < 0) { - return NULL; - } - } + initEpSetFromCfg(ip, port, &epSet); char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; @@ -284,32 +270,40 @@ _return: return pRequest; } -int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { - pEpSet->version = 0; +int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) { + SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); + SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); + SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort"); // init mnode ip set SEpSet* mgmtEpSet = &(pEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; + pEpSet->version = 0; - if (firstEp && firstEp[0] != 0) { - if (strlen(firstEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (ip != NULL) { + taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; - } - - if (secondEp && secondEp[0] != 0) { - if (strlen(secondEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; + if (port) { + mgmtEpSet->eps[0].port = port; + } + } else { + if (pFirst->str[0] != 0) { + if (strlen(pFirst->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); + mgmtEpSet->numOfEps++; + } + if (pSecond->str[0] != 0) { + if (strlen(pSecond->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]); + mgmtEpSet->numOfEps++; } - - taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; } if (mgmtEpSet->numOfEps == 0) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 35532f80ea..00d629ebb2 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -56,9 +56,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : cfgGetItem(tscCfg, "serverPort")->i32; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { user = TSDB_DEFAULT_USER; } @@ -67,7 +65,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, port); } void taos_close(TAOS* taos) { diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 643a313ec2..3e98cae279 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -4,7 +4,7 @@ #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -14,12 +14,10 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { pEp->port = atoi(temp+1); } -#if 0 if (pEp->port == 0) { - pEp->port = tsServerPort; + pEp->port = defaultPort; return -1; } -#endif return 0; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9129ccdcae..fc25dac0df 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -125,8 +125,6 @@ int8_t tsDeadLockKillQuery = 0; // For backward compatibility bool tsdbForceKeepFile = false; -int32_t tsDiskCfgNum = 0; - #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; #else @@ -257,21 +255,21 @@ void taosAddDataDir(int index, char *v1, int level, int primary) { } #ifndef _STORAGE -void taosReadDataDirCfg(char *v1, char *v2, char *v3) { - if (tsDiskCfgNum == 1) { - SDiskCfg *cfg = &tsDiskCfg[0]; - uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); - } - taosAddDataDir(0, v1, 0, 1); - tsDiskCfgNum = 1; -} +// void taosReadDataDirCfg(char *v1, char *v2, char *v3) { +// if (tsDiskCfgNum == 1) { +// SDiskCfg *cfg = &tsDiskCfg[0]; +// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); +// } +// taosAddDataDir(0, v1, 0, 1); +// tsDiskCfgNum = 1; +// } -void taosPrintDataDirCfg() { - for (int i = 0; i < tsDiskCfgNum; ++i) { - SDiskCfg *cfg = &tsDiskCfg[i]; - uInfo(" dataDir: %s", cfg->dir); - } -} +// void taosPrintDataDirCfg() { +// for (int i = 0; i < tsDiskCfgNum; ++i) { +// SDiskCfg *cfg = &tsDiskCfg[i]; +// uInfo(" dataDir: %s", cfg->dir); +// } +// } #endif diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 5f9a381273..1f68df6fe4 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -47,13 +47,6 @@ static int32_t dmnAddDirCfg(SConfig *pCfg) { static int32_t dmnCheckDirCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; - pItem = cfgGetItem(pCfg, "dataDir"); - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, pItem->str, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", pItem->str); - } - pItem = cfgGetItem(pCfg, "tmpDir"); if (taosDirExist(pItem->str) != 0) { return -1; @@ -204,12 +197,12 @@ SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; - objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.localEp, cfgGetItem(pCfg, "localEp")->str, sizeof(objCfg.localEp)); - tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "localFqdn")->str, sizeof(objCfg.localFqdn, cfgGetItem)); + tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); - + objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn, cfgGetItem)); + snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); return objCfg; } \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/dmnMain.c similarity index 100% rename from source/dnode/mgmt/daemon/src/daemon.c rename to source/dnode/mgmt/daemon/src/dmnMain.c diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 3d149a3e60..1748c52c66 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -296,7 +296,7 @@ PRASE_DNODE_OVER: if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep); taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e17f221925..ecd0f0437b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -71,6 +71,7 @@ int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; bool tsAsyncLog = true; +bool tsLogInited = false; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 1.0f; @@ -120,6 +121,8 @@ static int32_t taosStartLog() { } int32_t taosInitLog(const char *logName, int maxFiles) { + if (tsLogInited) return 0; + char fullName[PATH_MAX] = {0}; snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); @@ -127,6 +130,7 @@ int32_t taosInitLog(const char *logName, int maxFiles) { if (tsLogObj.logHandle == NULL) return -1; if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; + tsLogInited = true; return 0; } From 48283e42e91831015b63f2633801b2cf0dc70371 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 23 Feb 2022 16:04:06 +0800 Subject: [PATCH 22/36] new offset management --- example/src/tmq.c | 13 +- include/client/taos.h | 5 +- include/common/common.h | 45 +- include/common/tmsg.h | 83 ++-- include/common/tmsgdef.h | 2 +- include/dnode/mnode/sdb/sdb.h | 17 +- include/util/taoserror.h | 3 + include/util/tdef.h | 521 ++++++++++----------- source/client/src/tmq.c | 181 ++++--- source/common/src/tmsg.c | 92 +--- source/dnode/mgmt/impl/src/dndTransport.c | 11 +- source/dnode/mnode/impl/inc/mndDef.h | 45 +- source/dnode/mnode/impl/inc/mndOffset.h | 44 ++ source/dnode/mnode/impl/src/mndOffset.c | 224 +++++++++ source/dnode/mnode/impl/src/mndSubscribe.c | 19 +- source/dnode/mnode/impl/src/mnode.c | 8 +- source/dnode/vnode/src/tq/tq.c | 34 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 5 +- source/libs/executor/src/executor.c | 2 +- 19 files changed, 824 insertions(+), 530 deletions(-) create mode 100644 source/dnode/mnode/impl/inc/mndOffset.h create mode 100644 source/dnode/mnode/impl/src/mndOffset.c diff --git a/example/src/tmq.c b/example/src/tmq.c index 26e5ea82c1..094fd94bfc 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -66,7 +66,7 @@ int32_t init_env() { } int32_t create_topic() { - printf("create topic"); + printf("create topic\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -91,6 +91,10 @@ int32_t create_topic() { return 0; } +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d\n", resp); +} + tmq_t* build_consumer() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -103,6 +107,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; } @@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; + static const int MIN_COMMIT_COUNT = 1; int msg_count = 0; tmq_resp_err_t err; @@ -214,6 +219,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); } diff --git a/include/client/taos.h b/include/client/taos.h index a960d37937..2c8135c8ff 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ enum tmq_resp_err_t { + TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, - TMQ_RESP_ERR__FAIL = 1, }; typedef enum tmq_resp_err_t tmq_resp_err_t; @@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); @@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t * #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif +DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { diff --git a/include/common/common.h b/include/common/common.h index fd5b6717ab..8fa2d03d6d 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,7 +16,6 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H - #ifdef __cplusplus extern "C" { #endif @@ -43,14 +42,16 @@ extern "C" { // int16_t bytes; // } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +enum { + TMQ_CONF__RESET_OFFSET__LATEST = -1, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__NONE = -3, +}; typedef struct { uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray* pGroupList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; typedef struct SColumnDataAgg { @@ -79,14 +80,14 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray + SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { - int32_t *offset; // start position for each entry in the list + int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data uint32_t allocLen; // allocated buffer size } SVarColAttr; @@ -94,11 +95,11 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - bool hasNull;// if current column data has null value. - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull; // if current column data has null value. + char* pData; // the corresponding block data in memory union { - char *nullbitmap; // bitmap, one bit for each item in the list + char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; } SColumnInfoData; @@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); @@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); @@ -250,11 +249,11 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn* pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { @@ -271,7 +270,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5164fb6ccd..d92efd163d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -422,8 +422,8 @@ typedef struct { } SColumnInfo; typedef struct { - uint64_t uid; - TSKEY key; // last accessed ts, for subscription + int64_t uid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; } SDropDbRsp; int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); @@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SArray* pVgroupInfos; // Array of SVgroupInfo + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); @@ -725,7 +725,7 @@ typedef struct { int32_t vgId; int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; - uint64_t dbUid; + int64_t dbUid; int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; @@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); typedef struct { - int32_t vgId; - int32_t dnodeId; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + int32_t dnodeId; + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); @@ -796,7 +796,7 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - uint64_t dbId; + int64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -804,8 +804,8 @@ typedef struct { int8_t update; int32_t sversion; int32_t tversion; - uint64_t suid; - uint64_t tuid; + int64_t suid; + int64_t tuid; int32_t vgId; SSchema* pSchemas; } STableMetaRsp; @@ -1268,7 +1268,7 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; int32_t sverson; int32_t execLen; char* executor; @@ -1279,11 +1279,11 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; } SDDropTopicReq; typedef struct SVCreateTbReq { - uint64_t ver; // use a general definition + int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; @@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - uint64_t ver; // use a general definition - SArray* pArray; + int64_t ver; // use a general definition + SArray* pArray; } SVCreateTbBatchReq; typedef struct { @@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - uint64_t ver; + int64_t ver; char* name; uint8_t type; tb_uid_t suid; @@ -1760,35 +1760,19 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqOffset; -typedef struct { - int32_t vgId; - SArray* offsets; // SArray -} SMqVgOffsets; - typedef struct { int32_t num; SMqOffset* offsets; -} SMqCMResetOffsetReq; +} SMqCMCommitOffsetReq; typedef struct { int32_t reserved; -} SMqCMResetOffsetRsp; - -typedef struct { - int64_t leftForVer; - SMqVgOffsets offsets; -} SMqMVResetOffsetReq; - -typedef struct { - int32_t reserved; -} SMqMVResetOffsetRsp; +} SMqCMCommitOffsetRsp; int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset); int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); -int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq); -int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq); -int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq); -int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq); +int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); +int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); typedef struct { uint32_t nCols; @@ -1870,7 +1854,6 @@ typedef struct { typedef struct { int64_t consumerId; SSchemaWrapper* schemas; - int64_t committedOffset; int64_t reqOffset; int64_t rspOffset; int32_t skipLogNum; @@ -1881,22 +1864,18 @@ typedef struct { // one req for one vg+topic typedef struct { SMsgHead head; - // 0: commit only, current offset - // 1: consume only, poll next offset - // 2: commit current and consume next offset - int32_t reqType; - int64_t reqId; int64_t consumerId; int64_t blockingTime; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; + int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { int32_t vgId; + int64_t offset; SEpSet epSet; } SMqSubVgEp; @@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return buf; } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1a63ea73a5..b20d6de556 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -143,10 +143,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 1e967a6d2b..bf48a8523c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,14 +113,15 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_SUBSCRIBE = 10, - SDB_CONSUMER = 11, - SDB_TOPIC = 12, - SDB_VGROUP = 13, - SDB_STB = 14, - SDB_DB = 15, - SDB_FUNC = 16, - SDB_MAX = 17 + SDB_OFFSET = 10, + SDB_SUBSCRIBE = 11, + SDB_CONSUMER = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 92028b0837..b947142958 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -20,6 +20,8 @@ extern "C" { #endif +// clang-format off + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -260,6 +262,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode diff --git a/include/util/tdef.h b/include/util/tdef.h index a4c333a4d2..407d0ccd9e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -24,373 +24,370 @@ extern "C" { #define TSDB__packed -#define TSKEY int64_t -#define TSKEY_MIN INT64_MIN -#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY int64_t +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; // TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) -#define TSDB_KEYSIZE sizeof(TSKEY) -#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) // NULL definition -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF -#define TSDB_DATA_UTINYINT_NULL 0xFF -#define TSDB_DATA_USMALLINT_NULL 0xFFFF -#define TSDB_DATA_UINT_NULL 0xFFFFFFFF -#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" -#define TSDB_NETTEST_USER "nettestinternal" -#define TSDB_DEFAULT_USER "root" +#define TSDB_NETTEST_USER "nettestinternal" +#define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ -#define TSDB_DEFAULT_PASS "powerdb" +#define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) -#define TSDB_DEFAULT_PASS "tqueue" +#define TSDB_DEFAULT_PASS "tqueue" #elif (_TD_PRO_ == true) -#define TSDB_DEFAULT_PASS "prodb" +#define TSDB_DEFAULT_PASS "prodb" #else -#define TSDB_DEFAULT_PASS "taosdata" +#define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 +#define SHELL_MAX_PASSWORD_LEN 20 -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 #define TSDB_ERR -1 #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' -#define TSDB_TIME_PRECISION_MILLI 0 -#define TSDB_TIME_PRECISION_MICRO 1 -#define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" -#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) +#define TSDB_TICK_PER_SECOND(precision) \ + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ -do {\ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ -} while(0) -#define T_READ_MEMBER(src, type, target) \ -do { \ - (target) = *(type *)(src); \ - (src) = (void *)((char *)src + sizeof(type));\ -} while(0) - +#define T_APPEND_MEMBER(dst, ptr, type, member) \ + do { \ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ + } while (0) +#define T_READ_MEMBER(src, type, target) \ + do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type)); \ + } while (0) // TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 +#define TSDB_RELATION_INVALID 0 +#define TSDB_RELATION_LESS 1 +#define TSDB_RELATION_GREATER 2 +#define TSDB_RELATION_EQUAL 3 +#define TSDB_RELATION_LESS_EQUAL 4 #define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_NOT_LIKE 8 -#define TSDB_RELATION_ISNULL 9 -#define TSDB_RELATION_NOTNULL 10 -#define TSDB_RELATION_IN 11 -#define TSDB_RELATION_NOT_IN 12 +#define TSDB_RELATION_NOT_EQUAL 6 +#define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_NOT_LIKE 8 +#define TSDB_RELATION_ISNULL 9 +#define TSDB_RELATION_NOTNULL 10 +#define TSDB_RELATION_IN 11 +#define TSDB_RELATION_NOT_IN 12 -#define TSDB_RELATION_AND 13 -#define TSDB_RELATION_OR 14 -#define TSDB_RELATION_NOT 15 +#define TSDB_RELATION_AND 13 +#define TSDB_RELATION_OR 14 +#define TSDB_RELATION_NOT 15 -#define TSDB_RELATION_MATCH 16 -#define TSDB_RELATION_NMATCH 17 +#define TSDB_RELATION_MATCH 16 +#define TSDB_RELATION_NMATCH 17 -#define TSDB_BINARY_OP_ADD 4000 -#define TSDB_BINARY_OP_SUBTRACT 4001 -#define TSDB_BINARY_OP_MULTIPLY 4002 -#define TSDB_BINARY_OP_DIVIDE 4003 -#define TSDB_BINARY_OP_REMAINDER 4004 -#define TSDB_BINARY_OP_CONCAT 4005 +#define TSDB_BINARY_OP_ADD 4000 +#define TSDB_BINARY_OP_SUBTRACT 4001 +#define TSDB_BINARY_OP_MULTIPLY 4002 +#define TSDB_BINARY_OP_DIVIDE 4003 +#define TSDB_BINARY_OP_REMAINDER 4004 +#define TSDB_BINARY_OP_CONCAT 4005 -#define FUNCTION_CEIL 4500 -#define FUNCTION_FLOOR 4501 -#define FUNCTION_ABS 4502 -#define FUNCTION_ROUND 4503 +#define FUNCTION_CEIL 4500 +#define FUNCTION_FLOOR 4501 +#define FUNCTION_ABS 4502 +#define FUNCTION_ROUND 4503 -#define FUNCTION_LENGTH 4800 -#define FUNCTION_CONCAT 4801 -#define FUNCTION_LTRIM 4802 -#define FUNCTION_RTRIM 4803 +#define FUNCTION_LENGTH 4800 +#define FUNCTION_CONCAT 4801 +#define FUNCTION_LTRIM 4802 +#define FUNCTION_RTRIM 4803 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) -#define TSDB_NAME_DELIMITER_LEN 1 +#define TSDB_NAME_DELIMITER_LEN 1 -#define TSDB_UNI_LEN 24 -#define TSDB_USER_LEN TSDB_UNI_LEN +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 +#define TSDB_ACCT_ID_LEN 11 -#define TSDB_MAX_COLUMNS 4096 -#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_COMMENT_LEN 4096 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 -#define TSDB_FUNC_TYPE_SCALAR 1 -#define TSDB_FUNC_TYPE_AGGREGATE 2 -#define TSDB_FUNC_MAX_RETRIEVE 1024 +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_TYPE_STR_MAX_LEN 32 -#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN -#define TSDB_CONSUMER_GROUP_LEN 192 -#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_COL_NAME_LEN 65 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 1024 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_CONSUMER_GROUP_LEN 193 +#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 1024 +#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 - /** - * In some scenarios uint16_t (0~65535) is used to store the row len. - * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. - * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-(4096-1)*4 = 49151. - */ -#define TSDB_MAX_BYTES_PER_ROW 49151 -#define TSDB_MAX_TAGS_LEN 16384 -#define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_STB_COMMENT_LEN 1024 +/** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 -#define TSDB_AUTH_LEN 16 -#define TSDB_PASSWORD_LEN 32 -#define TSDB_USET_PASSWORD_LEN 129 -#define TSDB_VERSION_LEN 12 -#define TSDB_LABEL_LEN 8 +#define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 +#define TSDB_VERSION_LEN 12 +#define TSDB_LABEL_LEN 8 -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 -#define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 -#define TSDB_TRANS_STAGE_LEN 12 -#define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 64 +#define TSDB_TRANS_STAGE_LEN 12 +#define TSDB_TRANS_TYPE_LEN 16 +#define TSDB_TRANS_ERROR_LEN 64 -#define TSDB_STEP_NAME_LEN 32 -#define TSDB_STEP_DESC_LEN 128 +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 -#define TSDB_ERROR_MSG_LEN 1024 -#define TSDB_DNODE_CONFIG_LEN 128 -#define TSDB_DNODE_VALUE_LEN 256 +#define TSDB_ERROR_MSG_LEN 1024 +#define TSDB_DNODE_CONFIG_LEN 128 +#define TSDB_DNODE_VALUE_LEN 256 -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 -#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE +#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE -#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 -#define TSDB_DEFAULT_VN_PER_DB 2 +#define TSDB_DEFAULT_VN_PER_DB 2 -#define TSDB_DNODE_ROLE_ANY 0 -#define TSDB_DNODE_ROLE_MGMT 1 -#define TSDB_DNODE_ROLE_VNODE 2 +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) -#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 3650 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_KEEP 1 // data in db to be reserved. -#define TSDB_MAX_KEEP 365000 // data in db to be reserved. -#define TSDB_DEFAULT_KEEP 3650 // ten years +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MAX_FIELD_LEN 16384 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define PRIMARYKEY_TIMESTAMP_COL_ID 1 -#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_ID 1 +#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_MAX_RPC_THREADS 5 #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default - - +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u -#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u -#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file -#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 -#define TSDB_MAX_WAL_SIZE (1024*1024*3) +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 -#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P +#define TSDB_MAX_WAL_SIZE (1024 * 1024 * 3) -#define TFS_MAX_TIERS 3 +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +#define TFS_MAX_TIERS 3 #define TFS_MAX_DISKS_PER_TIER 16 -#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 +#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 229d3a9ec3..e206b1a2bb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -45,22 +45,24 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - bool auto_commit; + char clientId[256]; + char groupId[256]; + int8_t auto_commit; + int8_t resetOffset; + tmq_commit_cb* commit_cb; /*char* ip;*/ /*uint16_t port;*/ - tmq_commit_cb* commit_cb; }; struct tmq_t { // conf char groupId[256]; char clientId[256]; - bool autoCommit; + int8_t autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; + int32_t resetOffsetCfg; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -79,7 +81,6 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -115,21 +116,17 @@ typedef struct { } SMqConsumeCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; -} SMqCommitCbParam; - -typedef struct { - tmq_t* tmq; + tmq_t* tmq; + /*SMqClientVg* pVg;*/ + int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; -} SMqResetOffsetParam; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } @@ -157,6 +154,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { + if (strcmp(value, "none") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + return TMQ_CONF_OK; + } else if (strcmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + return TMQ_CONF_OK; + } else if (strcmp(value, "latest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } return TMQ_CONF_UNKNOWN; } @@ -190,14 +201,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) tsem_post(&pParam->rspSem); - return 0; -} - -int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param; - pParam->rspErr = code; - tsem_post(&pParam->rspSem); + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + free(param); + } return 0; } @@ -216,6 +225,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); @@ -223,18 +233,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - SRequestObj* pRequest = NULL; +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { + // TODO: add read write lock + SRequestObj* pRequest = NULL; + tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS; // build msg // send to mnode - SMqCMResetOffsetReq req; - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + SMqCMCommitOffsetReq req; + SArray* pArray = NULL; + + if (offsets == NULL) { + pArray = taosArrayInit(0, sizeof(SMqOffset)); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqOffset offset; + strcpy(offset.topicName, pTopic->topicName); + strcpy(offset.cgroup, tmq->groupId); + offset.vgId = pVg->vgId; + offset.offset = pVg->currentOffset; + taosArrayPush(pArray, &offset); + } + } + req.num = pArray->size; + req.offsets = pArray->pData; + } else { + req.num = offsets->cnt; + req.offsets = (SMqOffset*)offsets->elems; + } SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; void* buf = malloc(tlen); if (buf == NULL) { @@ -244,32 +276,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse tCoderClear(&encoder); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); tCoderClear(&encoder); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET); if (pRequest == NULL) { tscError("failed to malloc request"); } - SMqResetOffsetParam param = {0}; - tsem_init(¶m.rspSem, 0, 0); - param.tmq = tmq; + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + return -1; + } + pParam->tmq = tmq; + tsem_init(&pParam->rspSem, 0, 0); pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->param = ¶m; - sendInfo->fp = tmqResetOffsetCb; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + if (!async) { + tsem_wait(&pParam->rspSem); + resp = pParam->rspErr; + } - return param.rspErr; + if (pArray) { + taosArrayDestroy(pArray); + } + + return resp; } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { @@ -641,8 +682,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // clang-format off SMqClientVg clientVg = { .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, + .currentOffset = pVgEp->offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; @@ -708,23 +748,51 @@ END: return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, - SMqClientVg* pVg) { +tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { + const SMqOffset* pOffset = &offset->offset; + if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { + return TMQ_RESP_ERR__FAIL; + } + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { + int32_t vgSz = taosArrayGetSize(clientTopic->vgs); + for (int32_t j = 0; j < vgSz; j++) { + SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); + if (pVg->vgId == pOffset->vgId) { + pVg->currentOffset = pOffset->offset; + return TMQ_RESP_ERR__SUCCESS; + } + } + } + } + return TMQ_RESP_ERR__FAIL; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { + int64_t reqOffset; + if (pVg->currentOffset >= 0) { + reqOffset = pVg->currentOffset; + } else { + if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { + tscError("unable to poll since no committed offset but reset offset is set to none"); + return NULL; + } + reqOffset = tmq->resetOffsetCfg; + } + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } - pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { - pReq->offset = pVg->currentOffset; - } else { - pReq->offset = pVg->currentOffset + 1; - } + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +811,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - printf("over1\n"); + /*printf("over1\n");*/ usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { - printf("over2\n"); + /*printf("over2\n");*/ usleep(blocking_time * 1000); return NULL; } @@ -760,8 +828,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -821,6 +888,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { if (tmq_topic_vgroup_list != NULL) { // TODO @@ -831,7 +899,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; @@ -858,6 +926,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v return 0; } +#endif void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 46feab7791..f07cc0b0b5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); @@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { @@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU8(buf, pReq->type); return tlen; } void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeFixedU8(buf, &pReq->type); return buf; @@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; @@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; @@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; @@ -1669,8 +1669,8 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->tuid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->suid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->tuid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfColumns + pRsp->numOfTags; ++i) { SSchema *pSchema = &pRsp->pSchemas[i]; @@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; @@ -1692,8 +1692,8 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->tuid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->suid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->tuid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; @@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; @@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; @@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; tEndEncode(&encoder); @@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; tEndDecode(&decoder); @@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) { return 0; } -int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) { - if (tStartEncode(encoder) < 0) return -1; - if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1; - int32_t sz = taosArrayGetSize(pOffsets->offsets); - if (tEncodeI32(encoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SMqOffset *offset = taosArrayGet(pOffsets->offsets, i); - if (tEncodeSMqOffset(encoder, offset) < 0) return -1; - } - tEndEncode(encoder); - return encoder->pos; -} - -int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) { - int32_t sz; - if (tStartDecode(decoder) < 0) return -1; - if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1; - if (tDecodeI32(decoder, &sz) < 0) return -1; - pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset)); - for (int32_t i = 0; i < sz; i++) { - SMqOffset offset; - if (tDecodeSMqOffset(decoder, &offset) < 0) return -1; - taosArrayPush(pOffsets->offsets, &offset); - } - tEndDecode(decoder); - return 0; -} - -int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) { +int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) { if (tStartEncode(encoder) < 0) return -1; if (tEncodeI32(encoder, pReq->num) < 0) return -1; for (int32_t i = 0; i < pReq->num; i++) { @@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p return encoder->pos; } -int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { +int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); @@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { tEndDecode(decoder); return 0; } - -#if 0 -int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) { - if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tEncodeSMqOffset(encoder, &pReq->offsets[i]); - } - return encoder->pos; -} - -int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) { - if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); - if (pReq->offsets == NULL) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tDecodeSMqOffset(decoder, &pReq->offsets[i]); - } - return 0; -} -#endif diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 4e35baf905..0aae145d2f 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,8 +25,8 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { @@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; @@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode * pDnode = parent; + SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pRsp->msgType; @@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) { } static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode * pDnode = param; + SDnode *pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pReq->msgType; @@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e64191f715..bb2367ef72 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -123,6 +123,7 @@ typedef enum { TRN_TYPE_DROP_TOPIC = 1015, TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, + TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -176,7 +177,7 @@ typedef struct { SArray* undoActions; int64_t createdTime; int64_t lastExecTime; - uint64_t dbUid; + int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; } STrans; @@ -304,16 +305,16 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -346,8 +347,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t createdTime; int64_t updateTime; - uint64_t uid; - uint64_t dbUid; + int64_t uid; + int64_t dbUid; int32_t version; int32_t nextColId; int32_t numOfColumns; @@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { } } +typedef struct { + char key[TSDB_PARTITION_KEY_LEN]; + int64_t offset; +} SMqOffsetObj; + +static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} + typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h new file mode 100644 index 0000000000..e18eec973f --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_OFFSET_H_ +#define _TD_MND_OFFSET_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitOffset(SMnode *pMnode); +void mndCleanupOffset(SMnode *pMnode); + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key); +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); + +static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { + return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_OFFSET_H_*/ diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c new file mode 100644 index 0000000000..ac9e99ebd4 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndOffset.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_OFFSET_VER_NUMBER 1 +#define MND_OFFSET_RESERVE_SIZE 64 + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); + +int32_t mndInitOffset(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupOffset(SMnode *pMnode) {} + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset); + int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); + if (pRaw == NULL) goto OFFSET_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto OFFSET_ENCODE_OVER; + + void *abuf = buf; + tEncodeSMqOffsetObj(&abuf, pOffset); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRaw; +} + +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER; + + if (sver != MND_OFFSET_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto OFFSET_DECODE_OVER; + } + + int32_t size = sizeof(SMqOffsetObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto OFFSET_DECODE_OVER; + + SMqOffsetObj *pOffset = sdbGetRowObj(pRow); + if (pOffset == NULL) goto OFFSET_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto OFFSET_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); + + if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) { + goto OFFSET_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRow; +} + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { + int32_t code = 0; + int32_t sz = taosArrayGetSize(vgs); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + return -1; + } + offsetObj.offset = -1; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj); + if (pOffsetRaw == NULL) { + return -1; + } + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { + return -1; + } + } + return 0; +} + +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { + char key[TSDB_PARTITION_KEY_LEN]; + + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SMqCMCommitOffsetReq commitOffsetReq; + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER); + + tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg); + + for (int32_t i = 0; i < commitOffsetReq.num; i++) { + SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + return -1; + } + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); + ASSERT(pOffsetObj); + pOffsetObj->offset = pOffset->offset; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pOffsetRaw); + mndReleaseOffset(pMnode, pOffsetObj); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform insert action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform delete action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { + mTrace("offset:%s, perform update action", pOldOffset->key); + return 0; +} + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) { + SSdb *pSdb = pMnode->pSdb; + SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key); + if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST; + } + return pOffset; +} + +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pOffset); +} + +static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a5c4c41244..2ea157fea4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -19,6 +19,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" #include "mndTopic.h" @@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name); + char *key = mndMakeSubscribeKey(cgroup, pTopic->name); if (key == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); @@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); for (int32_t k = 0; k < vgsz; k++) { + char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; + SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = pOffsetObj->offset; + mndReleaseOffset(pMnode, pOffsetObj); + } taosArrayPush(topicEp.vgs, &vgEp); } taosArrayPush(rsp.topics, &topicEp); @@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SUB_ENCODE_OVER: tfree(buf); - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; @@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; + + mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); } SMqSubConsumer mqSubConsumer; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64b4aa6dd7..5b5bf661e5 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -23,6 +23,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" @@ -77,7 +78,7 @@ static void mndTransReExecute(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } @@ -89,7 +90,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } @@ -197,6 +198,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -440,7 +442,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; - void * ahandle = pMsg->rpcMsg.ahandle; + void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4dc46b3798..ac9dde3597 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; + int64_t fetchOffset; /*int64_t blockingTime = pReq->blockingTime;*/ + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); ASSERT(pConsumer->consumerId == consumerId); - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - if (pTopic->committedOffset < pReq->offset - 1) { - pTopic->committedOffset = pReq->offset - 1; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - } - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; + rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - SWalHead* pHead; while (1) { int8_t pos = fetchOffset % TQ_BUFFER_SIZE; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade280eecc..c3947da459 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - // ser request version + // set request version void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { - /*ASSERT(false);*/ // TODO: handle error + /*ASSERT(false);*/ + vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); } } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 18485249b3..4078ee9291 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } else { SStreamBlockScanInfo* pInfo = pOperator->info; if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg error while set stream msg, %s" PRIx64, id); + qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } return TSDB_CODE_SUCCESS; From 302463f635294975813d0d71a5bf728f4a01286b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 23 Feb 2022 16:13:23 +0800 Subject: [PATCH 23/36] set default offset config --- include/util/tdef.h | 524 ++++++++++++++++++++-------------------- source/client/src/tmq.c | 2 +- 2 files changed, 264 insertions(+), 262 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 407d0ccd9e..4eef8a30b7 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +// clang-format off #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H @@ -24,370 +25,371 @@ extern "C" { #define TSDB__packed -#define TSKEY int64_t -#define TSKEY_MIN INT64_MIN -#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY int64_t +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; // TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) -#define TSDB_KEYSIZE sizeof(TSKEY) -#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) // NULL definition -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF -#define TSDB_DATA_UTINYINT_NULL 0xFF -#define TSDB_DATA_USMALLINT_NULL 0xFFFF -#define TSDB_DATA_UINT_NULL 0xFFFFFFFF -#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" -#define TSDB_NETTEST_USER "nettestinternal" -#define TSDB_DEFAULT_USER "root" +#define TSDB_NETTEST_USER "nettestinternal" +#define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ -#define TSDB_DEFAULT_PASS "powerdb" +#define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) -#define TSDB_DEFAULT_PASS "tqueue" +#define TSDB_DEFAULT_PASS "tqueue" #elif (_TD_PRO_ == true) -#define TSDB_DEFAULT_PASS "prodb" +#define TSDB_DEFAULT_PASS "prodb" #else -#define TSDB_DEFAULT_PASS "taosdata" +#define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 +#define SHELL_MAX_PASSWORD_LEN 20 -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 #define TSDB_ERR -1 #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' -#define TSDB_TIME_PRECISION_MILLI 0 -#define TSDB_TIME_PRECISION_MICRO 1 -#define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" -#define TSDB_TICK_PER_SECOND(precision) \ - ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ - : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) +#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ - do { \ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ - } while (0) -#define T_READ_MEMBER(src, type, target) \ - do { \ - (target) = *(type *)(src); \ - (src) = (void *)((char *)src + sizeof(type)); \ - } while (0) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ +do {\ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ +} while(0) +#define T_READ_MEMBER(src, type, target) \ +do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type));\ +} while(0) + // TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 +#define TSDB_RELATION_INVALID 0 +#define TSDB_RELATION_LESS 1 +#define TSDB_RELATION_GREATER 2 +#define TSDB_RELATION_EQUAL 3 +#define TSDB_RELATION_LESS_EQUAL 4 #define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_NOT_LIKE 8 -#define TSDB_RELATION_ISNULL 9 -#define TSDB_RELATION_NOTNULL 10 -#define TSDB_RELATION_IN 11 -#define TSDB_RELATION_NOT_IN 12 +#define TSDB_RELATION_NOT_EQUAL 6 +#define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_NOT_LIKE 8 +#define TSDB_RELATION_ISNULL 9 +#define TSDB_RELATION_NOTNULL 10 +#define TSDB_RELATION_IN 11 +#define TSDB_RELATION_NOT_IN 12 -#define TSDB_RELATION_AND 13 -#define TSDB_RELATION_OR 14 -#define TSDB_RELATION_NOT 15 +#define TSDB_RELATION_AND 13 +#define TSDB_RELATION_OR 14 +#define TSDB_RELATION_NOT 15 -#define TSDB_RELATION_MATCH 16 -#define TSDB_RELATION_NMATCH 17 +#define TSDB_RELATION_MATCH 16 +#define TSDB_RELATION_NMATCH 17 -#define TSDB_BINARY_OP_ADD 4000 -#define TSDB_BINARY_OP_SUBTRACT 4001 -#define TSDB_BINARY_OP_MULTIPLY 4002 -#define TSDB_BINARY_OP_DIVIDE 4003 -#define TSDB_BINARY_OP_REMAINDER 4004 -#define TSDB_BINARY_OP_CONCAT 4005 +#define TSDB_BINARY_OP_ADD 4000 +#define TSDB_BINARY_OP_SUBTRACT 4001 +#define TSDB_BINARY_OP_MULTIPLY 4002 +#define TSDB_BINARY_OP_DIVIDE 4003 +#define TSDB_BINARY_OP_REMAINDER 4004 +#define TSDB_BINARY_OP_CONCAT 4005 -#define FUNCTION_CEIL 4500 -#define FUNCTION_FLOOR 4501 -#define FUNCTION_ABS 4502 -#define FUNCTION_ROUND 4503 +#define FUNCTION_CEIL 4500 +#define FUNCTION_FLOOR 4501 +#define FUNCTION_ABS 4502 +#define FUNCTION_ROUND 4503 -#define FUNCTION_LENGTH 4800 -#define FUNCTION_CONCAT 4801 -#define FUNCTION_LTRIM 4802 -#define FUNCTION_RTRIM 4803 +#define FUNCTION_LENGTH 4800 +#define FUNCTION_CONCAT 4801 +#define FUNCTION_LTRIM 4802 +#define FUNCTION_RTRIM 4803 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) -#define TSDB_NAME_DELIMITER_LEN 1 +#define TSDB_NAME_DELIMITER_LEN 1 -#define TSDB_UNI_LEN 24 -#define TSDB_USER_LEN TSDB_UNI_LEN +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 +#define TSDB_ACCT_ID_LEN 11 -#define TSDB_MAX_COLUMNS 4096 -#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_COMMENT_LEN 4096 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 -#define TSDB_FUNC_TYPE_SCALAR 1 -#define TSDB_FUNC_TYPE_AGGREGATE 2 -#define TSDB_FUNC_MAX_RETRIEVE 1024 +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_TYPE_STR_MAX_LEN 32 -#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN -#define TSDB_CONSUMER_GROUP_LEN 193 -#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) -#define TSDB_COL_NAME_LEN 65 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 1024 -#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_CONSUMER_GROUP_LEN 192 +#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 1024 +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 -/** - * In some scenarios uint16_t (0~65535) is used to store the row len. - * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. - * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-(4096-1)*4 = 49151. - */ -#define TSDB_MAX_BYTES_PER_ROW 49151 -#define TSDB_MAX_TAGS_LEN 16384 -#define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_STB_COMMENT_LEN 1024 + /** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 -#define TSDB_AUTH_LEN 16 -#define TSDB_PASSWORD_LEN 32 -#define TSDB_USET_PASSWORD_LEN 129 -#define TSDB_VERSION_LEN 12 -#define TSDB_LABEL_LEN 8 +#define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 +#define TSDB_VERSION_LEN 12 +#define TSDB_LABEL_LEN 8 -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 -#define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 -#define TSDB_TRANS_STAGE_LEN 12 -#define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 64 +#define TSDB_TRANS_STAGE_LEN 12 +#define TSDB_TRANS_TYPE_LEN 16 +#define TSDB_TRANS_ERROR_LEN 64 -#define TSDB_STEP_NAME_LEN 32 -#define TSDB_STEP_DESC_LEN 128 +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 -#define TSDB_ERROR_MSG_LEN 1024 -#define TSDB_DNODE_CONFIG_LEN 128 -#define TSDB_DNODE_VALUE_LEN 256 +#define TSDB_ERROR_MSG_LEN 1024 +#define TSDB_DNODE_CONFIG_LEN 128 +#define TSDB_DNODE_VALUE_LEN 256 -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 -#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE -#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 -#define TSDB_DEFAULT_VN_PER_DB 2 +#define TSDB_DEFAULT_VN_PER_DB 2 -#define TSDB_DNODE_ROLE_ANY 0 -#define TSDB_DNODE_ROLE_MGMT 1 -#define TSDB_DNODE_ROLE_VNODE 2 +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) -#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta + +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 3650 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_KEEP 1 // data in db to be reserved. -#define TSDB_MAX_KEEP 365000 // data in db to be reserved. -#define TSDB_DEFAULT_KEEP 3650 // ten years +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_ID 1 +#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) -#define TSDB_MAX_FIELD_LEN 16384 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 -#define PRIMARYKEY_TIMESTAMP_COL_ID 1 -#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) +#define TSDB_MAX_RPC_THREADS 5 -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u -#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u -#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file -#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 +#define TSDB_MAX_WAL_SIZE (1024*1024*3) -#define TSDB_MAX_WAL_SIZE (1024 * 1024 * 3) +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P -#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P - -#define TFS_MAX_TIERS 3 +#define TFS_MAX_TIERS 3 #define TFS_MAX_DISKS_PER_TIER 16 -#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 +#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index e206b1a2bb..9a1025c4bd 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -126,7 +126,7 @@ typedef struct { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; - conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; return conf; } From 906b8d75f8b8cb0735d7d389df2194767fef2c03 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 23 Feb 2022 16:27:04 +0800 Subject: [PATCH 24/36] type compatibitily --- include/common/tmsg.h | 4 ++-- include/common/tmsgdef.h | 1 - include/util/tdef.h | 1 + source/common/src/tmsg.c | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d92efd163d..ae3586e735 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -804,8 +804,8 @@ typedef struct { int8_t update; int32_t sversion; int32_t tversion; - int64_t suid; - int64_t tuid; + uint64_t suid; + uint64_t tuid; int32_t vgId; SSchema* pSchemas; } STableMetaRsp; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b20d6de556..18718b0c52 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -194,4 +194,3 @@ enum { TDMT_MAX #endif }; -// clang-format on diff --git a/include/util/tdef.h b/include/util/tdef.h index 7ea574f398..d0f2b77f1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -14,6 +14,7 @@ */ // clang-format off + #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f07cc0b0b5..8101c18ebf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1669,8 +1669,8 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->suid) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->tuid) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->tuid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfColumns + pRsp->numOfTags; ++i) { SSchema *pSchema = &pRsp->pSchemas[i]; @@ -1692,8 +1692,8 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1; - if (tDecodeI64(pDecoder, &pRsp->suid) < 0) return -1; - if (tDecodeI64(pDecoder, &pRsp->tuid) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->tuid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; From 4e573e41e0e6cde4045d6ad81eebbb5bc34893ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 17:08:18 +0800 Subject: [PATCH 25/36] sim config --- include/os/osEnv.h | 2 - include/util/tlog.h | 27 +++++------ source/client/src/clientCfg.c | 11 ++--- source/common/src/tglobal.c | 9 ---- source/dnode/mgmt/daemon/src/dmnCfg.c | 2 +- source/libs/config/src/config.c | 66 ++++++++++----------------- source/os/src/osEnv.c | 3 -- 7 files changed, 42 insertions(+), 78 deletions(-) diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a7fd86776c..fec3b1abc4 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -22,8 +22,6 @@ extern "C" { extern char tsOsName[]; extern char tsDataDir[]; -extern char tsLogDir[]; -extern char tsScriptDir[]; extern char configDir[]; #ifdef __cplusplus diff --git a/include/util/tlog.h b/include/util/tlog.h index 7499eaa7b4..cdcac84ab7 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,7 +22,8 @@ extern "C" { #endif -// log +extern char tsLogDir[]; +extern bool tsLogInited; extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; @@ -41,20 +42,21 @@ extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U - +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U #define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +#define DEBUG_FILE 128U int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); +void taosSetAllDebugFlag(int32_t flag); +void taosDumpData(unsigned char *msg, int32_t len); void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) #ifdef __GNUC__ @@ -68,11 +70,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -void taosDumpData(unsigned char *msg, int32_t len); - - -void taosSetAllDebugFlag(int32_t flag); - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c index 352495178b..ca671ab150 100644 --- a/source/client/src/clientCfg.c +++ b/source/client/src/clientCfg.c @@ -83,6 +83,8 @@ static int32_t tscSetLogCfg(SConfig *pCfg) { } int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + if (tsLogInited) return 0; + SConfig *pCfg = cfgInit(); if (pCfg == NULL) return -1; @@ -111,6 +113,7 @@ int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUr return -1; } + cfgDumpCfg(pCfg); cfgCleanup(pCfg); return 0; } @@ -151,7 +154,7 @@ static int32_t tscAddCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; - // if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; @@ -174,12 +177,6 @@ SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apol SConfig *pCfg = cfgInit(); if (pCfg == NULL) return NULL; - if (tscAddLogCfg(pCfg) != 0) { - uError("failed to add log cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - if (tscAddCfg(pCfg) != 0) { uError("failed to init tsc cfg since %s", terrstr()); cfgCleanup(pCfg); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fc25dac0df..9bd9386244 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -280,15 +280,6 @@ static void doInitGlobalConfig(void) { #if 0 SGlobalCfg cfg = {0}; - cfg.option = "scriptDir"; - cfg.ptr = tsScriptDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 1f68df6fe4..d3cdc30870 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -202,7 +202,7 @@ SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; - tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn, cfgGetItem)); + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); return objCfg; } \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index 25b65a0d1b..17a652d2b5 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -15,11 +15,14 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" -#include "tmsg.h" #include "tep.h" #include "tlocale.h" +#include "tmsg.h" #include "ttimezone.h" +#define CFG_NAME_PRINT_LEN 22 +#define CFG_SRC_PRINT_LEN 12 + SConfig *cfgInit() { SConfig *pCfg = calloc(1, sizeof(SConfig)); if (pCfg == NULL) { @@ -505,56 +508,37 @@ const char *cfgDtypeStr(ECfgDataType type) { } } -void cfgPrintCfg(SConfig *pCfg) { - uInfo("taos global config"); - uInfo("=================================="); - - SConfigItem *pItem = cfgIterate(pCfg, NULL); - while (pItem != NULL) { - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: - uInfo("cfg:%s, value:%u src:%s", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_INT32: - uInfo("cfg:%s, value:%d src:%s", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_INT64: - uInfo("cfg:%s, value:%" PRId64 " src:%s", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_FLOAT: - uInfo("cfg:%s, value:%f src:%s", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_IPSTR: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - uInfo("cfg:%s, value:%s src:%s", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); - break; - } - pItem = cfgIterate(pCfg, pItem); - } -} - void cfgDumpCfg(SConfig *pCfg) { - uInfo("global config"); - uInfo("=================================="); + uInfo(" global config"); + uInfo("================================================================="); + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; SConfigItem *pItem = cfgIterate(pCfg, NULL); while (pItem != NULL) { + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { + if (src[i] == 0) src[i] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) { + if (name[i] == 0) name[i] = ' '; + } + switch (pItem->dtype) { case CFG_DTYPE_BOOL: - uInfo("cfg:%s, value:%u src:%s", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + uInfo("%s %s %u", src, name, pItem->bval); break; case CFG_DTYPE_INT32: - uInfo("cfg:%s, value:%d src:%s", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + uInfo("%s %s %d", src, name, pItem->i32); break; case CFG_DTYPE_INT64: - uInfo("cfg:%s, value:%" PRId64 " src:%s", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + uInfo("%s %s %" PRId64, src, name, pItem->i64); break; case CFG_DTYPE_FLOAT: - uInfo("cfg:%s, value:%f src:%s", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + uInfo("%s %s %f", src, name, pItem->fval); break; case CFG_DTYPE_STRING: case CFG_DTYPE_IPSTR: @@ -562,13 +546,13 @@ void cfgDumpCfg(SConfig *pCfg) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - uInfo("cfg:%s, value:%s src:%s", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + uInfo("%s %s %s", src, name, pItem->str); break; } pItem = cfgIterate(pCfg, pItem); } - uInfo("=================================="); + uInfo("================================================================="); } #if 0 // int32_t cfgCheck(SConfig *pCfg) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7bc5e0a7fc..38e564fa9d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -23,7 +23,6 @@ char tsOsName[10] = "Windows"; char configDir[PATH_MAX] = "C:/TDengine/cfg"; char tsDataDir[PATH_MAX] = "C:/TDengine/data"; char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; extern taosWinSocketInit(); @@ -47,7 +46,6 @@ char tsOsName[10] = "Darwin"; char configDir[PATH_MAX] = "/usr/local/etc/taos"; char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; char tsTempDir[PATH_MAX] = "/tmp/taosd"; void osInit() {} @@ -58,7 +56,6 @@ char tsOsName[10] = "Linux"; char configDir[PATH_MAX] = "/etc/taos"; char tsDataDir[PATH_MAX] = "/var/lib/taos"; char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/etc/taos"; char tsTempDir[PATH_MAX] = "/tmp/"; void osInit() {} From a74188f45691404e0334a6f4f288b93bf5eac379 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 17:34:06 +0800 Subject: [PATCH 26/36] fix conflicts --- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 1 + source/libs/index/test/fstUT.cc | 4 ++-- source/libs/scalar/test/filter/filterTests.cpp | 4 +--- source/libs/scalar/test/scalar/scalarTests.cpp | 4 +--- source/libs/transport/test/transUT.cc | 6 ++---- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index ac199107a0..d32bdf29c6 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -33,6 +33,7 @@ void Testbase::InitLog(const char* path) { taosRemoveDir(path); taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d10f5f47b8..5df44a7433 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -30,8 +30,8 @@ static void EnvInit() { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); // init log file - snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str()); - if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("tindex.idx", 1) != 0) { printf("failed to init log"); } // init index file diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index db9df9ff08..420371fa04 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -52,9 +52,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2f0a60362d..24bc8eaf40 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -51,9 +51,7 @@ void scltInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index d1fefe2c72..f5b3ed4c32 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -148,7 +148,6 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; @@ -156,9 +155,8 @@ class TransObj { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } cli = new Client; From 7cc1d78ec6868a2da7cf25614281404e95c0e1d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 17:49:24 +0800 Subject: [PATCH 27/36] numOfCores should large than 2 --- source/dnode/mgmt/daemon/src/dmnCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index d3cdc30870..85a9a28750 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -70,7 +70,7 @@ static int32_t dmnAddDnodeCfg(SConfig *pCfg) { if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; From 22938fcc5edeb78314c277ee44049895717afae3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Feb 2022 17:51:07 +0800 Subject: [PATCH 28/36] add more UT --- source/libs/index/src/index_fst.c | 2 ++ source/libs/index/src/index_fst_automation.c | 21 +++++++++----------- source/libs/index/test/fstUT.cc | 15 ++++++++++---- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 3664bcfad0..a6cabbd439 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -235,6 +235,7 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); + bool null = false; uint8_t inp = fstStateCommInput(&st, &null); if (null == true) { @@ -936,6 +937,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } + void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 590ff294bf..ed1ad7a374 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,24 +16,24 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { + StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + if (sv == NULL) { return NULL; } - nsv->kind = kind; - nsv->type = ty; + sv->kind = kind; + sv->type = ty; if (ty == FST_INT) { - nsv->val = *(int*)val; + sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - nsv->ptr = (char*)calloc(1, len + 1); - memcpy(nsv->ptr, val, len); + sv->ptr = (char*)calloc(1, len + 1); + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, // nsv->arr = taosArrayFromList() } - return nsv; + return sv; } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; @@ -146,11 +146,9 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (atype == AUTOMATION_ALWAYS) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type @@ -160,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)malloc(len * sizeof(char) + 1); + dst = (char*)calloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); - dst[len] = 0; } ctx->data = dst; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d10f5f47b8..536f7c1a4c 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -99,6 +99,7 @@ class FstReadMemory { fstSliceDestroy(&skey); return ok; } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { int64_t s = taosGetTimestampUs(); bool ok = this->Get(key, val); @@ -120,8 +121,6 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) { - } std::cout << std::endl; return true; } @@ -137,7 +136,7 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); + // tfCleanup(); } private: @@ -196,6 +195,10 @@ class TFst { } return fr->Get(k, v); } + bool Search(AutomationCtx* ctx, std::vector& result) { + // add more + return fr->Search(ctx, result); + } private: FstWriter* fw; @@ -229,5 +232,9 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Get("a", &val) == false); assert(fst->Get("aa", &val) == true); assert(val == 0); + + std::vector rlt; + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + assert(fst->Search(ctx, rlt) == true); } -TEST_F(FstEnv, writeExcpet) {} +TEST_F(FstEnv, WriteMillonrRecord) {} From b04530219612a6f0cc8dcdbe3e01f1c18fc2edd7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 17:55:10 +0800 Subject: [PATCH 29/36] disable telem --- source/dnode/mnode/impl/src/mndTelem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 147735f356..5292dd0a41 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -245,7 +245,7 @@ SEND_OVER: static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; - // if (!pMgmt->enable) return 0; + if (!pMgmt->enable) return 0; taosWLockLatch(&pMgmt->lock); mndSendTelemetryReport(pMnode); From 84b30e078402e33ee37eafc81289de64c3aad5ce Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 18:18:27 +0800 Subject: [PATCH 30/36] remove global variables --- include/common/tglobal.h | 6 ------ source/common/src/tglobal.c | 33 ++++++--------------------------- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 7458e5d837..c4d134b928 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -21,12 +21,6 @@ extern "C" { #endif #include "tdef.h" -#include "tcfg.h" - -// cluster -extern int32_t tsStatusInterval; -extern int8_t tsEnableTelemetryReporting; -extern int32_t tsNumOfSupportVnodes; // common extern int tsRpcTimer; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9bd9386244..cd31988af4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -27,11 +27,6 @@ #include "tutil.h" #include "ulog.h" -// cluster -int32_t tsStatusInterval = 1; // second -int8_t tsEnableTelemetryReporting = 0; -char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 128; // common int32_t tsRpcTimer = 300; @@ -125,12 +120,6 @@ int8_t tsDeadLockKillQuery = 0; // For backward compatibility bool tsdbForceKeepFile = false; -#ifndef _STORAGE -SDiskCfg tsDiskCfg[1]; -#else -SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; -#endif - /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -247,12 +236,12 @@ int32_t taosCfgDynamicOptions(char *msg) { return false; } -void taosAddDataDir(int index, char *v1, int level, int primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; - uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); -} +// void taosAddDataDir(int index, char *v1, int level, int primary) { +// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); +// tsDiskCfg[index].level = level; +// tsDiskCfg[index].primary = primary; +// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +// } #ifndef _STORAGE // void taosReadDataDirCfg(char *v1, char *v2, char *v3) { @@ -302,16 +291,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "telemetryReporting"; - cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // timer cfg.option = "maxTmrCtrl"; cfg.ptr = &tsMaxTmrCtrl; From 01907592807146e97f94399f31b9c5755d39a8d1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 18:21:45 +0800 Subject: [PATCH 31/36] remove rpc var --- include/common/tglobal.h | 3 --- source/common/src/tglobal.c | 34 ----------------------------- source/libs/transport/src/rpcMain.c | 6 +++++ 3 files changed, 6 insertions(+), 37 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c4d134b928..49d8f84d38 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -23,9 +23,6 @@ extern "C" { #include "tdef.h" // common -extern int tsRpcTimer; -extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cd31988af4..0042239e1a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -29,9 +29,6 @@ // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second @@ -302,37 +299,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "rpcTimer"; - cfg.ptr = &tsRpcTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 3000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "rpcForceTcp"; - cfg.ptr = &tsRpcForceTcp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcMaxTime"; - cfg.ptr = &tsRpcMaxTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 7200; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 72c1ff6893..44bb6fdb70 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default + SHashObj *tsFqdnHash; #ifndef USE_UV @@ -143,6 +145,10 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; + +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; + // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 From e9a5ee3a5f8d3c79dfeae26a7e33bfd42feceef2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 18:31:22 +0800 Subject: [PATCH 32/36] remove global variables --- include/common/tglobal.h | 2 -- source/client/src/clientCfg.c | 3 ++- source/client/src/clientEnv.c | 2 +- source/common/src/tglobal.c | 12 ------------ source/libs/executor/src/executorimpl.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 +- 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 49d8f84d38..b7f8721459 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -23,8 +23,6 @@ extern "C" { #include "tdef.h" // common -extern int32_t tsMaxConnections; -extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c index ca671ab150..22b1407a15 100644 --- a/source/client/src/clientCfg.c +++ b/source/client/src/clientCfg.c @@ -159,8 +159,9 @@ static int32_t tscAddCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; - if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + + if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1; return 0; } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c7640e90d8..37227cea6f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -90,7 +90,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.sessions = tsMaxConnections; + rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0042239e1a..f9f00fa9a3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -29,8 +29,6 @@ // common -int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; @@ -440,16 +438,6 @@ static void doInitGlobalConfig(void) { taosAddConfigOption(cfg); - cfg.option = "maxConnections"; - cfg.ptr = &tsMaxConnections; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "minimalLogDirGB"; cfg.ptr = &tsMinimalLogDirGB; cfg.valType = TAOS_CFG_VTYPE_FLOAT; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5508b5ecd7..b35ce9e7c3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5301,7 +5301,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; + rpcInit.sessions = 50000; //tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; rpcInit.idleTime = tsShellActivityTimer * 1000; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cd040c238..9b381ad823 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -87,7 +87,7 @@ int32_t initTaskQueue() { double factor = 4.0; int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); - int32_t queueSize = tsMaxConnections * 2; + int32_t queueSize = 25000; //tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { qError("failed to init task queue"); From 76d69529c827e7373a62593966f3eb47006913ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 18:38:48 +0800 Subject: [PATCH 33/36] remove global variables --- include/common/tglobal.h | 6 ------ include/util/ttimer.h | 1 - source/client/src/clientEnv.c | 2 +- source/common/src/tglobal.c | 27 ------------------------- source/libs/executor/src/executorimpl.c | 7 +++++-- source/libs/qcom/src/queryUtil.c | 4 +++- source/libs/transport/test/rsclient.c | 2 +- 7 files changed, 10 insertions(+), 39 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index b7f8721459..ec3bac0736 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -23,13 +23,7 @@ extern "C" { #include "tdef.h" // common -extern int32_t tsShellActivityTimer; -extern uint32_t tsMaxTmrCtrl; -extern float tsNumOfThreadsPerCore; -extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryCores; extern int8_t tsDaylight; -extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 89ec6cd8d9..01d70c7d02 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -26,7 +26,6 @@ typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern int taosTmrThreads; -extern uint32_t tsMaxTmrCtrl; #define MSECONDS_PER_TICK 5 diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 37227cea6f..38f94ecae8 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -93,7 +93,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)auth; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f9f00fa9a3..86af06d31a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -29,12 +29,7 @@ // common -int32_t tsShellActivityTimer = 3; // second -float tsNumOfThreadsPerCore = 1.0f; -int32_t tsNumOfCommitThreads = 4; -float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; -int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; @@ -286,17 +281,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // timer - cfg.option = "maxTmrCtrl"; - cfg.ptr = &tsMaxTmrCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 8; - cfg.maxValue = 2048; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -490,17 +474,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "enableCoreFile"; - cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b35ce9e7c3..62c0e3623f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,7 +132,8 @@ do { \ } while (0) int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; + // todo + return 6; //tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo *pExprInfo) { @@ -5301,10 +5302,12 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = qProcessFetchRsp; + // todo rpcInit.sessions = 50000; //tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; + // todo + rpcInit.idleTime = 6; //tsShellActivityTimer * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9b381ad823..b29817a318 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,7 +85,9 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + // todo + // int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2); int32_t queueSize = 25000; //tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index f9bcc3da96..26a02eb05b 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; From 4f353da40c9b57fb0b08fd34cd88efca2f0115ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 20:35:09 +0800 Subject: [PATCH 34/36] global config --- include/common/tglobal.h | 11 +------- include/os/osEnv.h | 15 +++++++++++ include/os/osSysinfo.h | 25 ++++++++++++------- include/util/tlog.h | 1 - source/common/src/tglobal.c | 36 --------------------------- source/dnode/mgmt/daemon/src/dmnCfg.c | 16 ++++++------ source/dnode/mgmt/daemon/src/dmnLog.c | 2 ++ source/dnode/mgmt/impl/src/dndEnv.c | 12 --------- source/os/src/osEnv.c | 24 ++++++++++++++++++ source/util/src/tlog.c | 24 +++--------------- 10 files changed, 69 insertions(+), 97 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ec3bac0736..3ad16c4cf4 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -56,16 +56,7 @@ extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; // system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; + extern uint32_t tsVersion; // build info diff --git a/include/os/osEnv.h b/include/os/osEnv.h index fec3b1abc4..2745520ae3 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -21,9 +21,24 @@ extern "C" { #endif extern char tsOsName[]; + extern char tsDataDir[]; +extern char tsLogDir[]; +extern char tsTempDir[]; extern char configDir[]; +extern struct SDiskSpace tsLogSpace; +extern struct SDiskSpace tsTempSpace; +extern struct SDiskSpace tsDataSpace; + +void taosUpdateLogSpace(); +void taosUpdateTempSpace(); +void taosUpdateDataSpace(); +bool taosLogSpaceAvailable(); +bool taosTmpSpaceAvailable(); +bool taosDataSpaceAvailable(); +void taosUpdateAllSpace(); + #ifdef __cplusplus } #endif diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9dcb075489..6ab2a104df 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -25,21 +25,28 @@ extern "C" { #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string - typedef struct { int64_t total; int64_t used; int64_t avail; } SDiskSize; +typedef struct SDiskSpace { + int64_t reserved; + SDiskSize size; +} SDiskSpace; + +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; +extern char tsTimezone[]; +extern char tsLocale[]; +extern char tsCharset[]; // default encode string + + + int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); diff --git a/include/util/tlog.h b/include/util/tlog.h index cdcac84ab7..166e186508 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,7 +22,6 @@ extern "C" { #endif -extern char tsLogDir[]; extern bool tsLogInited; extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 86af06d31a..72783bf4ab 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -119,13 +119,6 @@ bool tsdbForceKeepFile = false; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; @@ -422,35 +415,6 @@ static void doInitGlobalConfig(void) { taosAddConfigOption(cfg); - cfg.option = "minimalLogDirGB"; - cfg.ptr = &tsMinimalLogDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsReservedTmpDirectorySpace; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalDataDirGB"; - cfg.ptr = &tsMinimalDataDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 85a9a28750..7bf538caed 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -41,17 +41,17 @@ static int32_t dmnAddEpCfg(SConfig *pCfg) { static int32_t dmnAddDirCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; - if (cfgAddDir(pCfg, "tmpDir", tsTempDir) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; return 0; } + static int32_t dmnCheckDirCfg(SConfig *pCfg) { - SConfigItem *pItem = NULL; - - pItem = cfgGetItem(pCfg, "tmpDir"); - if (taosDirExist(pItem->str) != 0) { - return -1; - } - + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; return 0; } diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c index 3ec43cb7f3..22a7d7b080 100644 --- a/source/dnode/mgmt/daemon/src/dmnLog.c +++ b/source/dnode/mgmt/daemon/src/dmnLog.c @@ -18,6 +18,7 @@ int32_t dmnAddLogCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; @@ -41,6 +42,7 @@ int32_t dmnAddLogCfg(SConfig *pCfg) { int32_t dmnSetLogCfg(SConfig *pCfg) { tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 02dced53c2..58517c8151 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -325,18 +325,6 @@ void taosGetDisk() { SDiskSize diskSize = tfsGetSize(pTfs); tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } #endif } \ No newline at end of file diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 38e564fa9d..a7868323e2 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -16,6 +16,30 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "osEnv.h" +#include "osSysinfo.h" + +SDiskSpace tsLogSpace; +SDiskSpace tsTempSpace; +SDiskSpace tsDataSpace; + +void taosUpdateLogSpace() { taosGetDiskSize(tsLogDir, &tsLogSpace.size); } + +void taosUpdateTempSpace() { taosGetDiskSize(tsTempDir, &tsTempSpace.size); } + +void taosUpdateDataSpace() { taosGetDiskSize(tsDataDir, &tsDataSpace.size); } + +bool taosLogSpaceAvailable() { return tsDataSpace.reserved > tsDataSpace.size.avail; } + +bool taosTempSpaceAvailable() { return tsTempSpace.reserved > tsTempSpace.size.avail; } + +bool taosDataSpaceAvailable() { return tsDataSpace.reserved > tsDataSpace.size.avail; } + +void taosUpdateAllSpace() { + taosUpdateLogSpace(); + taosUpdateTempSpace(); + taosUpdateDataSpace(); +} #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ecd0f0437b..79209d09f6 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -72,9 +72,6 @@ int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; bool tsAsyncLog = true; bool tsLogInited = false; -float tsTotalLogDirGB = 0; -float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 1.0f; int64_t asyncLogLostLines = 0; int32_t writeInterval = DEFAULT_LOG_INTERVAL; @@ -383,12 +380,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; @@ -443,12 +435,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } void taosDumpData(unsigned char *msg, int32_t len) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -471,12 +458,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; From 10a21ba801a432d11de4120e2851c9b06ee7065e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Feb 2022 20:39:21 +0800 Subject: [PATCH 35/36] version --- include/common/tglobal.h | 6 ----- include/util/version.h | 33 ++++++++++++++++++++++++ source/client/src/clientMain.c | 1 + source/dnode/mgmt/daemon/inc/dmnInt.h | 1 + source/dnode/mnode/impl/src/mndProfile.c | 1 + source/os/src/osEnv.c | 6 ++--- source/util/src/tlog.c | 1 + tools/shell/src/shellLinux.c | 1 + 8 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 include/util/version.h diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 3ad16c4cf4..8bce28d34c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -59,12 +59,6 @@ extern int64_t tsMaxRetentWindow; extern uint32_t tsVersion; -// build info -extern char version[]; -extern char compatible_version[]; -extern char gitinfo[]; -extern char gitinfoOfInternal[]; -extern char buildinfo[]; // lossy extern char tsLossyColumns[]; diff --git a/include/util/version.h b/include/util/version.h new file mode 100644 index 0000000000..01efecc59d --- /dev/null +++ b/include/util/version.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_VERSION_H +#define _TD_UTIL_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_VERSION_H*/ diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 00d629ebb2..5bcc287116 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -7,6 +7,7 @@ #include "tmsg.h" #include "tglobal.h" #include "catalog.h" +#include "version.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h index 7ffb0bdeb7..19ec147769 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -22,6 +22,7 @@ #include "taoserror.h" #include "tglobal.h" #include "ulog.h" +#include "version.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df892e2242..2dfdc8dc7f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -21,6 +21,7 @@ #include "mndStb.h" #include "mndUser.h" #include "tglobal.h" +#include "version.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index a7868323e2..b90d9fbe94 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -29,11 +29,11 @@ void taosUpdateTempSpace() { taosGetDiskSize(tsTempDir, &tsTempSpace.size); } void taosUpdateDataSpace() { taosGetDiskSize(tsDataDir, &tsDataSpace.size); } -bool taosLogSpaceAvailable() { return tsDataSpace.reserved > tsDataSpace.size.avail; } +bool taosLogSpaceAvailable() { return tsLogSpace.reserved < tsLogSpace.size.avail; } -bool taosTempSpaceAvailable() { return tsTempSpace.reserved > tsTempSpace.size.avail; } +bool taosTempSpaceAvailable() { return tsTempSpace.reserved < tsTempSpace.size.avail; } -bool taosDataSpaceAvailable() { return tsDataSpace.reserved > tsDataSpace.size.avail; } +bool taosDataSpaceAvailable() { return tsDataSpace.reserved < tsDataSpace.size.avail; } void taosUpdateAllSpace() { taosUpdateLogSpace(); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 79209d09f6..f5d02df80d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -119,6 +119,7 @@ static int32_t taosStartLog() { int32_t taosInitLog(const char *logName, int maxFiles) { if (tsLogInited) return 0; + taosUpdateLogSpace(); char fullName[PATH_MAX] = {0}; snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 81d8c0a35b..89b477f08a 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -20,6 +20,7 @@ #include "shellCommand.h" #include "tkey.h" #include "ulog.h" +#include "version.h" #include #include From 2271dee643450b3e20c8e7a964df71faed801f8a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Feb 2022 23:10:56 +0800 Subject: [PATCH 36/36] refactor trans code --- include/libs/transport/trpc.h | 3 + source/client/inc/clientInt.h | 187 +++++++++++------------ source/client/src/clientEnv.c | 110 ++++++------- source/client/src/clientImpl.c | 5 +- source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/trans.c | 5 + source/libs/transport/src/transCli.c | 15 +- source/libs/transport/src/transSrv.c | 18 --- 8 files changed, 168 insertions(+), 176 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 538aeb1a0e..f1bab81573 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -78,6 +78,9 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to keep conn or not + bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cd..944da5a307 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,17 +20,17 @@ extern "C" { #endif -#include "taos.h" #include "common.h" -#include "tmsg.h" +#include "parser.h" +#include "query.h" +#include "taos.h" #include "tdef.h" #include "tep.h" #include "thash.h" #include "tlist.h" +#include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" -#include "query.h" -#include "parser.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -46,12 +46,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; typedef struct SHbConnInfo { - void *param; - SClientHbReq *req; + void* param; + SClientHbReq* req; } SHbConnInfo; typedef struct SAppHbMgr { - char *key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -62,15 +62,13 @@ typedef struct SAppHbMgr { // connection SAppInstInfo* pAppInstInfo; // info - SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash + SHashObj* activeInfo; // hash + SHashObj* connInfo; // hash } SAppHbMgr; +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); - -typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); - +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct SClientHbMgr { int8_t inited; @@ -83,63 +81,62 @@ typedef struct SClientHbMgr { FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; - typedef struct SQueryExecMetric { - int64_t start; // start timestamp - int64_t parsed; // start to parse - int64_t send; // start to send to server - int64_t rsp; // receive response from server + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server } SQueryExecMetric; typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj } SInstanceSummary; typedef struct SHeartBeatInfo { - void *pTimer; // timer, used to send request msg to mnode + void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList *pConnList; // STscObj linked list - int64_t clusterId; - void *pTransporter; - struct SAppHbMgr *pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + struct SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char *ep; + char* ep; int32_t pid; int32_t numOfThreads; - SHashObj *pInstMap; + SHashObj* pInstMap; pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo *pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SMqConsumer { @@ -147,49 +144,49 @@ typedef struct SMqConsumer { } SMqConsumer; typedef struct SReqResultInfo { - const char *pRspMsg; - const char *pData; - TAOS_FIELD *fields; - uint32_t numOfCols; - int32_t *length; - TAOS_ROW row; - char **pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + TAOS_ROW row; + char** pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { - int64_t execId; // showId/queryId - int32_t vgId; - SArray *pArray; // SArray - int32_t currentIndex; // current accessed vgroup index. + int64_t execId; // showId/queryId + int32_t vgId; + SArray* pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now + tsem_t rspSem; // not used now void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; -#define ERROR_MSG_BUF_DEFAULT_SIZE 512 +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 typedef struct SRequestObj { - uint64_t requestId; - int32_t type; // request type - STscObj *pTscObj; - char *sqlstr; // sql string - int32_t sqlLen; - int64_t self; - char *msgBuf; - void *pInfo; // sql parse info, generated by parser module - int32_t code; - SQueryExecMetric metric; + uint64_t requestId; + int32_t type; // request type + STscObj* pTscObj; + char* sqlstr; // sql string + int32_t sqlLen; + int64_t self; + char* msgBuf; + void* pInfo; // sql parse info, generated by parser module + int32_t code; + SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -198,51 +195,52 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo); -void destroyTscObj(void*pObj); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); uint64_t generateRequestId(); -void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getDbOfConnection(STscObj* pObj); +char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *str); +int taos_options_imp(TSDB_OPTION option, const char* str); -void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); +void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, + uint16_t port); -void *doFetchRow(SRequestObj* pRequest); +void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); - -int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); -// --- heartbeat +// --- heartbeat // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(void); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); @@ -253,7 +251,6 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa..af131f7624 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" #include "clientInt.h" #include "clientLog.h" +#include "os.h" #include "query.h" #include "scheduler.h" -#include "tmsg.h" #include "tcache.h" #include "tconfig.h" #include "tglobal.h" +#include "tmsg.h" #include "tnote.h" #include "tref.h" #include "trpc.h" @@ -30,16 +30,16 @@ #include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 +#define TSC_VAR_RELEASED 0 -SAppInfo appInfo; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; +SAppInfo appInfo; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj* pRequest) { +static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -53,23 +53,25 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self, - pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 + ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } -static void deregisterRequest(SRequestObj* pRequest) { +static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj* pTscObj = pRequest->pTscObj; - SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary; + STscObj * pTscObj = pRequest->pTscObj; + SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, - pRequest->requestId, duration, num, currentInst); + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 + " ms, current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } @@ -79,8 +81,8 @@ static void tscInitLogFile() { printf("failed to create log dir:%s\n", tsLogDir); } - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; + const char * defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; char temp[128] = {0}; sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); @@ -90,23 +92,24 @@ static void tscInitLogFile() { } // todo close the transporter properly -void closeTransporter(STscObj* pTscObj) { +void closeTransporter(STscObj *pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { +void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,7 +118,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.spi = 1; rpcInit.secret = (char *)auth; - void* pDnodeConn = rpcOpen(&rpcInit); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); return NULL; @@ -130,12 +133,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -153,11 +156,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - tscDebug("connObj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; } -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { +void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); @@ -166,20 +169,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return NULL; } - pRequest->requestId = generateRequestId(); + pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; // not used it yet + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { +static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { tfree(pResInfo->pRspMsg); tfree(pResInfo->length); tfree(pResInfo->row); @@ -187,9 +190,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { tfree(pResInfo->fields); } -static void doDestroyRequest(void* p) { +static void doDestroyRequest(void *p) { assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; + SRequestObj *pRequest = (SRequestObj *)p; assert(RID_VALID(pRequest->self)); @@ -208,7 +211,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest); } -void destroyRequest(SRequestObj* pRequest) { +void destroyRequest(SRequestObj *pRequest) { if (pRequest == NULL) { return; } @@ -252,14 +255,14 @@ void taos_init_imp(void) { initTaskQueue(); clientConnRefPool = taosOpenRef(200, destroyTscObj); - clientReqRefPool = taosOpenRef(40960, doDestroyRequest); + clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); pthread_mutex_init(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("client is initialized successfully"); } @@ -281,7 +284,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -296,7 +300,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -313,8 +318,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { char sep = '.'; - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); + if (strlen(tsLocale) == 0) { // locale does not set yet + char *defaultLocale = setlocale(LC_CTYPE, ""); // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. // The launch of current system will abort soon. @@ -329,10 +334,10 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { // set the user specified locale char *locale = setlocale(LC_CTYPE, str); - if (locale != NULL) { // failed to set the user specified locale + if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale + } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } @@ -360,11 +365,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } free(charset); - } else { // it may be windows system + } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -394,7 +400,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -410,7 +417,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -434,7 +442,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -448,9 +456,9 @@ uint64_t generateRequestId() { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c39..3b35b6bdd4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -370,7 +370,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; @@ -398,7 +397,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } - +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; +} void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; assert(pMsg->ahandle != NULL); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a36b671eb4..d080db753d 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -66,6 +66,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + bool (*pfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48c15ca286..453b5184d2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } + + // register callback handle pRpc->cfp = pInit->cfp; + pRpc->afp = pInit->afp; + pRpc->pfp = pInit->pfp; + if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8312c0217c..d5fa4c8c47 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -134,8 +134,7 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { + if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); @@ -185,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - SCliMsg* pMsg = pConn->data; - - tmsg_t msgType = TDMT_MND_CONNECT; - if (pMsg != NULL) { - msgType = pMsg->msg.msgType; - } + SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = msgType + 1; + rpcMsg.msgType = pMsg->msg.msgType + 1; if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); @@ -445,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) { addrlen = sizeof(pConn->locaddr); uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); - tTrace("client conn %p create", pConn); + tTrace("client conn %p connect to server successfully", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -524,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect + tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 9fca371bf3..f0db054797 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -413,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } else { uvStartSendResp(msg); } - // uv_buf_t wb; - // uvPrepareSendData(msg, &wb); - // uv_timer_stop(conn->pTimer); - - // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -490,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -730,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -774,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); - tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { SSrvConn* pConn = thandle; - // struct sockaddr* pPeerName = &pConn->peername; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);