From 02f0f85aabb8f19a2d8d595aa991b0a12872c5e1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 28 Feb 2022 14:10:34 +0800 Subject: [PATCH 01/82] sync modify timer --- include/libs/sync/sync.h | 4 +- source/libs/sync/inc/syncEnv.h | 10 ++-- source/libs/sync/inc/syncIO.h | 6 --- source/libs/sync/inc/syncInt.h | 38 ++++++++-------- source/libs/sync/inc/syncRaft.h | 4 ++ source/libs/sync/inc/syncRaftStore.h | 4 +- source/libs/sync/src/syncEnv.c | 47 ++++++++++++++++++- source/libs/sync/src/syncIO.c | 63 +++++++++++++++++--------- source/libs/sync/src/syncMain.c | 47 ++++++++++++++----- source/libs/sync/src/syncRaft.c | 4 ++ source/libs/sync/src/syncRaftStore.c | 16 +++---- source/libs/sync/test/syncEnvTest.cpp | 9 ++-- source/libs/sync/test/syncPingTest.cpp | 15 ++++-- source/libs/sync/test/syncTest.cpp | 4 +- 14 files changed, 187 insertions(+), 84 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a619e66622..5b387852fa 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -34,7 +34,9 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 0, TAOS_SYNC_STATE_CANDIDATE = 1, TAOS_SYNC_STATE_LEADER = 2, -} ESyncState; +} ESyncRole; + +typedef ESyncRole ESyncState; typedef struct SSyncBuffer { void* data; diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index f1c4327b69..960afe37b0 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -26,19 +26,21 @@ extern "C" { #include "syncInt.h" #include "taosdef.h" #include "trpc.h" +#include "ttimer.h" typedef struct SSyncEnv { - void *pTimer; - void *pTimerManager; + tmr_h pEnvTickTimer; + tmr_h pTimerManager; + char name[128]; } SSyncEnv; int32_t syncEnvStart(); int32_t syncEnvStop(); -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); +tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param); -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); +void syncEnvStopTimer(tmr_h *pTimer); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 4b788efd79..49edeb638b 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -61,12 +61,6 @@ int32_t syncIOStop(); int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); SSyncIO *syncIOCreate(); -static int32_t doSyncIOStart(SSyncIO *io); -static int32_t doSyncIOStop(SSyncIO *io); -static int32_t doSyncIOPing(SSyncIO *io); -static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t doSyncIODestroy(SSyncIO *io); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index ad8484662a..a8451b3dc4 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -26,6 +26,7 @@ extern "C" { #include "sync.h" #include "taosdef.h" #include "tlog.h" +#include "ttimer.h" extern int32_t sDebugFlag; @@ -87,15 +88,28 @@ typedef struct SyncAppendEntries SyncAppendEntries; struct SyncAppendEntriesReply; typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; +typedef struct SRaftId { + SyncNodeId addr; + SyncGroupId vgId; +} SRaftId; + typedef struct SSyncNode { - int8_t replica; - int8_t quorum; + int8_t replica; + int8_t quorum; + int32_t refCount; + int64_t rid; SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SRaft* pRaft; + ESyncRole role; + SRaftId raftId; + SSyncFSM* pFsm; + + tmr_h pPingTimer; + tmr_h pElectionTimer; + tmr_h pHeartbeatTimer; int32_t (*FpPing)(struct SSyncNode* ths, const SyncPing* pMsg); @@ -123,23 +137,11 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); -static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); +void syncNodePingAll(SSyncNode* pSyncNode); -static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); +void syncNodePingPeers(SSyncNode* pSyncNode); -static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); - -static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg); - -static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg); - -static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); - -static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); - -static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); - -static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); +void syncNodePingSelf(SSyncNode* pSyncNode); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index a247a29fc4..bc5cf26a4c 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -27,6 +27,8 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" +#if 0 + typedef struct SRaftId { SyncNodeId addr; SyncGroupId vgId; @@ -82,6 +84,8 @@ int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); +#endif + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 0fdbd7a150..c54f1a1c83 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -34,8 +34,8 @@ extern "C" { typedef struct SRaftStore { SyncTerm currentTerm; SRaftId voteFor; - //FileFd fd; - char path[RAFT_STORE_PATH_LEN]; + // FileFd fd; + char path[RAFT_STORE_PATH_LEN]; } SRaftStore; SRaftStore *raftStoreOpen(const char *path); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index e71cf55cb1..0447fc246a 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -18,6 +18,14 @@ SSyncEnv *gSyncEnv = NULL; +// local function ----------------- +static void syncEnvTick(void *param, void *tmrId); +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); +static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param); +static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer); +// -------------------------------- + int32_t syncEnvStart() { int32_t ret; gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); @@ -31,6 +39,41 @@ int32_t syncEnvStop() { return ret; } -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { return 0; } +tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param) { + return doSyncEnvStartTimer(gSyncEnv, fp, mseconds, param); +} -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } +void syncEnvStopTimer(tmr_h *pTimer) { doSyncEnvStopTimer(gSyncEnv, pTimer); } + +// local function ----------------- +static void syncEnvTick(void *param, void *tmrId) { + SSyncEnv *pSyncEnv = (SSyncEnv *)param; + sTrace("syncEnvTick ... name:%s ", pSyncEnv->name); + + pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); +} + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { + snprintf(pSyncEnv->name, sizeof(pSyncEnv->name), "SyncEnv_%p", pSyncEnv); + + // start tmr thread + pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); + + pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); + + sTrace("SyncEnv start ok, name:%s", pSyncEnv->name); + + return 0; +} + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { + taosTmrCleanUp(pSyncEnv->pTimerManager); + return 0; +} + +static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param) { + return taosTmrStart(fp, mseconds, pSyncEnv, pSyncEnv->pTimerManager); +} + +static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer) {} +// -------------------------------- \ No newline at end of file diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index bb20d11e37..34eaf5fe1e 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -22,12 +22,49 @@ SSyncIO *gSyncIO = NULL; +// local function ------------ +static void *syncConsumer(void *param); +static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); +static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncTick(void *param, void *tmrId); + +static int32_t doSyncIOStart(SSyncIO *io); +static int32_t doSyncIOStop(SSyncIO *io); +static int32_t doSyncIOPing(SSyncIO *io); +static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t doSyncIODestroy(SSyncIO *io); +// ---------------------------- + int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } -int32_t syncIOStart() { return 0; } +int32_t syncIOStart() { + gSyncIO = syncIOCreate(); + assert(gSyncIO != NULL); + + return 0; +} int32_t syncIOStop() { return 0; } +SSyncIO *syncIOCreate() { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->start = doSyncIOStart; + io->stop = doSyncIOStop; + io->ping = doSyncIOPing; + io->onMsg = doSyncIOOnMsg; + io->destroy = doSyncIODestroy; + + return io; +} + +// local function ------------ static void syncTick(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; sDebug("syncTick ... "); @@ -46,14 +83,15 @@ static void syncTick(void *param, void *tmrId) { taosWriteQitem(io->pMsgQ, pTemp); - io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); + bool b = taosTmrReset(syncTick, 1000, io, io->syncTimerManager, io->syncTimer); + assert(b); } -void *syncConsumer(void *param) { +static void *syncConsumer(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); @@ -114,23 +152,6 @@ static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { taosWriteQitem(io->pMsgQ, pTemp); } -SSyncIO *syncIOCreate() { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->start = doSyncIOStart; - io->stop = doSyncIOStop; - io->ping = doSyncIOPing; - io->onMsg = doSyncIOOnMsg; - io->destroy = doSyncIODestroy; - - return io; -} - static int32_t doSyncIOStart(SSyncIO *io) { taosBlockSIGPIPE(); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bd2952505e..75b40dc625 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -18,9 +18,26 @@ #include "syncInt.h" #include "syncRaft.h" -int32_t syncInit() { return 0; } +static int32_t tsNodeRefId = -1; -void syncCleanUp() {} +// ------ local funciton --------- +static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); +static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); +static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); +static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg); +static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg); +static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); +static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); +static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); +static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); +// --------------------------------- + +int32_t syncInit() { + sTrace("syncInit ok"); + return 0; +} + +void syncCleanUp() { sTrace("syncCleanUp ok"); } int64_t syncStart(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); @@ -59,51 +76,57 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { void syncNodeClose(SSyncNode* pSyncNode) { assert(pSyncNode != NULL); - raftClose(pSyncNode->pRaft); free(pSyncNode); } +void syncNodePingAll(SSyncNode* pSyncNode) { sTrace("syncNodePingAll %p ", pSyncNode); } + +void syncNodePingPeers(SSyncNode* pSyncNode) {} + +void syncNodePingSelf(SSyncNode* pSyncNode) {} + +// ------ local funciton --------- static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { - int32_t ret = ths->pRaft->FpPing(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg) { - int32_t ret = ths->pRaft->FpOnPing(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg) { - int32_t ret = ths->pRaft->FpOnPingReply(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg) { - int32_t ret = ths->pRaft->FpRequestVote(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg) { - int32_t ret = ths->pRaft->FpOnRequestVote(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg) { - int32_t ret = ths->pRaft->FpOnRequestVoteReply(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg) { - int32_t ret = ths->pRaft->FpAppendEntries(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg) { - int32_t ret = ths->pRaft->FpOnAppendEntries(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg) { - int32_t ret = ths->pRaft->FpOnAppendEntriesReply(ths->pRaft, pMsg); + int32_t ret = 0; return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c index 9f139730d1..b07c6ea797 100644 --- a/source/libs/sync/src/syncRaft.c +++ b/source/libs/sync/src/syncRaft.c @@ -16,6 +16,8 @@ #include "syncRaft.h" #include "sync.h" +#if 0 + SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm) { SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft)); assert(pRaft != NULL); @@ -64,3 +66,5 @@ static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesRepl int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; } + +#endif \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 964cc78490..1c88be72e7 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -18,24 +18,22 @@ // to complie success: FileIO interface is modified -SRaftStore *raftStoreOpen(const char *path) { return NULL;} +SRaftStore *raftStoreOpen(const char *path) { return NULL; } -static int32_t raftStoreInit(SRaftStore *pRaftStore) { return 0;} +static int32_t raftStoreInit(SRaftStore *pRaftStore) { return 0; } -int32_t raftStoreClose(SRaftStore *pRaftStore) { return 0;} +int32_t raftStoreClose(SRaftStore *pRaftStore) { return 0; } -int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0;} +int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0; } -static bool raftStoreFileExist(char *path) { return 0;} +static bool raftStoreFileExist(char *path) { return 0; } -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } void raftStorePrint(SRaftStore *pRaftStore) {} - - #if 0 SRaftStore *raftStoreOpen(const char *path) { diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 1d050e7094..1ac4357c5a 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -34,19 +34,20 @@ void doSync() { } int main() { - //taosInitLog((char*)"syncEnvTest.log", 100000, 10); + // taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; + int32_t ret; logTest(); - int32_t ret = syncIOStart(); - assert(ret == 0); + // ret = syncIOStart(); + // assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); - doSync(); + // doSync(); while (1) { taosMsleep(1000); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index e62d051946..110c8c472f 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -13,7 +13,7 @@ void logTest() { sFatal("--- sync log test: fatal"); } -void doSync() { +SSyncNode* doSync() { SSyncFSM* pFsm; SSyncInfo syncInfo; @@ -40,10 +40,17 @@ void doSync() { gSyncIO->FpOnPing = pSyncNode->FpOnPing; gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +void timerPingAll(void *param, void *tmrId) { + SSyncNode *pSyncNode = (SSyncNode*)param; + syncNodePingAll(pSyncNode); } int main() { - //taosInitLog((char*)"syncPingTest.log", 100000, 10); + // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; @@ -55,7 +62,9 @@ int main() { ret = syncEnvStart(); assert(ret == 0); - doSync(); + SSyncNode* pSyncNode = doSync(); + + pSyncNode->pPingTimer = syncEnvStartTimer(timerPingAll, 1000, pSyncNode); while (1) { taosMsleep(1000); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 0f72fd822f..0843a48bb8 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,8 +1,8 @@ #include +#include "gtest/gtest.h" #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" -#include "gtest/gtest.h" void *pingFunc(void *param) { SSyncIO *io = (SSyncIO *)param; @@ -15,7 +15,7 @@ void *pingFunc(void *param) { } int main() { - //taosInitLog((char *)"syncTest.log", 100000, 10); + // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; From c9efe3dea71e1be4eb9de260ce0210295edb6f73 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 28 Feb 2022 16:36:57 +0800 Subject: [PATCH 02/82] sync modify timer --- source/libs/sync/inc/syncEnv.h | 4 ++ source/libs/sync/inc/syncIO.h | 13 +++-- source/libs/sync/inc/syncInt.h | 37 ++++++++++--- source/libs/sync/src/syncEnv.c | 2 +- source/libs/sync/src/syncIO.c | 75 +++++++++++++------------- source/libs/sync/src/syncMain.c | 48 +++++++++++++++-- source/libs/sync/test/syncPingTest.cpp | 12 +++-- 7 files changed, 131 insertions(+), 60 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 960afe37b0..c4735d4481 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,12 +28,16 @@ extern "C" { #include "trpc.h" #include "ttimer.h" +#define TIMER_MAX_MS 0x7FFFFFFF + typedef struct SSyncEnv { tmr_h pEnvTickTimer; tmr_h pTimerManager; char name[128]; } SSyncEnv; +extern SSyncEnv *gSyncEnv; + int32_t syncEnvStart(); int32_t syncEnvStop(); diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 49edeb638b..ec887aa855 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -30,10 +30,10 @@ extern "C" { #include "trpc.h" typedef struct SSyncIO { - void * serverRpc; - void * clientRpc; + void *serverRpc; + void *clientRpc; STaosQueue *pMsgQ; - STaosQset * pQset; + STaosQset *pQset; pthread_t tid; int8_t isStart; @@ -56,10 +56,9 @@ typedef struct SSyncIO { extern SSyncIO *gSyncIO; -int32_t syncIOStart(); -int32_t syncIOStop(); -int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); -SSyncIO *syncIOCreate(); +int32_t syncIOStart(); +int32_t syncIOStop(); +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index a8451b3dc4..e02e2d7374 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -88,6 +88,9 @@ typedef struct SyncAppendEntries SyncAppendEntries; struct SyncAppendEntriesReply; typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; +struct SSyncEnv; +typedef struct SSyncEnv SSyncEnv; + typedef struct SRaftId { SyncNodeId addr; SyncGroupId vgId; @@ -103,32 +106,46 @@ typedef struct SSyncNode { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + SNodeInfo me; + SNodeInfo peers[TSDB_MAX_REPLICA]; + int32_t peersNum; + ESyncRole role; SRaftId raftId; SSyncFSM* pFsm; - tmr_h pPingTimer; - tmr_h pElectionTimer; - tmr_h pHeartbeatTimer; + tmr_h pPingTimer; + int32_t pingTimerMS; + uint8_t pingTimerStart; + TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp + uint64_t pingTimerCounter; - int32_t (*FpPing)(struct SSyncNode* ths, const SyncPing* pMsg); + tmr_h pElectTimer; + int32_t electTimerMS; + uint8_t electTimerStart; + TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp + uint64_t electTimerCounter; + tmr_h pHeartbeatTimer; + int32_t heartbeatTimerMS; + uint8_t heartbeatTimerStart; + TAOS_TMR_CALLBACK FpHeartbeatTimer; // Timer Fp + uint64_t heartbeatTimerCounter; + + // callback int32_t (*FpOnPing)(struct SSyncNode* ths, SyncPing* pMsg); int32_t (*FpOnPingReply)(struct SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpRequestVote)(struct SSyncNode* ths, const SyncRequestVote* pMsg); - int32_t (*FpOnRequestVote)(struct SSyncNode* ths, SyncRequestVote* pMsg); int32_t (*FpOnRequestVoteReply)(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpAppendEntries)(struct SSyncNode* ths, const SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntries)(struct SSyncNode* ths, SyncAppendEntries* pMsg); int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + // passed from outside int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); } SSyncNode; @@ -143,6 +160,10 @@ void syncNodePingPeers(SSyncNode* pSyncNode); void syncNodePingSelf(SSyncNode* pSyncNode); +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); + +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 0447fc246a..bfdf2a9c5e 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -59,7 +59,7 @@ static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); + // pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); sTrace("SyncEnv start ok, name:%s", pSyncEnv->name); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 34eaf5fe1e..5c27f9ec3a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -23,17 +23,18 @@ SSyncIO *gSyncIO = NULL; // local function ------------ -static void *syncConsumer(void *param); -static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncTick(void *param, void *tmrId); - static int32_t doSyncIOStart(SSyncIO *io); static int32_t doSyncIOStop(SSyncIO *io); static int32_t doSyncIOPing(SSyncIO *io); static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t doSyncIODestroy(SSyncIO *io); + +static SSyncIO *syncIOCreate(); +static void * syncIOConsumer(void *param); +static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); +static void syncIODoReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncIODoRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncIOTick(void *param, void *tmrId); // ---------------------------- int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } @@ -47,27 +48,10 @@ int32_t syncIOStart() { int32_t syncIOStop() { return 0; } -SSyncIO *syncIOCreate() { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->start = doSyncIOStart; - io->stop = doSyncIOStop; - io->ping = doSyncIOPing; - io->onMsg = doSyncIOOnMsg; - io->destroy = doSyncIODestroy; - - return io; -} - // local function ------------ -static void syncTick(void *param, void *tmrId) { +static void syncIOTick(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; - sDebug("syncTick ... "); + sDebug("syncIOTick ... "); SRpcMsg rpcMsg; rpcMsg.pCont = rpcMallocCont(10); @@ -83,15 +67,15 @@ static void syncTick(void *param, void *tmrId) { taosWriteQitem(io->pMsgQ, pTemp); - bool b = taosTmrReset(syncTick, 1000, io, io->syncTimerManager, io->syncTimer); + bool b = taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, io->syncTimer); assert(b); } -static void *syncConsumer(void *param) { +static void *syncIOConsumer(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); @@ -129,19 +113,19 @@ static void *syncConsumer(void *param) { return NULL; } -static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { +static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { // app shall retrieve the auth info based on meterID from DB or a data file // demo code here only for simple demo int ret = 0; return ret; } -static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sDebug("processResponse ... "); +static void syncIODoReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + sDebug("syncIODoReply ... "); rpcFreeCont(pMsg->pCont); } -static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { +static void syncIODoRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SSyncIO *io = pParent; SRpcMsg *pTemp; @@ -152,6 +136,23 @@ static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { taosWriteQitem(io->pMsgQ, pTemp); } +static SSyncIO *syncIOCreate() { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->start = doSyncIOStart; + io->stop = doSyncIOStop; + io->ping = doSyncIOPing; + io->onMsg = doSyncIOOnMsg; + io->destroy = doSyncIODestroy; + + return io; +} + static int32_t doSyncIOStart(SSyncIO *io) { taosBlockSIGPIPE(); @@ -164,7 +165,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { rpcInit.localPort = 0; rpcInit.label = "SYNC-IO-CLIENT"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; + rpcInit.cfp = syncIODoReply; rpcInit.sessions = 100; rpcInit.idleTime = 100; rpcInit.user = "sync-io"; @@ -187,10 +188,10 @@ static int32_t doSyncIOStart(SSyncIO *io) { rpcInit.localPort = 38000; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; + rpcInit.cfp = syncIODoRequest; rpcInit.sessions = 1000; rpcInit.idleTime = 2 * 1500; - rpcInit.afp = retrieveAuthInfo; + rpcInit.afp = syncIOAuth; rpcInit.parent = io; rpcInit.connType = TAOS_CONN_SERVER; @@ -206,7 +207,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { // start consumer thread { - if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { + if (pthread_create(&io->tid, NULL, syncIOConsumer, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -215,7 +216,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { // start tmr thread io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); + io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 75b40dc625..7cfc470f60 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -15,6 +15,7 @@ #include #include "sync.h" +#include "syncEnv.h" #include "syncInt.h" #include "syncRaft.h" @@ -30,6 +31,7 @@ static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVote static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); +static void syncNodePingTimerCb(void* param, void* tmrId); // --------------------------------- int32_t syncInit() { @@ -58,16 +60,19 @@ void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); + memset(pSyncNode, 0, sizeof(SSyncNode)); + + pSyncNode->pPingTimer = NULL; + pSyncNode->pingTimerMS = 1000; + atomic_store_8(&pSyncNode->pingTimerStart, 0); + pSyncNode->FpPingTimer = syncNodePingTimerCb; + pSyncNode->pingTimerCounter = 0; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; - - pSyncNode->FpPing = doSyncNodePing; pSyncNode->FpOnPing = onSyncNodePing; pSyncNode->FpOnPingReply = onSyncNodePingReply; - pSyncNode->FpRequestVote = doSyncNodeRequestVote; pSyncNode->FpOnRequestVote = onSyncNodeRequestVote; pSyncNode->FpOnRequestVoteReply = onSyncNodeRequestVoteReply; - pSyncNode->FpAppendEntries = doSyncNodeAppendEntries; pSyncNode->FpOnAppendEntries = onSyncNodeAppendEntries; pSyncNode->FpOnAppendEntriesReply = onSyncNodeAppendEntriesReply; @@ -85,6 +90,25 @@ void syncNodePingPeers(SSyncNode* pSyncNode) {} void syncNodePingSelf(SSyncNode* pSyncNode) {} +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { + if (pSyncNode->pPingTimer == NULL) { + pSyncNode->pPingTimer = + taosTmrStart(pSyncNode->FpPingTimer, pSyncNode->pingTimerCounter, pSyncNode, gSyncEnv->pTimerManager); + } else { + taosTmrReset(pSyncNode->FpPingTimer, pSyncNode->pingTimerCounter, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + } + + atomic_store_8(&pSyncNode->pingTimerStart, 1); + return 0; +} + +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { + atomic_store_8(&pSyncNode->pingTimerStart, 0); + pSyncNode->pingTimerCounter = TIMER_MAX_MS; + return 0; +} + // ------ local funciton --------- static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { int32_t ret = 0; @@ -129,4 +153,20 @@ static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; return ret; +} + +static void syncNodePingTimerCb(void* param, void* tmrId) { + SSyncNode* pSyncNode = (SSyncNode*)param; + if (atomic_load_8(&pSyncNode->pingTimerStart)) { + ++(pSyncNode->pingTimerCounter); + // pSyncNode->pingTimerMS += 100; + + sTrace("pSyncNode->pingTimerCounter:%lu, pSyncNode->pingTimerMS:%d, pSyncNode->pPingTimer:%p, tmrId:%p ", + pSyncNode->pingTimerCounter, pSyncNode->pingTimerMS, pSyncNode->pPingTimer, tmrId); + + taosTmrReset(syncNodePingTimerCb, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + + syncNodePingSelf(pSyncNode); + } } \ No newline at end of file diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 110c8c472f..bf0d342ca4 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -44,8 +44,8 @@ SSyncNode* doSync() { return pSyncNode; } -void timerPingAll(void *param, void *tmrId) { - SSyncNode *pSyncNode = (SSyncNode*)param; +void timerPingAll(void* param, void* tmrId) { + SSyncNode* pSyncNode = (SSyncNode*)param; syncNodePingAll(pSyncNode); } @@ -64,7 +64,13 @@ int main() { SSyncNode* pSyncNode = doSync(); - pSyncNode->pPingTimer = syncEnvStartTimer(timerPingAll, 1000, pSyncNode); + ret = syncNodeStartPingTimer(pSyncNode); + assert(ret == 0); + + taosMsleep(5000); + + ret = syncNodeStopPingTimer(pSyncNode); + assert(ret == 0); while (1) { taosMsleep(1000); From eaf6142c793e0ccdfcf116c866d7856df062d6fd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 28 Feb 2022 17:47:47 +0800 Subject: [PATCH 03/82] sync modify timer --- include/libs/sync/sync.h | 4 ++- source/libs/sync/inc/syncEnv.h | 7 +++-- source/libs/sync/inc/syncIO.h | 6 ++-- source/libs/sync/inc/syncInt.h | 3 +- source/libs/sync/inc/syncUtil.h | 40 ++++++++++++++++++++++++++ source/libs/sync/src/syncMain.c | 38 ++++++++++++++++++++++-- source/libs/sync/src/syncUtil.c | 22 ++++++++++++++ source/libs/sync/test/syncPingTest.cpp | 1 + 8 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 source/libs/sync/inc/syncUtil.h create mode 100644 source/libs/sync/src/syncUtil.c diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 5b387852fa..c6f4a4f5c0 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -137,7 +137,9 @@ typedef struct SSyncInfo { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; SSyncFSM* pFsm; - int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); + + void* rpcClient; + int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); } SSyncInfo; diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index c4735d4481..7e60822a28 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -34,17 +34,18 @@ typedef struct SSyncEnv { tmr_h pEnvTickTimer; tmr_h pTimerManager; char name[128]; + } SSyncEnv; -extern SSyncEnv *gSyncEnv; +extern SSyncEnv* gSyncEnv; int32_t syncEnvStart(); int32_t syncEnvStop(); -tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param); +tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void* param); -void syncEnvStopTimer(tmr_h *pTimer); +void syncEnvStopTimer(tmr_h* pTimer); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index ec887aa855..6723fa66c5 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -30,10 +30,10 @@ extern "C" { #include "trpc.h" typedef struct SSyncIO { - void *serverRpc; - void *clientRpc; + void * serverRpc; + void * clientRpc; STaosQueue *pMsgQ; - STaosQset *pQset; + STaosQset * pQset; pthread_t tid; int8_t isStart; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e02e2d7374..b2922a8676 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -146,7 +146,8 @@ typedef struct SSyncNode { int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); // passed from outside - int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); + void* rpcClient; + int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); } SSyncNode; diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h new file mode 100644 index 0000000000..f3d4c2ed07 --- /dev/null +++ b/source/libs/sync/inc/syncUtil.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_UTIL_H +#define _TD_LIBS_SYNC_UTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "syncMessage.h" +#include "taosdef.h" + +void nodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); + +void raftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); + +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_UTIL_H*/ diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7cfc470f60..9b7d068ada 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -18,10 +18,14 @@ #include "syncEnv.h" #include "syncInt.h" #include "syncRaft.h" +#include "syncUtil.h" static int32_t tsNodeRefId = -1; // ------ local funciton --------- +static int32_t doSyncNodeSendMsgById(SRaftId* destRaftId, struct SSyncNode* pSyncNode, SRpcMsg* pMsg); +static int32_t doSyncNodeSendMsgByInfo(SNodeInfo* nodeInfo, struct SSyncNode* pSyncNode, SRpcMsg* pMsg); + static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); @@ -68,7 +72,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpPingTimer = syncNodePingTimerCb; pSyncNode->pingTimerCounter = 0; + pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; + pSyncNode->FpOnPing = onSyncNodePing; pSyncNode->FpOnPingReply = onSyncNodePingReply; pSyncNode->FpOnRequestVote = onSyncNodeRequestVote; @@ -84,7 +90,11 @@ void syncNodeClose(SSyncNode* pSyncNode) { free(pSyncNode); } -void syncNodePingAll(SSyncNode* pSyncNode) { sTrace("syncNodePingAll %p ", pSyncNode); } +void syncNodePingAll(SSyncNode* pSyncNode) { + sTrace("syncNodePingAll %p ", pSyncNode); + SyncPing msg; + doSyncNodePing(pSyncNode, &msg); +} void syncNodePingPeers(SSyncNode* pSyncNode) {} @@ -110,8 +120,30 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { } // ------ local funciton --------- + +static int32_t doSyncNodeSendMsgById(SRaftId* destRaftId, struct SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + raftId2EpSet(destRaftId, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + +static int32_t doSyncNodeSendMsgByInfo(SNodeInfo* nodeInfo, struct SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + nodeInfo2EpSet(nodeInfo, &epSet); + + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { - int32_t ret = 0; + int32_t ret; + for (int i = 0; i < ths->syncCfg.replicaNum; ++i) { + SRpcMsg* rpcMsg; + syncPing2RpcMsg(pMsg, rpcMsg); + doSyncNodeSendMsgByInfo(&ths->syncCfg.nodeInfo[i], ths, rpcMsg); + } + return ret; } @@ -167,6 +199,6 @@ static void syncNodePingTimerCb(void* param, void* tmrId) { taosTmrReset(syncNodePingTimerCb, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); - syncNodePingSelf(pSyncNode); + syncNodePingAll(pSyncNode); } } \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c new file mode 100644 index 0000000000..d68b1def4d --- /dev/null +++ b/source/libs/sync/src/syncUtil.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncUtil.h" + +void nodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet) {} + +void raftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) {} + +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) {} \ No newline at end of file diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index bf0d342ca4..43196bdd1f 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -18,6 +18,7 @@ SSyncNode* doSync() { SSyncInfo syncInfo; syncInfo.vgId = 1; + syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.pFsm = pFsm; snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); From 738e40a37d2619d6dccc07614bf075c7187c8125 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Feb 2022 19:33:23 +0800 Subject: [PATCH 04/82] minor changes --- source/common/src/ttypes.c | 205 ++++++++++++++++++----------------- source/common/src/tvariant.c | 199 +++++++++++++++++----------------- 2 files changed, 205 insertions(+), 199 deletions(-) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index f32698c98a..0c1bafbeb8 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -49,16 +49,16 @@ const int32_t TYPE_BYTES[15] = { } \ } while (0) -static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (data[i] == TSDB_DATA_BOOL_NULL) { (*numOfNull) += 1; @@ -76,9 +76,9 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { (*numOfNull) += 1; @@ -122,9 +122,9 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { (*numOfNull) += 1; @@ -133,15 +133,14 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); } - } static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint16_t *data = (uint16_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -169,9 +168,9 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { (*numOfNull) += 1; @@ -182,12 +181,12 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint32_t *data = (uint32_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -208,16 +207,16 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { (*numOfNull) += 1; @@ -228,12 +227,12 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint64_t *data = (uint64_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -254,91 +253,91 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - float *data = (float *)pData; - float fmin = FLT_MAX; - float fmax = -FLT_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - +static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + float *data = (float *)pData; + float fmin = FLT_MAX; + float fmax = -FLT_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { (*numOfNull) += 1; continue; } - float fv = GET_FLOAT_VAL((const char*)&(data[i])); + float fv = GET_FLOAT_VAL((const char *)&(data[i])); dsum += fv; if (fmin > fv) { fmin = fv; *minIndex = i; } - + if (fmax < fv) { fmax = fv; *maxIndex = i; } } - + SET_DOUBLE_VAL(sum, dsum); SET_DOUBLE_VAL(max, fmax); SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; - double dmin = DBL_MAX; - double dmax = -DBL_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { (*numOfNull) += 1; continue; } - + double dv = 0; - dv = GET_DOUBLE_VAL((const char*)&(data[i])); + dv = GET_DOUBLE_VAL((const char *)&(data[i])); dsum += dv; if (dmin > dv) { dmin = dv; *minIndex = i; } - + if (dmax < dv) { dmax = dv; *maxIndex = i; } } - + SET_DOUBLE_PTR(sum, &dsum); SET_DOUBLE_PTR(max, &dmax); SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - const char* data = pData; +static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char *data = pData; assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(data, TSDB_DATA_TYPE_BINARY)) { (*numOfNull) += 1; } - + data += varDataTLen(data); } - + *sum = 0; *max = 0; *min = 0; @@ -346,19 +345,19 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in *maxIndex = 0; } -static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - const char* data = pData; +static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char *data = pData; assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { (*numOfNull) += 1; } - + data += varDataTLen(data); } - + *sum = 0; *max = 0; *min = 0; @@ -367,21 +366,28 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i } tDataTypeDescriptor tDataTypes[15] = { - {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, - {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, - {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, - {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, - {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, - {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64}, - {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, - {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, - {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, - {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, - {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, - {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, - {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, + getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, + tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, + getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, + tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, + getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, + tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, + getStatics_u64}, }; char tTokenTypeSwitcher[13] = { @@ -400,10 +406,10 @@ char tTokenTypeSwitcher[13] = { TSDB_DATA_TYPE_NCHAR, // TK_NCHAR }; -float floatMin = -FLT_MAX, floatMax = FLT_MAX; +float floatMin = -FLT_MAX, floatMax = FLT_MAX; double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void* getDataMin(int32_t type) { +FORCE_INLINE void *getDataMin(int32_t type) { switch (type) { case TSDB_DATA_TYPE_FLOAT: return &floatMin; @@ -414,7 +420,7 @@ FORCE_INLINE void* getDataMin(int32_t type) { } } -FORCE_INLINE void* getDataMax(int32_t type) { +FORCE_INLINE void *getDataMax(int32_t type) { switch (type) { case TSDB_DATA_TYPE_FLOAT: return &floatMax; @@ -425,17 +431,15 @@ FORCE_INLINE void* getDataMax(int32_t type) { } } -bool isValidDataType(int32_t type) { - return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; -} +bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } -void setVardataNull(void* val, int32_t type) { +void setVardataNull(void *val, int32_t type) { if (type == TSDB_DATA_TYPE_BINARY) { varDataSetLen(val, sizeof(int8_t)); - *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL; + *(uint8_t *)varDataVal(val) = TSDB_DATA_BINARY_NULL; } else if (type == TSDB_DATA_TYPE_NCHAR) { varDataSetLen(val, sizeof(int32_t)); - *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; + *(uint32_t *)varDataVal(val) = TSDB_DATA_NCHAR_NULL; } else { assert(0); } @@ -533,9 +537,8 @@ static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL}; static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL}; static const void *nullValues[] = { - &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, - &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, - &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, &nullFloat, &nullDouble, + &nullBinary, &nullBigInt, &nullNchar, &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; const void *getNullValue(int32_t type) { @@ -590,7 +593,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); break; case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); + *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); break; case TSDB_DATA_TYPE_SMALLINT: *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2); @@ -629,14 +632,14 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf) { switch (type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: { TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); break; } - + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { @@ -652,19 +655,19 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); break; } - + case TSDB_DATA_TYPE_FLOAT: { TSWAP(*(float *)(pLeft), *(float *)(pRight), float); break; } - + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); break; } - + default: { memcpy(buf, pLeft, size); memcpy(pLeft, pRight, size); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ceef12e986..e885d842d7 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -35,9 +35,9 @@ assert(0); \ } while (0) -int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* isSigned) { +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *isSigned) { errno = 0; - char* endPtr = NULL; + char *endPtr = NULL; int32_t index = 0; @@ -63,15 +63,15 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* *isSigned = specifiedSign || (val <= INT64_MAX); if (*isSigned) { - *value = (z[0] == '-')? -val:val; + *value = (z[0] == '-') ? -val : val; } else { - *(uint64_t*) value = val; + *(uint64_t *)value = val; } return 0; } -void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { +void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { int32_t ret = 0; memset(pVar, 0, sizeof(SVariant)); @@ -90,7 +90,7 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_INT:{ + case TSDB_DATA_TYPE_INT: { bool sign = true; int32_t base = 10; @@ -104,11 +104,11 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { ret = toInteger(z, n, base, &pVar->i, &sign); if (ret != 0) { - pVar->nType = -1; // -1 means error type + pVar->nType = -1; // -1 means error type return; } - pVar->nType = (sign)? TSDB_DATA_TYPE_BIGINT:TSDB_DATA_TYPE_UBIGINT; + pVar->nType = (sign) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; break; } case TSDB_DATA_TYPE_DOUBLE: @@ -123,15 +123,15 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { } case TSDB_DATA_TYPE_TIMESTAMP: { assert(0); - pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - break; - } - + pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + break; + } + default: { // nType == 0 means the null value type = TSDB_DATA_TYPE_NULL; } } - + pVar->nType = type; } @@ -196,13 +196,13 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin pVar->d = GET_FLOAT_VAL(pz); break; } - case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length + case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; - + break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method @@ -211,18 +211,18 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin pVar->nLen = (int32_t)len; break; } - + default: pVar->i = GET_INT32_VAL(pz); pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes; } - + pVar->nType = type; } void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; - + if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { tfree(pVar->pz); pVar->nLen = 0; @@ -231,8 +231,8 @@ void taosVariantDestroy(SVariant *pVar) { // NOTE: this is only for string array if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { size_t num = taosArrayGetSize(pVar->arr); - for(size_t i = 0; i < num; i++) { - void* p = taosArrayGetP(pVar->arr, i); + for (size_t i = 0; i < num; i++) { + void *p = taosArrayGetP(pVar->arr, i); free(p); } taosArrayDestroy(pVar->arr); @@ -250,11 +250,11 @@ bool taosVariantIsValid(SVariant *pVar) { void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; - + pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; - char* p = realloc(pDst->pz, len); + char *p = realloc(pDst->pz, len); assert(p); memset(p, 0, len); @@ -263,28 +263,27 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { memcpy(pDst->pz, pSrc->pz, pSrc->nLen); pDst->nLen = pSrc->nLen; return; - } if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); - pDst->arr = taosArrayInit(num, sizeof(char*)); - for(size_t i = 0; i < num; i++) { - char* p = (char*)taosArrayGetP(pSrc->arr, i); - char* n = strdup(p); + pDst->arr = taosArrayInit(num, sizeof(char *)); + for (size_t i = 0; i < num; i++) { + char *p = (char *)taosArrayGetP(pSrc->arr, i); + char *n = strdup(p); taosArrayPush(pDst->arr, &n); } } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - size_t num = taosArrayGetSize(pSrc->arr); - pDst->arr = taosArrayInit(num, sizeof(int64_t)); - pDst->nLen = pSrc->nLen; - assert(pSrc->nLen == num); - for(size_t i = 0; i < num; i++) { - int64_t *p = taosArrayGet(pSrc->arr, i); - taosArrayPush(pDst->arr, p); - } + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(int64_t)); + pDst->nLen = pSrc->nLen; + assert(pSrc->nLen == num); + for (size_t i = 0; i < num; i++) { + int64_t *p = taosArrayGet(pSrc->arr, i); + taosArrayPush(pDst->arr, p); + } } if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) { @@ -292,7 +291,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { } } -int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) { +int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { return 0; } @@ -309,39 +308,39 @@ int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) { if (p1->nLen == p2->nLen) { return memcmp(p1->pz, p2->pz, p1->nLen); } else { - return p1->nLen > p2->nLen? 1:-1; + return p1->nLen > p2->nLen ? 1 : -1; } } else if (p1->nType == TSDB_DATA_TYPE_FLOAT || p1->nType == TSDB_DATA_TYPE_DOUBLE) { if (p1->d == p2->d) { return 0; } else { - return p1->d > p2->d? 1:-1; + return p1->d > p2->d ? 1 : -1; } } else if (IS_UNSIGNED_NUMERIC_TYPE(p1->nType)) { if (p1->u == p2->u) { return 0; } else { - return p1->u > p2->u? 1:-1; + return p1->u > p2->u ? 1 : -1; } } else { if (p1->i == p2->i) { return 0; } else { - return p1->i > p2->i? 1:-1; + return p1->i > p2->i ? 1 : -1; } } } int32_t taosVariantToString(SVariant *pVar, char *dst) { if (pVar == NULL || dst == NULL) return 0; - + switch (pVar->nType) { case TSDB_DATA_TYPE_BINARY: { int32_t len = sprintf(dst, "\'%s\'", pVar->pz); assert(len <= pVar->nLen + sizeof("\'") * 2); // two more chars return len; } - + case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); @@ -350,7 +349,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { dst[len + 1] = 0; return len + 1; } - + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -359,7 +358,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: return sprintf(dst, "%d", (int32_t)pVar->i); - + case TSDB_DATA_TYPE_BIGINT: return sprintf(dst, "%" PRId64, pVar->i); case TSDB_DATA_TYPE_UBIGINT: @@ -367,7 +366,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: return sprintf(dst, "%.9lf", pVar->d); - + default: return 0; } @@ -399,27 +398,27 @@ static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { const int32_t INITIAL_ALLOC_SIZE = 40; - char * pBuf = NULL; + char *pBuf = NULL; // it is a in-place convert type for SVariant, local buffer is needed if (*pDest == pVariant->pz) { pBuf = calloc(1, INITIAL_ALLOC_SIZE); } - + if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE; if (pBuf != NULL) { if (newSize >= INITIAL_ALLOC_SIZE) { pBuf = realloc(pBuf, newSize + 1); } - + taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); free(pVariant->wpz); pBuf[newSize] = 0; } else { taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); } - + } else { if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i); @@ -431,26 +430,26 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { setNull(pBuf == NULL ? *pDest : pBuf, TSDB_DATA_TYPE_BINARY, 0); } } - + if (pBuf != NULL) { *pDest = pBuf; } - + *pDestSize = (int32_t)strlen(*pDest); return 0; } static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { char tmpBuf[40] = {0}; - - char * pDst = tmpBuf; + + char *pDst = tmpBuf; int32_t nLen = 0; // convert the number to string, than convert it to wchar string. if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { nLen = sprintf(pDst, "%" PRId64, pVariant->i); } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { - nLen = sprintf(pDst, "%"PRIu64, pVariant->u); + nLen = sprintf(pDst, "%" PRIu64, pVariant->u); } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { nLen = sprintf(pDst, "%lf", pVariant->d); } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { @@ -459,10 +458,10 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { nLen = sprintf(pDst, "%s", (pVariant->i == TSDB_TRUE) ? "TRUE" : "FALSE"); } - + if (*pDest == pVariant->pz) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { tfree(pWStr); return -1; @@ -472,14 +471,14 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { free(pVariant->wpz); } - + pVariant->wpz = pWStr; *pDestSize = twcslen(pVariant->wpz); - + // shrink the allocate memory, no need to check here. - char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE); + char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); - + pVariant->wpz = (wchar_t *)tmp; } else { int32_t output = 0; @@ -493,21 +492,22 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { *pDestSize = output; } } - + return 0; } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { -// SToken stoken = {.z = pStr, .n = len}; -// if (TK_ILLEGAL == tGetNumericStringType(&stoken)) { -// return -1; -// } -// -// *value = strtod(pStr, NULL); + // SToken stoken = {.z = pStr, .n = len}; + // if (TK_ILLEGAL == tGetNumericStringType(&stoken)) { + // return -1; + // } + // + // *value = strtod(pStr, NULL); return 0; } -static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) { +static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned, + bool releaseVariantPtr, bool *converted) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { setNull((char *)result, type, tDataTypes[type].bytes); return 0; @@ -645,7 +645,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { if ((ret = convertToBoolImpl(pVariant->pz, pVariant->nLen)) < 0) { return ret; } - + *pDest = ret; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { int32_t ret = 0; @@ -656,7 +656,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *pDest = TSDB_DATA_BOOL_NULL; } - + assert(*pDest == TSDB_TRUE || *pDest == TSDB_FALSE || *pDest == TSDB_DATA_BOOL_NULL); return 0; } @@ -665,11 +665,12 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType * to column type defined in schema */ -int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) { +int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, + char *extInfo) { if (converted) { *converted = false; } - + if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) { return -1; } @@ -686,25 +687,25 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *(int8_t *)payload = (int8_t)result; break; } - + case TSDB_DATA_TYPE_TINYINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo); return -1; } - *((int8_t *)payload) = (int8_t) result; + *((int8_t *)payload) = (int8_t)result; break; } case TSDB_DATA_TYPE_UTINYINT: { - if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo); return -1; } - *((uint8_t *)payload) = (uint8_t) result; + *((uint8_t *)payload) = (uint8_t)result; break; } - + case TSDB_DATA_TYPE_SMALLINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo); @@ -722,7 +723,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *((uint16_t *)payload) = (uint16_t)result; break; } - + case TSDB_DATA_TYPE_INT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo); @@ -740,7 +741,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *((uint32_t *)payload) = (uint32_t)result; break; } - + case TSDB_DATA_TYPE_BIGINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); @@ -775,19 +776,20 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc if (converted) { *converted = true; } - + if (value > FLT_MAX || value < -FLT_MAX) { SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo); return -1; } SET_FLOAT_VAL(payload, value); } - } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || + IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { if (converted) { *converted = true; } - - if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) { + + if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) { SET_EXT_INFO(converted, pVariant->i, -FLT_MAX, FLT_MAX, extInfo); return -1; } @@ -797,12 +799,12 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc if (converted) { *converted = true; } - - if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) { + + if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) { SET_EXT_INFO(converted, pVariant->d, -FLT_MAX, FLT_MAX, extInfo); return -1; } - + SET_FLOAT_VAL(payload, pVariant->d); } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL; @@ -831,7 +833,8 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc SET_DOUBLE_VAL(payload, value); } - } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || + IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { SET_DOUBLE_VAL(payload, pVariant->i); } else if (IS_FLOAT_TYPE(pVariant->nType)) { SET_DOUBLE_VAL(payload, pVariant->d); @@ -847,11 +850,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc break; } - + case TSDB_DATA_TYPE_BINARY: { if (!includeLengthPrefix) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - *(uint8_t*) payload = TSDB_DATA_BINARY_NULL; + *(uint8_t *)payload = TSDB_DATA_BINARY_NULL; } else { if (pVariant->nType != TSDB_DATA_TYPE_BINARY) { toBinary(pVariant, &payload, &pVariant->nLen); @@ -918,11 +921,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc assert(p == varDataVal(payload)); } } - + break; } } - + return 0; } @@ -944,13 +947,13 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { if (pVariant == NULL || pVariant->nType == 0) { // value is not set return 0; } - + switch (type) { case TSDB_DATA_TYPE_BOOL: { // bool if (convertToBool(pVariant, &pVariant->i) < 0) { return -1; } - + pVariant->nType = type; break; } @@ -971,7 +974,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { free(pVariant->pz); return -1; } - + free(pVariant->pz); pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { @@ -981,14 +984,14 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { free(pVariant->pz); return -1; } - + free(pVariant->pz); pVariant->d = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - double tmp = (double) pVariant->i; + double tmp = (double)pVariant->i; pVariant->d = tmp; } - + pVariant->nType = TSDB_DATA_TYPE_DOUBLE; break; } @@ -1009,6 +1012,6 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { break; } } - + return 0; } \ No newline at end of file From 3a6eecbabfc8a0edac16fcbe8b58038cda5c2bab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Mar 2022 13:25:24 +0800 Subject: [PATCH 05/82] [td-13039] refactor. --- source/libs/executor/inc/executorimpl.h | 4 ++-- source/libs/executor/src/tlinearhash.c | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 4e7076530b..b3d8542166 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -233,7 +233,7 @@ typedef struct STaskAttr { SArray* pUdfInfo; // no need to free } STaskAttr; -typedef int32_t (*__optr_prepare_fn_t)(void* param); +typedef int32_t (*__optr_open_fn_t)(void* param); typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); @@ -318,7 +318,7 @@ typedef struct SOperatorInfo { struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_prepare_fn_t prepareFn; + __optr_open_fn_t prepareFn; __operator_fn_t exec; __optr_cleanup_fn_t cleanupFn; } SOperatorInfo; diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 3a58253d81..312fd66b40 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -29,8 +29,8 @@ typedef struct SLHashBucket { typedef struct SLHashObj { SDiskbasedBuf *pBuf; _hash_fn_t hashFn; - int32_t tuplesPerPage; SLHashBucket **pBucket; // entry list + int32_t tuplesPerPage; int32_t numOfAlloc; // number of allocated bucket ptr slot int32_t bits; // the number of bits used in hash int32_t numOfBuckets; // the number of buckets @@ -142,7 +142,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket pBucket->size -= 1; } -static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { +static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); if (numOfPages <= 1) { return; @@ -253,6 +253,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ return NULL; } + // disable compress when flushing to disk setBufPageCompressOnDisk(pHashObj->pBuf, false); /** @@ -367,7 +368,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data releaseBufPage(pHashObj->pBuf, p); } - doCompressBucketPages(pHashObj, pBucket); + doTrimBucketPages(pHashObj, pBucket); } return TSDB_CODE_SUCCESS; From d57320031fd92607f7401ca59fccdf049e8ea1d1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 1 Mar 2022 15:16:44 +0800 Subject: [PATCH 06/82] sync ping --- include/libs/sync/sync.h | 14 ++-- source/libs/sync/inc/syncInt.h | 16 ++-- source/libs/sync/inc/syncMessage.h | 72 +++++++++++----- source/libs/sync/inc/syncUtil.h | 23 ++++- source/libs/sync/src/syncMain.c | 130 +++++++++++++++++------------ source/libs/sync/src/syncMessage.c | 130 ++++++++++++++++++++++++++++- source/libs/sync/src/syncUtil.c | 61 +++++++++++++- 7 files changed, 348 insertions(+), 98 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index c6f4a4f5c0..cd04783dbc 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -71,15 +71,15 @@ typedef struct SSyncFSM { // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); // when log entry is updated by a new one, FpRollBackCb is called // user can do something to roll back. for example, delete data from tsdb, or just ignore it - void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); // user should implement this function, use "data" to take snapshot into "snapshot" int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); @@ -95,10 +95,10 @@ typedef struct SSyncLogStore { void* data; // append one log entry - int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); + int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SRpcMsg* pBuf); // get one log entry, user need to free pBuf->data - int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); + int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SRpcMsg* pBuf); // update log store commit index with "index" int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); @@ -153,8 +153,8 @@ int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); -int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); ESyncState syncGetMyRole(int64_t rid); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index b2922a8676..82108acf7b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -92,8 +92,8 @@ struct SSyncEnv; typedef struct SSyncEnv SSyncEnv; typedef struct SRaftId { - SyncNodeId addr; - SyncGroupId vgId; + SyncNodeId addr; // typedef uint64_t SyncNodeId; + SyncGroupId vgId; // typedef int32_t SyncGroupId; } SRaftId; typedef struct SSyncNode { @@ -133,17 +133,17 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // callback - int32_t (*FpOnPing)(struct SSyncNode* ths, SyncPing* pMsg); + int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); - int32_t (*FpOnPingReply)(struct SSyncNode* ths, SyncPingReply* pMsg); + int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpOnRequestVote)(struct SSyncNode* ths, SyncRequestVote* pMsg); + int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); - int32_t (*FpOnRequestVoteReply)(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); + int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpOnAppendEntries)(struct SSyncNode* ths, SyncAppendEntries* pMsg); + int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); // passed from outside void* rpcClient; diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 2ee5e0109c..0de8e8cf4b 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -27,6 +27,7 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" +// encode as uint64 typedef enum ESyncMessageType { SYNC_PING = 0, SYNC_PING_REPLY, @@ -38,29 +39,47 @@ typedef enum ESyncMessageType { SYNC_APPEND_ENTRIES_REPLY, } ESyncMessageType; +/* +typedef struct SRaftId { + SyncNodeId addr; // typedef uint64_t SyncNodeId; + SyncGroupId vgId; // typedef int32_t SyncGroupId; +} SRaftId; +*/ + typedef struct SyncPing { - ESyncMessageType msgType; - const SSyncBuffer *pData; -} SyncPing, RaftPing; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + uint32_t dataLen; + char* data; +} SyncPing; + +#define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) typedef struct SyncPingReply { - ESyncMessageType msgType; - const SSyncBuffer *pData; -} SyncPingReply, RaftPingReply; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + uint32_t dataLen; + char* data; +} SyncPingReply; typedef struct SyncClientRequest { - ESyncMessageType msgType; - const SSyncBuffer *pData; - int64_t seqNum; - bool isWeak; -} SyncClientRequest, RaftClientRequest; + ESyncMessageType msgType; + char* data; + uint32_t dataLen; + int64_t seqNum; + bool isWeak; +} SyncClientRequest; typedef struct SyncClientRequestReply { - ESyncMessageType msgType; - int32_t errCode; - const SSyncBuffer *pErrMsg; - const SSyncBuffer *pLeaderHint; -} SyncClientRequestReply, RaftClientRequestReply; + ESyncMessageType msgType; + int32_t errCode; + SSyncBuffer* pErrMsg; + SSyncBuffer* pLeaderHint; +} SyncClientRequestReply; typedef struct SyncRequestVote { ESyncMessageType msgType; @@ -69,7 +88,7 @@ typedef struct SyncRequestVote { SyncGroupId vgId; SyncIndex lastLogIndex; SyncTerm lastLogTerm; -} SyncRequestVote, RaftRequestVote; +} SyncRequestVote; typedef struct SyncRequestVoteReply { ESyncMessageType msgType; @@ -77,7 +96,7 @@ typedef struct SyncRequestVoteReply { SyncNodeId nodeId; SyncGroupId vgId; bool voteGranted; -} SyncRequestVoteReply, RaftRequestVoteReply; +} SyncRequestVoteReply; typedef struct SyncAppendEntries { ESyncMessageType msgType; @@ -86,9 +105,9 @@ typedef struct SyncAppendEntries { SyncIndex prevLogIndex; SyncTerm prevLogTerm; int32_t entryCount; - SSyncRaftEntry * logEntries; + SSyncRaftEntry* logEntries; SyncIndex commitIndex; -} SyncAppendEntries, RaftAppendEntries; +} SyncAppendEntries; typedef struct SyncAppendEntriesReply { ESyncMessageType msgType; @@ -96,7 +115,18 @@ typedef struct SyncAppendEntriesReply { SyncNodeId nodeId; bool success; SyncIndex matchIndex; -} SyncAppendEntriesReply, RaftAppendEntriesReply; +} SyncAppendEntriesReply; + +// ---- message build ---- +SyncPing* syncPingBuild(uint32_t dataLen); + +void syncPingDestroy(SyncPing* pSyncPing); + +void syncPingSerialize(const SyncPing* pSyncPing, char* buf, uint32_t bufLen); + +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing); + +void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index f3d4c2ed07..93d2c12525 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -27,11 +27,28 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" -void nodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); +// ---- encode / decode -void raftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); +uint64_t syncUtilAddr2U64(const char* host, uint16_t port); -void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); +void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port); + +void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); + +void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); + +void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); + +// ---- SSyncBuffer ---- +#if 0 +void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len); + +void syncUtilbufDestroy(SSyncBuffer* syncBuf); + +void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest); + +void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); +#endif #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9b7d068ada..4d9e5887cd 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -23,19 +23,20 @@ static int32_t tsNodeRefId = -1; // ------ local funciton --------- -static int32_t doSyncNodeSendMsgById(SRaftId* destRaftId, struct SSyncNode* pSyncNode, SRpcMsg* pMsg); -static int32_t doSyncNodeSendMsgByInfo(SNodeInfo* nodeInfo, struct SSyncNode* pSyncNode, SRpcMsg* pMsg); - -static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); -static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); -static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); -static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg); -static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg); -static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); -static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); -static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); -static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); +static int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +static int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); static void syncNodePingTimerCb(void* param, void* tmrId); + +static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg); +static int32_t syncNodeRequestVote(SSyncNode* ths, const SyncRequestVote* pMsg); +static int32_t syncNodeAppendEntries(SSyncNode* ths, const SyncAppendEntries* pMsg); + +static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); +static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +static int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); +static int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +static int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); +static int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); // --------------------------------- int32_t syncInit() { @@ -55,7 +56,9 @@ void syncStop(int64_t rid) {} int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } -int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { return 0; } +// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { return 0; } + +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak) { return 0; } ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } @@ -75,12 +78,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; - pSyncNode->FpOnPing = onSyncNodePing; - pSyncNode->FpOnPingReply = onSyncNodePingReply; - pSyncNode->FpOnRequestVote = onSyncNodeRequestVote; - pSyncNode->FpOnRequestVoteReply = onSyncNodeRequestVoteReply; - pSyncNode->FpOnAppendEntries = onSyncNodeAppendEntries; - pSyncNode->FpOnAppendEntriesReply = onSyncNodeAppendEntriesReply; + pSyncNode->FpOnPing = syncNodeOnPingCb; + pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; return pSyncNode; } @@ -92,13 +95,35 @@ void syncNodeClose(SSyncNode* pSyncNode) { void syncNodePingAll(SSyncNode* pSyncNode) { sTrace("syncNodePingAll %p ", pSyncNode); - SyncPing msg; - doSyncNodePing(pSyncNode, &msg); + int32_t ret = 0; + for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { + SyncPing* pSyncPing; + SRaftId raftId; + syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &raftId); + ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + assert(ret == 0); + } } -void syncNodePingPeers(SSyncNode* pSyncNode) {} +void syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncPing* pSyncPing; + SRaftId raftId; + syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &raftId); + ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + assert(ret == 0); + } +} -void syncNodePingSelf(SSyncNode* pSyncNode) {} +void syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pSyncPing; + SRaftId raftId; + syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &raftId); + ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + assert(ret == 0); +} int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { if (pSyncNode->pPingTimer == NULL) { @@ -120,69 +145,64 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { } // ------ local funciton --------- +static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + int32_t ret = 0; + SRpcMsg* rpcMsg; + syncPing2RpcMsg(pMsg, rpcMsg); + syncNodeSendMsgById(destRaftId, pSyncNode, rpcMsg); + return ret; +} -static int32_t doSyncNodeSendMsgById(SRaftId* destRaftId, struct SSyncNode* pSyncNode, SRpcMsg* pMsg) { +static int32_t syncNodeRequestVote(SSyncNode* ths, const SyncRequestVote* pMsg) { + int32_t ret = 0; + return ret; +} + +static int32_t syncNodeAppendEntries(SSyncNode* ths, const SyncAppendEntries* pMsg) { + int32_t ret = 0; + return ret; +} + +static int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - raftId2EpSet(destRaftId, &epSet); + syncUtilraftId2EpSet(destRaftId, &epSet); pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); return 0; } -static int32_t doSyncNodeSendMsgByInfo(SNodeInfo* nodeInfo, struct SSyncNode* pSyncNode, SRpcMsg* pMsg) { +static int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - nodeInfo2EpSet(nodeInfo, &epSet); - + syncUtilnodeInfo2EpSet(nodeInfo, &epSet); pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); return 0; } -static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { - int32_t ret; - for (int i = 0; i < ths->syncCfg.replicaNum; ++i) { - SRpcMsg* rpcMsg; - syncPing2RpcMsg(pMsg, rpcMsg); - doSyncNodeSendMsgByInfo(&ths->syncCfg.nodeInfo[i], ths, rpcMsg); - } - - return ret; -} - -static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg) { +static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; return ret; } -static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg) { +static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { int32_t ret = 0; return ret; } -static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg) { +static int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; return ret; } -static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg) { +static int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; return ret; } -static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg) { +static int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; return ret; } -static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg) { - int32_t ret = 0; - return ret; -} - -static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg) { - int32_t ret = 0; - return ret; -} - -static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg) { +static int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 8937303725..baff4ed50b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -15,5 +15,133 @@ #include "syncMessage.h" #include "syncRaft.h" +#include "tcoding.h" -void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file +void onMessage(SRaft* pRaft, void* pMsg) {} + +// ---- message build ---- +SyncPing* syncPingBuild(uint32_t dataLen) { + uint32_t bytes = SYNC_PING_FIX_LEN + dataLen; + SyncPing* pSyncPing = malloc(bytes); + memset(pSyncPing, 0, bytes); + pSyncPing->bytes = bytes; + pSyncPing->msgType = SYNC_PING; + pSyncPing->dataLen = dataLen; +} + +void syncPingDestroy(SyncPing* pSyncPing) { + if (pSyncPing != NULL) { + free(pSyncPing); + } +} + +void syncPingSerialize(const SyncPing* pSyncPing, char* buf, uint32_t bufLen) { + assert(pSyncPing->bytes <= bufLen); + memcpy(buf, pSyncPing, pSyncPing->bytes); +} + +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { + uint32_t* pU32 = (uint32_t*)buf; + uint32_t bytes = *pU32; + pSyncPing = (SyncPing*)malloc(bytes); + memcpy(pSyncPing, buf, len); + assert(len == pSyncPing->bytes); + assert(pSyncPing->bytes == SYNC_PING_FIX_LEN + pSyncPing->dataLen); +} + +void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg) { + pRpcMsg->msgType = pSyncPing->msgType; + uint32_t bufLen = pSyncPing->bytes; + char* buf = malloc(bufLen); + syncPingSerialize(pSyncPing, buf, bufLen); + pRpcMsg->contLen = bufLen; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + memcpy(pRpcMsg->pCont, buf, pRpcMsg->contLen); + free(buf); +} + +/* +typedef struct SRaftId { + SyncNodeId addr; // typedef uint64_t SyncNodeId; + SyncGroupId vgId; // typedef int32_t SyncGroupId; +} SRaftId; + +typedef struct SyncPing { + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + uint32_t dataLen; + char* data; +} SyncPing; +*/ + +/* +void syncPingSerialize(const SyncPing* pSyncPing, char** ppBuf, uint32_t* bufLen) { + *bufLen = sizeof(SyncPing) + pSyncPing->dataLen; + *ppBuf = (char*)malloc(*bufLen); + void* pStart = *ppBuf; + uint32_t allBytes = *bufLen; + + int len = 0; + len = taosEncodeFixedU32(&pStart, pSyncPing->msgType); + allBytes -= len; + assert(len > 0); + pStart += len; + + len = taosEncodeFixedU64(&pStart, pSyncPing->srcId.addr); + allBytes -= len; + assert(len > 0); + pStart += len; + + len = taosEncodeFixedI32(&pStart, pSyncPing->srcId.vgId); + allBytes -= len; + assert(len > 0); + pStart += len; + + len = taosEncodeFixedU64(&pStart, pSyncPing->destId.addr); + allBytes -= len; + assert(len > 0); + pStart += len; + + len = taosEncodeFixedI32(&pStart, pSyncPing->destId.vgId); + allBytes -= len; + assert(len > 0); + pStart += len; + + len = taosEncodeFixedU32(&pStart, pSyncPing->dataLen); + allBytes -= len; + assert(len > 0); + pStart += len; + + memcpy(pStart, pSyncPing->data, pSyncPing->dataLen); + allBytes -= pSyncPing->dataLen; + assert(allBytes == 0); +} + + +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { + void* pStart = (void*)buf; + uint64_t u64; + int32_t i32; + uint32_t u32; + + pStart = taosDecodeFixedU64(pStart, &u64); + pSyncPing->msgType = u64; + + pStart = taosDecodeFixedU64(pStart, &u64); + pSyncPing->srcId.addr = u64; + + pStart = taosDecodeFixedI32(pStart, &i32); + pSyncPing->srcId.vgId = i32; + + pStart = taosDecodeFixedU64(pStart, &u64); + pSyncPing->destId.addr = u64; + + pStart = taosDecodeFixedI32(pStart, &i32); + pSyncPing->destId.vgId = i32; + + pStart = taosDecodeFixedU32(pStart, &u32); + pSyncPing->dataLen = u32; +} +*/ \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index d68b1def4d..080840bbf6 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -14,9 +14,64 @@ */ #include "syncUtil.h" +#include +#include +#include -void nodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet) {} +// ---- encode / decode +uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { + uint64_t u64; + uint32_t hostU32 = (uint32_t)inet_addr(host); + assert(hostU32 != (uint32_t)-1); + u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); + return u64; +} -void raftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) {} +void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port) { + uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); -void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) {} \ No newline at end of file + struct in_addr addr; + addr.s_addr = hostU32; + snprintf(host, len, "%s", inet_ntoa(addr)); + *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); +} + +void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet) { + pEpSet->inUse = 0; + addEpIntoEpSet(pEpSet, pNodeInfo->nodeFqdn, pNodeInfo->nodePort); +} + +void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { + char host[TSDB_FQDN_LEN]; + uint16_t port; + + syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); + pEpSet->inUse = 0; + addEpIntoEpSet(pEpSet, host, port); +} + +void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { + raftId->addr = syncUtilAddr2U64(pNodeInfo->nodeFqdn, pNodeInfo->nodePort); + raftId->vgId = vgId; +} + +// ---- SSyncBuffer ----- +#if 0 +void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { + syncBuf->len = len; + syncBuf->data = malloc(syncBuf->len); +} + +void syncUtilbufDestroy(SSyncBuffer* syncBuf) { free(syncBuf->data); } + +void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { + dest->len = src->len; + dest->data = src->data; +} + +void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { + dest->len = src->len; + dest->data = malloc(dest->len); + memcpy(dest->data, src->data, dest->len); +} +#endif \ No newline at end of file From 48bed2020c6f24492852e26150d0f846cf3e6f0c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 1 Mar 2022 16:19:42 +0800 Subject: [PATCH 07/82] sync encode test --- source/libs/sync/inc/syncMessage.h | 7 ++- source/libs/sync/src/syncMessage.c | 39 ++++++++------- source/libs/sync/test/CMakeLists.txt | 14 ++++++ source/libs/sync/test/syncEncodeTest.cpp | 60 ++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 18 deletions(-) create mode 100644 source/libs/sync/test/syncEncodeTest.cpp diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 0de8e8cf4b..5f73601576 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "cJSON.h" #include "sync.h" #include "syncRaftEntry.h" #include "taosdef.h" @@ -52,7 +53,7 @@ typedef struct SyncPing { SRaftId srcId; SRaftId destId; uint32_t dataLen; - char* data; + char data[]; } SyncPing; #define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) @@ -63,7 +64,7 @@ typedef struct SyncPingReply { SRaftId srcId; SRaftId destId; uint32_t dataLen; - char* data; + char data[]; } SyncPingReply; typedef struct SyncClientRequest { @@ -128,6 +129,8 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing); void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg); +cJSON* syncPing2Json(const SyncPing* pSyncPing); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index baff4ed50b..dc584c4e7b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -41,9 +41,11 @@ void syncPingSerialize(const SyncPing* pSyncPing, char* buf, uint32_t bufLen) { } void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { + /* uint32_t* pU32 = (uint32_t*)buf; uint32_t bytes = *pU32; pSyncPing = (SyncPing*)malloc(bytes); + */ memcpy(pSyncPing, buf, len); assert(len == pSyncPing->bytes); assert(pSyncPing->bytes == SYNC_PING_FIX_LEN + pSyncPing->dataLen); @@ -60,23 +62,28 @@ void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg) { free(buf); } -/* -typedef struct SRaftId { - SyncNodeId addr; // typedef uint64_t SyncNodeId; - SyncGroupId vgId; // typedef int32_t SyncGroupId; -} SRaftId; +cJSON* syncPing2Json(const SyncPing* pSyncPing) { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pSyncPing->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pSyncPing->msgType); -typedef struct SyncPing { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - uint32_t dataLen; - char* data; -} SyncPing; -*/ + cJSON* pSrcId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pSrcId, "addr", pSyncPing->srcId.addr); + cJSON_AddNumberToObject(pSrcId, "vgId", pSyncPing->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); -/* + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pSyncPing->destId.addr); + cJSON_AddNumberToObject(pDestId, "vgId", pSyncPing->destId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pDestId); + + cJSON_AddNumberToObject(pRoot, "dataLen", pSyncPing->dataLen); + cJSON_AddStringToObject(pRoot, "data", pSyncPing->data); + + return pRoot; +} + +#if 0 void syncPingSerialize(const SyncPing* pSyncPing, char** ppBuf, uint32_t* bufLen) { *bufLen = sizeof(SyncPing) + pSyncPing->dataLen; *ppBuf = (char*)malloc(*bufLen); @@ -144,4 +151,4 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { pStart = taosDecodeFixedU32(pStart, &u32); pSyncPing->dataLen = u32; } -*/ \ No newline at end of file +#endif \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index e655ac01be..8e1c99d179 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(syncTest "") add_executable(syncEnvTest "") add_executable(syncPingTest "") +add_executable(syncEncodeTest "") target_sources(syncTest @@ -15,6 +16,10 @@ target_sources(syncPingTest PRIVATE "syncPingTest.cpp" ) +target_sources(syncEncodeTest + PRIVATE + "syncEncodeTest.cpp" +) target_include_directories(syncTest @@ -32,6 +37,11 @@ target_include_directories(syncPingTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEncodeTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -46,6 +56,10 @@ target_link_libraries(syncPingTest sync gtest_main ) +target_link_libraries(syncEncodeTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp new file mode 100644 index 0000000000..1e8007e422 --- /dev/null +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -0,0 +1,60 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +#define PING_MSG_LEN 20 + +int main() { + // taosInitLog((char*)"syncPingTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + char msg[PING_MSG_LEN]; + snprintf(msg, sizeof(msg), "%s", "test ping"); + SyncPing* pSyncPing = syncPingBuild(PING_MSG_LEN); + pSyncPing->srcId.addr = 1; + pSyncPing->srcId.vgId = 2; + pSyncPing->destId.addr = 3; + pSyncPing->destId.vgId = 4; + memcpy(pSyncPing->data, msg, PING_MSG_LEN); + + { + cJSON* pJson = syncPing2Json(pSyncPing); + char* serialized = cJSON_Print(pJson); + printf("SyncPing: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pSyncPing->bytes; + char* buf = (char*)malloc(bufLen); + syncPingSerialize(pSyncPing, buf, bufLen); + + SyncPing* pSyncPing2 = (SyncPing*)malloc(pSyncPing->bytes); + syncPingDeserialize(buf, bufLen, pSyncPing2); + + { + cJSON* pJson = syncPing2Json(pSyncPing2); + char* serialized = cJSON_Print(pJson); + printf("SyncPing2: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncPingDestroy(pSyncPing); + syncPingDestroy(pSyncPing2); + free(buf); + + return 0; +} From 12050c9a2a0734290f83821e33ca2f283f3fb8ae Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 1 Mar 2022 16:42:37 +0800 Subject: [PATCH 08/82] sync encode test --- source/libs/sync/inc/syncMessage.h | 2 + source/libs/sync/src/syncMessage.c | 12 +++--- source/libs/sync/test/syncEncodeTest.cpp | 55 +++++++++++++++++++++--- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 5f73601576..121314c589 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -129,6 +129,8 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing); void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg); +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pSyncPing); + cJSON* syncPing2Json(const SyncPing* pSyncPing); #ifdef __cplusplus diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index dc584c4e7b..80dd184018 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -53,13 +53,13 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg) { pRpcMsg->msgType = pSyncPing->msgType; - uint32_t bufLen = pSyncPing->bytes; - char* buf = malloc(bufLen); - syncPingSerialize(pSyncPing, buf, bufLen); - pRpcMsg->contLen = bufLen; + pRpcMsg->contLen = pSyncPing->bytes; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - memcpy(pRpcMsg->pCont, buf, pRpcMsg->contLen); - free(buf); + syncPingSerialize(pSyncPing, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pSyncPing) { + syncPingDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pSyncPing); } cJSON* syncPing2Json(const SyncPing* pSyncPing) { diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 1e8007e422..029e2bd496 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -15,10 +15,8 @@ void logTest() { #define PING_MSG_LEN 20 -int main() { - // taosInitLog((char*)"syncPingTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; +void test1() { + sTrace("test1: ----"); char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); @@ -40,7 +38,7 @@ int main() { uint32_t bufLen = pSyncPing->bytes; char* buf = (char*)malloc(bufLen); syncPingSerialize(pSyncPing, buf, bufLen); - + SyncPing* pSyncPing2 = (SyncPing*)malloc(pSyncPing->bytes); syncPingDeserialize(buf, bufLen, pSyncPing2); @@ -55,6 +53,53 @@ int main() { syncPingDestroy(pSyncPing); syncPingDestroy(pSyncPing2); free(buf); +} + +void test2() { + sTrace("test2: ----"); + + char msg[PING_MSG_LEN]; + snprintf(msg, sizeof(msg), "%s", "hello raft"); + SyncPing* pSyncPing = syncPingBuild(PING_MSG_LEN); + pSyncPing->srcId.addr = 100; + pSyncPing->srcId.vgId = 200; + pSyncPing->destId.addr = 300; + pSyncPing->destId.vgId = 400; + memcpy(pSyncPing->data, msg, PING_MSG_LEN); + + { + cJSON* pJson = syncPing2Json(pSyncPing); + char* serialized = cJSON_Print(pJson); + printf("SyncPing: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pSyncPing, &rpcMsg); + SyncPing* pSyncPing2 = (SyncPing*)malloc(pSyncPing->bytes); + syncPingFromRpcMsg(&rpcMsg, pSyncPing2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncPing2Json(pSyncPing2); + char* serialized = cJSON_Print(pJson); + printf("SyncPing2: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncPingDestroy(pSyncPing); + syncPingDestroy(pSyncPing2); +} + +int main() { + // taosInitLog((char*)"syncPingTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + test1(); + test2(); return 0; } From bbcfc1b7b21436a87232dd9f4bd19f478a256457 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Mar 2022 17:13:12 +0800 Subject: [PATCH 09/82] update telemetry formater to cjson --- include/os/osSysinfo.h | 21 +- source/dnode/mnode/impl/src/mndTelem.c | 262 +++++++------------------ source/dnode/mnode/impl/src/mnode.c | 2 +- source/os/src/osSysinfo.c | 97 ++++++++- source/util/src/tjson.c | 53 ++++- 5 files changed, 229 insertions(+), 206 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a0771dc734..35ac032a8a 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -33,21 +33,26 @@ typedef struct { SDiskSize size; } SDiskSpace; -int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -int32_t taosGetCpuCores(); void taosGetSystemInfo(); +bool taosGetEmail(char *email, int32_t maxLen); +bool taosGetOsReleaseName(char *releaseName, int32_t maxLen); +bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); +int32_t taosGetCpuCores(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); +bool taosGetTotalSysMemoryKB(uint64_t *kb); +bool taosGetProcMemory(float *memoryUsedMB); // +bool taosGetSysMemory(float *memoryUsedMB); // +void taosGetDisk(); +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); bool taosReadProcIO(int64_t *rchars, int64_t *wchars); bool taosGetProcIO(float *readKB, float *writeKB); bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); bool taosGetBandSpeed(float *bandSpeedKb); -void taosGetDisk(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetProcMemory(float *memoryUsedMB); -bool taosGetSysMemory(float *memoryUsedMB); -int taosSystem(const char *cmd); + +int32_t taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); -char * taosGetCmdlineByPID(int pid); +char *taosGetCmdlineByPID(int32_t pid); void taosSetCoreDump(bool enable); typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index d6a4c76c62..744cefcb4f 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -18,186 +18,84 @@ #include "mndCluster.h" #include "mndSync.h" #include "tbuffer.h" -#include "tversion.h" +#include "tjson.h" -#define TELEMETRY_SERVER "telemetry.taosdata.com" +#define TELEMETRY_SERVER "localhost" #define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 -static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } - -static void mndCloseObject(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, '}'); - } else { - tbufWriteChar(bw, '}'); - } -} - -static void mndWriteString(SBufferWriter* bw, const char* str) { - tbufWriteChar(bw, '"'); - tbufWrite(bw, str, strlen(str)); - tbufWriteChar(bw, '"'); -} - -static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { - mndWriteString(bw, k); - tbufWriteChar(bw, ':'); - char buf[32] = {0}; - sprintf(buf, "%" PRId64, v); - tbufWrite(bw, buf, strlen(buf)); - tbufWriteChar(bw, ','); -} - -static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) { - mndWriteString(bw, k); - tbufWriteChar(bw, ':'); - mndWriteString(bw, v); - tbufWriteChar(bw, ','); -} - -static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - int32_t done = 0; - - // FILE* fp = fopen("/proc/cpuinfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { - const char* v = strchr(line, ':') + 2; - mndAddStringField(bw, "cpuModel", v); - done |= 1; - } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { - const char* v = strchr(line, ':') + 2; - mndWriteString(bw, "numOfCpu"); - tbufWriteChar(bw, ':'); - tbufWrite(bw, v, strlen(v)); - tbufWriteChar(bw, ','); - done |= 2; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); -} - -static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - - // FILE* fp = fopen("/etc/os-release", "r"); - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char* p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - mndAddStringField(bw, "os", p); - break; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); -} - -static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - - // FILE* fp = fopen("/proc/meminfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "MemTotal", 8) == 0) { - const char* p = strchr(line, ':') + 1; - while (*p == ' ') p++; - mndAddStringField(bw, "memory", p); - break; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); -} - -static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - mndAddStringField(bw, "version", version); - mndAddStringField(bw, "buildInfo", buildinfo); - mndAddStringField(bw, "gitInfo", gitinfo); - mndAddStringField(bw, "email", pMgmt->email); -} - -static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { +static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { SMnodeLoad load = {0}; - if (mndGetLoad(pMnode, &load) != 0) { - return; - } + if (mndGetLoad(pMnode, &load) != 0) return; - mndAddIntField(bw, "numOfDnode", load.numOfDnode); - mndAddIntField(bw, "numOfMnode", load.numOfMnode); - mndAddIntField(bw, "numOfVgroup", load.numOfVgroup); - mndAddIntField(bw, "numOfDatabase", load.numOfDatabase); - mndAddIntField(bw, "numOfSuperTable", load.numOfSuperTable); - mndAddIntField(bw, "numOfChildTable", load.numOfChildTable); - mndAddIntField(bw, "numOfColumn", load.numOfColumn); - mndAddIntField(bw, "numOfPoint", load.totalPoints); - mndAddIntField(bw, "totalStorage", load.totalStorage); - mndAddIntField(bw, "compStorage", load.compStorage); + tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); + tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); + tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup); + tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase); + tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable); + tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable); + tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn); + tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints); + tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage); + tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage); } -static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - SBufferWriter bw = tbufInitWriter(NULL, false); - int32_t code = -1; - char buf[128] = {0}; - SOCKET fd = 0; +static char* mndBuildTelemetryReport(SMnode* pMnode) { + char tmp[4096] = {0}; + STelemMgmt* pMgmt = &pMnode->telemMgmt; + + SJson* pJson = tjsonCreateObject(); + if (pJson == NULL) return NULL; char clusterName[64] = {0}; - if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { - goto SEND_OVER; + mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + tjsonAddStringToObject(pJson, "instanceId", clusterName); + tjsonAddDoubleToObject(pJson, "reportVersion", 1); + + if (taosGetOsReleaseName(tmp, sizeof(tmp))) { + tjsonAddStringToObject(pJson, "os", tmp); } - mndBeginObject(&bw); - mndAddStringField(&bw, "instanceId", clusterName); - mndAddIntField(&bw, "reportVersion", 1); - mndAddOsInfo(pMnode, &bw); - mndAddCpuInfo(pMnode, &bw); - mndAddMemoryInfo(pMnode, &bw); - mndAddVersionInfo(pMnode, &bw); - mndAddRuntimeInfo(pMnode, &bw); - mndCloseObject(&bw); + int32_t numOfCores = 0; + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) { + tjsonAddStringToObject(pJson, "cpuModel", tmp); + tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + } else { + tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores()); + } + + uint64_t memoryKB = 0; + if (taosGetTotalSysMemoryKB(&memoryKB)) { + snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); + } + + tjsonAddStringToObject(pJson, "version", version); + tjsonAddStringToObject(pJson, "buildInfo", buildinfo); + tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + tjsonAddStringToObject(pJson, "email", pMgmt->email); + + mndBuildRuntimeInfo(pMnode, pJson); + + char* pCont = tjsonToString(pJson); + tjsonDelete(pJson); + return pCont; +} + +static void mndSendTelemetryReport(const char* pCont) { + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; + int32_t contLen = strlen(pCont); uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + mError("failed to get telemetry server ip"); goto SEND_OVER; } fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + mError("failed to create telemetry socket"); goto SEND_OVER; } @@ -208,24 +106,24 @@ static void mndSendTelemetryReport(SMnode* pMnode) { "Content-Type: application/json\n" "Content-Length: "; if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + mError("failed to send telemetry header"); goto SEND_OVER; } - int32_t contLen = (int32_t)(tbufTell(&bw)); - sprintf(buf, "%d\n\n", contLen); + snprintf(buf, sizeof(buf), "%d\n\n", contLen); if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + mError("failed to send telemetry contlen"); goto SEND_OVER; } - const char* pCont = tbufGetData(&bw, false); if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + mError("failed to send telemetry content"); goto SEND_OVER; } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to receive response since %s", terrstr()); + mError("failed to receive telemetry response"); goto SEND_OVER; } @@ -233,12 +131,10 @@ static void mndSendTelemetryReport(SMnode* pMnode) { code = 0; SEND_OVER: - tbufCloseWriter(&bw); - taosCloseSocket(fd); - if (code != 0) { mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); } + taosCloseSocket(fd); } static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { @@ -247,33 +143,23 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { if (!pMgmt->enable) return 0; taosWLockLatch(&pMgmt->lock); - mndSendTelemetryReport(pMnode); + char* pCont = mndBuildTelemetryReport(pMnode); + if (pCont != NULL) { + mndSendTelemetryReport(pCont); + free(pCont); + } taosWUnLockLatch(&pMgmt->lock); return 0; } -static void mndGetEmail(SMnode* pMnode, char* filepath) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); - if (pFile == NULL) { - return; - } - - if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { - mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } - - taosCloseFile(&pFile); -} - int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - pMgmt->enable = tsEnableTelemetryReporting; - taosInitRWLatch(&pMgmt->lock); - mndGetEmail(pMnode, "/usr/local/taos/email"); + taosInitRWLatch(&pMgmt->lock); + pMgmt->enable = tsEnableTelemetryReporting; + taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index ab5d0d794b..879f866214 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -132,7 +132,7 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { + if (taosTmrReset(mndPullupTelem, 300, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cf9c557f5e..163fad803f 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -888,4 +888,99 @@ SysNameInfo taosGetSysNameInfo() { return info; } -#endif \ No newline at end of file +bool taosGetEmail(char *email, int32_t maxLen) { + const char *filepath = "/usr/local/taos/email"; + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) return false; + + if (taosReadFile(pFile, (void *)email, maxLen) < 0) { + taosCloseFile(&pFile); + return false; + } + + taosCloseFile(&pFile); + return true; +} + +bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { + char *line = NULL; + size_t size = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "PRETTY_NAME", 11) == 0) { + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(releaseName, p, maxLen); + ret = true; + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return ret; +} + +bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { + char *line = NULL; + size_t size = 0; + int32_t done = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { + const char *v = strchr(line, ':') + 2; + tstrncpy(cpuModel, v, maxLen); + ret = true; + done |= 1; + } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { + const char *v = strchr(line, ':') + 2; + *numOfCores = atoi(v); + done |= 2; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + + return ret; +} + +bool taosGetTotalSysMemoryKB(uint64_t *kb) { + char *line = NULL; + size_t size = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "MemTotal", 8) == 0) { + const char *p = strchr(line, ':') + 1; + while (*p == ' ') p++; + ret = true; + *kb = atoll(p); + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return ret; +} + +#endif diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 4b68467450..2c117771b1 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -18,36 +18,73 @@ #include "cJSON.h" #include "taoserror.h" -SJson* tjsonCreateObject() { return cJSON_CreateObject(); } +SJson* tjsonCreateObject() { + SJson* pJson = cJSON_CreateObject(); + if (pJson == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return pJson; +} void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); } int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) { char tmp[40] = {0}; - snprintf(tmp, tListLen(tmp), "%" PRId64, number); + snprintf(tmp, sizeof(tmp), "%" PRId64, number); return tjsonAddStringToObject(pJson, pName, tmp); } int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { - return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) { - return (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { - return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); } +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { + SJson* ret = (SJson*)cJSON_AddArrayToObject((cJSON*)pJson, pName); + if (ret == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return ret; +} int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) { - return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); + if (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem)) { + return TSDB_CODE_SUCCESS; + } + + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) { - return (cJSON_AddItemToArray((cJSON*)pJson, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); + if (cJSON_AddItemToArray((cJSON*)pJson, pItem)) { + return TSDB_CODE_SUCCESS; + } + + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) { From 03629aabaff7f9192ba4eb0bd9f695a914192383 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Mar 2022 17:31:05 +0800 Subject: [PATCH 10/82] [td-13039] refactor. --- include/os/os.h | 1 + include/util/thash.h | 92 +-- include/util/tpagedbuf.h | 2 +- source/client/src/clientHb.c | 3 +- source/dnode/mgmt/impl/src/dndVnodes.c | 2 +- source/libs/catalog/src/catalog.c | 18 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/function/src/tpercentile.c | 2 +- source/libs/parser/src/insertParser.c | 2 +- source/util/src/tcache.c | 5 +- source/util/src/thash.c | 790 +++++++++++------------- source/util/src/tpagedbuf.c | 25 +- source/util/test/pageBufferTest.cpp | 6 +- 13 files changed, 423 insertions(+), 527 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index f020af5a65..a58e798d38 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -44,6 +44,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/include/util/thash.h b/include/util/thash.h index 5c344f3f0f..017cc8696f 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -28,11 +28,6 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) -#define HASH_DEFAULT_LOAD_FACTOR (0.75) - -#define HASH_INDEX(v, c) ((v) & ((c)-1)) - #define HASH_NODE_EXIST(code) (code == -2) /** @@ -62,41 +57,17 @@ typedef struct SHashNode { uint32_t hashVal; // the hash value of key uint32_t dataLen; // length of data uint32_t keyLen; // length of the key - uint16_t count; // reference count + uint16_t refCount; // reference count int8_t removed; // flag to indicate removed char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) - typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, } SHashLockTypeE; -typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; -} SHashEntry; - -typedef struct SHashObj { - SHashEntry **hashList; - uint32_t capacity; // number of slots - uint32_t size; // number of elements in hash table - - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function - _equal_fn_t equalFp; // equal function - _hash_before_fn_t callbackFp; // function invoked before return the value to caller - - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry -} SHashObj; +typedef struct SHashObj SHashObj; /** * init the hash table @@ -126,8 +97,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj); */ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); -int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded); - /** * return the payload data with the specified key * @@ -146,17 +115,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param destBuf * @return */ -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); +int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); /** - * Clone the result to interval allocated buffer + * * @param pHashObj * @param key * @param keyLen * @param destBuf + * @param size * @return */ -void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); +int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size); /** * remove item with the specified key @@ -207,37 +177,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); */ void taosHashCancelIterate(SHashObj *pHashObj, void *p); -/** - * Get the corresponding key information for a given data in hash table - * @param data - * @return - */ -int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); - -/** - * Get the corresponding key information for a given data in hash table, using memcpy - * @param data - * @param dst - * @return - */ -static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) { - if (NULL == data || NULL == dst) { - return -1; - } - - SHashNode *node = GET_HASH_PNODE(data); - void *key = GET_HASH_NODE_KEY(node); - memcpy(dst, key, node->keyLen); - - return 0; -} - -/** - * Get the corresponding data length for a given data in hash table - * @param data - * @return - */ -int32_t taosHashGetDataLen(void *data); + /** + * Get the corresponding key information for a given data in hash table + * @param data + * @param keyLen + * @return + */ +void *taosHashGetKey(void *data, size_t* keyLen); /** * return the payload data with the specified key(reference number added) @@ -258,8 +204,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRelease(SHashObj *pHashObj, void *p); +/** + * + * @param pHashObj + * @param fp + */ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); +/** + * + * @param pHashObj + * @param fp + */ +void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp); + #ifdef __cplusplus } #endif diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index ce9a57c2c3..acaff759b7 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis { * @param handle * @return */ -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir); /** * diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 45c1858948..9413f748eb 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -482,7 +482,8 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { free(pAppHbMgr); return NULL; } - pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; + + taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); // init getInfoFunc pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index f20493aa7f..ce7cbe0977 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -85,7 +85,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { int32_t refCount = 0; taosRLockLatch(&pMgmt->latch); - taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2b6caa2f8f..945a4101b1 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -162,8 +162,7 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { size_t len = 0; dbCache = (SCtgDBCache *)pIter; - - taosHashGetKey(dbCache, (void **)&dbFName, &len); + dbFName = taosHashGetKey(dbCache, &len); CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); @@ -532,9 +531,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } - size_t sz = 0; + int32_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz); + int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { @@ -545,8 +544,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } *exist = 1; - - tbMeta = *pTableMeta; + STableMeta* tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); @@ -1110,7 +1108,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { void *pIter = taosHashIterate(cache->stbCache, NULL); while (pIter) { uint64_t *suid = NULL; - taosHashGetKey(pIter, (void **)&suid, NULL); + suid = taosHashGetKey(pIter, NULL); if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); @@ -1305,7 +1303,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); + ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1343,7 +1341,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { int32_t *vgId = NULL; void *pIter = taosHashIterate(src->vgHash, NULL); while (pIter) { - taosHashGetKey(pIter, (void **)&vgId, NULL); + vgId = taosHashGetKey(pIter, NULL); if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); @@ -2296,7 +2294,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { + if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c400ded1c1..a8109e7363 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4619,7 +4619,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp"); + int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp"); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 1aa0c04ec4..06c58430a4 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp"); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp"); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 745982e869..4b58bc4e69 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -153,7 +153,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData, char*); diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 64d822f750..560e5348c2 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -305,8 +305,9 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return NULL; } + // TODO remove it SCacheDataNode *ptNode = NULL; - taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode); + ptNode = taosHashAcquire(pCacheObj->pHashTable, key, keyLen); // taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); void *pData = (ptNode != NULL) ? ptNode->data : NULL; @@ -535,7 +536,7 @@ static bool travHashTableEmptyFn(void *param, void *data) { void taosCacheEmpty(SCacheObj *pCacheObj) { SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); +// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); taosTrashcanEmpty(pCacheObj, false); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 219fc739ca..05bc94caef 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -15,69 +15,127 @@ #define _DEFAULT_SOURCE #include "thash.h" -#include "tdef.h" +#include "taoserror.h" +#include "os.h" #include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT -#define MAX_WARNING_REF_COUNT 10000 -#define EXT_SIZE 1024 -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define MAX_WARNING_REF_COUNT 10000 +#define EXT_SIZE 1024 +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_DEFAULT_LOAD_FACTOR (0.75) +#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define DO_FREE_HASH_NODE(_n) \ - do { \ - tfree(_n); \ - } while (0) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -#define FREE_HASH_NODE(_h, _n) \ - do { \ - if ((_h)->freeFp) { \ - (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ - } \ - \ - DO_FREE_HASH_NODE(_n); \ +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) + +#define FREE_HASH_NODE(_n) \ + do { \ + tfree(_n); \ } while (0); -static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; +} SHashEntry; + +typedef struct SHashObj { + SHashEntry **hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // hash node free callback function + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry + _hash_before_fn_t callbackFp; // function invoked before return the value to caller +} SHashObj; + +/* + * Function definition + */ +static FORCE_INLINE void taosHashWLock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosWLockLatch(lock); + taosWLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashWUnlock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosRLockLatch(lock); + + taosWUnLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashRLock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosRUnLockLatch(lock); + + taosRLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosWUnLockLatch(lock); + + taosRUnLockLatch(&pHashObj->lock); +} + +static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + taosWLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosWUnLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosRLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosRUnLockLatch(&pe->latch); } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = TMIN(length, HASH_MAX_CAPACITY); + int32_t len = MIN(length, HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); return i; } -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, - uint32_t hashVal) { +static FORCE_INLINE SHashNode * +doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && + ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -90,60 +148,57 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr } /** - * Resize the hash list if the threshold is reached + * resize the hash list if the threshold is reached * * @param pHashObj */ static void taosHashTableResize(SHashObj *pHashObj); /** + * allocate and initialize a hash node + * * @param key key of object for hash, usually a null-terminated string * @param keyLen length of key - * @param pData actually data. Requires a consecutive memory block, no pointer is allowed in pData. - * Pointer copy causes memory access error. + * @param pData data to be stored in hash node * @param dsize size of data * @return SHashNode */ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal); /** - * Update the hash node + * update the hash node * - * @param pNode hash node - * @param key key for generate hash value - * @param keyLen key length - * @param pData actual data - * @param dsize size of actual data - * @return hash node + * @param pHashObj hash table object + * @param pe hash table entry to operate on + * @param prev previous node + * @param pNode the old node with requested key + * @param pNewNode the new node with requested key */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, - SHashNode *pNewNode) { +static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - pNode->count--; + atomic_sub_fetch_32(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; } else { pe->next = pNewNode; } - if (pNode->count <= 0) { + if (pNode->refCount <= 0) { pNewNode->next = pNode->next; - DO_FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode); } else { pNewNode->next = pNode; pe->num++; - atomic_add_fetch_32(&pHashObj->size, 1); + atomic_add_fetch_64(&pHashObj->size, 1); } - - return pNewNode; } /** * insert the hash node at the front of the linked list * - * @param pHashObj - * @param pNode + * @param pHashObj hash table object + * @param pNode the old node with requested key */ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); @@ -156,47 +211,70 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); /** - * Get the next element in hash table for iterator - * @param pIter - * @return + * initialize a hash table + * + * @param capacity initial capacity of the hash table + * @param fn hash function + * @param update whether the hash table allows in place update + * @param type whether the hash table has per entry lock + * @return hash table object */ - SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { - assert(fn != NULL); + if (fn == NULL) { + assert(0); + return NULL; + } + if (capacity == 0) { capacity = 4; } SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); if (pHashObj == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } // the max slots is not defined by user pHashObj->capacity = taosHashCapacity((int32_t)capacity); - assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; + ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { free(pHashObj); - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; - } else { - pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); - - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); - } - - taosArrayPush(pHashObj->pMemBlock, &p); } + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); + if (pHashObj->pMemBlock == NULL) { + free(pHashObj->hashList); + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + if (p == NULL) { + taosArrayDestroy(pHashObj->pMemBlock); + free(pHashObj->hashList); + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + } + + taosArrayPush(pHashObj->pMemBlock, &p); + return pHashObj; } @@ -206,16 +284,28 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { } } -int32_t taosHashGetSize(const SHashObj *pHashObj) { - if (!pHashObj) { - return 0; +void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp) { + if (pHashObj != NULL && fp != NULL) { + pHashObj->freeFp = fp; } - return (int32_t)atomic_load_32(&pHashObj->size); } -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } +int32_t taosHashGetSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} + +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { + if (pHashObj == NULL || key == NULL || keyLen == 0 || data == NULL || size == 0) { + return -1; + } -int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); if (pNewNode == NULL) { @@ -224,19 +314,17 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashWLock(pHashObj); taosHashTableResize(pHashObj); - __wr_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashWUnlock(pHashObj); } - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; if (pe->num > 0) { @@ -245,9 +333,10 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void assert(pNode == NULL); } - SHashNode *prev = NULL; + SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && + (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -260,24 +349,13 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table pushfrontNodeInEntryList(pe, pNewNode); + assert(pe->next != NULL); - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); // enable resize - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - atomic_add_fetch_32(&pHashObj->size, 1); - - if (newAdded) { - *newAdded = true; - } + taosHashRUnlock(pHashObj); + atomic_add_fetch_64(&pHashObj->size, 1); return 0; } else { @@ -285,131 +363,67 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void if (pHashObj->enableUpdate) { doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { - DO_FREE_HASH_NODE(pNewNode); + FREE_HASH_NODE(pNewNode); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); // enable resize - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashRUnlock(pHashObj); - if (newAdded) { - *newAdded = false; - } - - return pHashObj->enableUpdate ? 0 : -2; + return pHashObj->enableUpdate ? 0 : -1; } } -int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - return taosHashPutImpl(pHashObj, key, keyLen, data, size, NULL); -} - -int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { - return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded); -} +static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef); void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetClone(pHashObj, key, keyLen, NULL); + void* p = NULL; + return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false); } -// TODO(yihaoDeng), merge with taosHashGetClone -void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, - size_t *sz) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { +int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) { + terrno = 0; + /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); + return terrno; +} + +int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) { + terrno = 0; + + /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); + return terrno; +} + +void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashRUnlock(pHashObj); return NULL; } char *data = NULL; + taosHashEntryRLock(pHashObj, pe); - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - - if (pe->num > 0) { - assert(pe->next != NULL); - } else { - assert(pe->next == NULL); - } - - SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); - if (pNode != NULL) { - if (fp != NULL) { - fp(GET_HASH_NODE_DATA(pNode)); - } - - if (*d == NULL) { - *sz = pNode->dataLen + EXT_SIZE; - *d = calloc(1, *sz); - } else if (*sz < pNode->dataLen) { - *sz = pNode->dataLen + EXT_SIZE; - *d = realloc(*d, *sz); - } - memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); - // just make runtime happy - if ((*sz) - pNode->dataLen > 0) { - memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); - } - - data = GET_HASH_NODE_DATA(pNode); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } - - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return data; -} - -void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *d, bool acquire) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { - return NULL; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - - // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return NULL; - } - - char *data = NULL; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - +#if 0 if (pe->num > 0) { assert(pe->next != NULL); } else { assert(pe->next == NULL); } +#endif SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); if (pNode != NULL) { @@ -417,108 +431,115 @@ void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v pHashObj->callbackFp(GET_HASH_NODE_DATA(pNode)); } - if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); + if (size != NULL) { + if (*d == NULL) { + *size = pNode->dataLen; + *d = calloc(1, *size); + if (*d == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } else if (*size < pNode->dataLen) { + *size = pNode->dataLen; + char* tmp = realloc(*d, *size); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + *d = tmp; + } } - if (acquire) { - atomic_add_fetch_16(&pNode->count, 1); + if (addRef) { + atomic_add_fetch_16(&pNode->refCount, 1); + } + + if (*d != NULL) { + memcpy(*d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } data = GET_HASH_NODE_DATA(pNode); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } + taosHashEntryRUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *d) { - return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false); -} - -void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true); -} - -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen /*, void *data, size_t dsize*/) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) { return -1; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); // double check after locked if (pe->num == 0) { assert(pe->next == NULL); - taosWUnLockLatch(&pe->latch); - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); return -1; } - int32_t code = -1; + int code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && - pNode->removed == 0) - break; + if ((pNode->keyLen == keyLen) && + ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { + code = 0; // it is found - prevNode = pNode; - pNode = pNode->next; - } + atomic_sub_fetch_32(&pNode->refCount, 1); + pNode->removed = 1; + if (pNode->refCount <= 0) { + if (prevNode == NULL) { + pe->next = pNode->next; + } else { + prevNode->next = pNode->next; + } - if (pNode) { - code = 0; // it is found + if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pNode->count--; - pNode->removed = 1; - if (pNode->count <= 0) { - if (prevNode) { - prevNode->next = pNode->next; - } else { - pe->next = pNode->next; + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pNode); } - - // if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - - pe->num--; - atomic_sub_fetch_32(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pNode); + } else { + prevNode = pNode; + pNode = pNode->next; } } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); return code; } -int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { - return 0; +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); +} + +void taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || fp == NULL) { + return; } // disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t numOfEntries = (int32_t)pHashObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { @@ -527,63 +548,32 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi continue; } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pEntry->latch); - } + taosHashEntryWLock(pHashObj, pEntry); - // todo remove the first node - SHashNode *pNode = NULL; - while ((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { - pEntry->num -= 1; - atomic_sub_fetch_32(&pHashObj->size, 1); - - pEntry->next = pNode->next; - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNode); + SHashNode *pPrevNode = NULL; + SHashNode *pNode = pEntry->next; + while (pNode != NULL) { + if (fp(param, GET_HASH_NODE_DATA(pNode))) { + pPrevNode = pNode; + pNode = pNode->next; } else { - break; - } - } - - // handle the following node - if (pNode != NULL) { - assert(pNode == pEntry->next); - SHashNode *pNext = NULL; - - while ((pNext = pNode->next) != NULL) { - // not qualified, remove it - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { - pNode->next = pNext->next; - pEntry->num -= 1; - atomic_sub_fetch_32(&pHashObj->size, 1); - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNext); + if (pPrevNode == NULL) { + pEntry->next = pNode->next; } else { - pNode = pNext; + pPrevNode->next = pNode->next; } + pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + SHashNode *next = pNode->next; + FREE_HASH_NODE(pNode); + pNode = next; } } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pEntry->latch); - } + taosHashEntryWUnlock(pHashObj, pEntry); } - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return 0; + taosHashRUnlock(pHashObj); } void taosHashClear(SHashObj *pHashObj) { @@ -593,12 +583,12 @@ void taosHashClear(SHashObj *pHashObj) { SHashNode *pNode, *pNext; - __wr_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashWLock(pHashObj); for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; if (pEntry->num == 0) { - assert(pEntry->next == 0); + assert(pEntry->next == NULL); continue; } @@ -607,7 +597,7 @@ void taosHashClear(SHashObj *pHashObj) { while (pNode) { pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + FREE_HASH_NODE(pNode); pNode = pNext; } @@ -616,10 +606,11 @@ void taosHashClear(SHashObj *pHashObj) { pEntry->next = NULL; } - atomic_store_32(&pHashObj->size, 0); - __wr_unlock((void *)&pHashObj->lock, pHashObj->type); + pHashObj->size = 0; + taosHashWUnlock(pHashObj); } +// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj) void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; @@ -636,26 +627,29 @@ void taosHashCleanup(SHashObj *pHashObj) { } taosArrayDestroy(pHashObj->pMemBlock); - - memset(pHashObj, 0, sizeof(SHashObj)); free(pHashObj); } // for profile only -int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } int32_t num = 0; + taosHashRLock((SHashObj*) pHashObj); for (int32_t i = 0; i < pHashObj->size; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; + + // fine grain per entry lock is not held since this is used + // for profiling only and doesn't need an accurate count. if (num < pEntry->num) { num = pEntry->num; } } + taosHashRUnlock((SHashObj*) pHashObj); return num; } @@ -664,28 +658,24 @@ void taosHashTableResize(SHashObj *pHashObj) { return; } - // double the original capacity - SHashNode *pNode = NULL; - SHashNode *pNext = NULL; - - int32_t newSize = (int32_t)(pHashObj->capacity << 1u); - if (newSize > HASH_MAX_CAPACITY) { - // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", - // pHashObj->capacity, HASH_MAX_CAPACITY); + int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); + if (newCapacity > HASH_MAX_CAPACITY) { +// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", +// pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); - if (pNewEntryList == NULL) { // todo handle error - // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + if (pNewEntryList == NULL) { +// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } pHashObj->hashList = pNewEntryList; - size_t inc = newSize - pHashObj->capacity; - void *p = calloc(inc, sizeof(SHashEntry)); + size_t inc = newCapacity - pHashObj->capacity; + void * p = calloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -693,92 +683,62 @@ void taosHashTableResize(SHashObj *pHashObj) { taosArrayPush(pHashObj->pMemBlock, &p); - pHashObj->capacity = newSize; - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pe = pHashObj->hashList[i]; - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } + pHashObj->capacity = newCapacity; + for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { + SHashEntry *pe = pHashObj->hashList[idx]; + SHashNode *pNode; + SHashNode *pNext; + SHashNode *pPrev = NULL; if (pe->num == 0) { assert(pe->next == NULL); continue; } - while ((pNode = pe->next) != NULL) { - int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - pe->next = pNode->next; + pNode = pe->next; - if (pe->num == 0) { - assert(pe->next == NULL); + assert(pNode != NULL); + + while (pNode != NULL) { + int32_t newIdx = HASH_INDEX(pNode->hashVal, pHashObj->capacity); + pNext = pNode->next; + if (newIdx != idx) { + pe->num -= 1; + if (pPrev == NULL) { + pe->next = pNext; } else { - assert(pe->next != NULL); + pPrev->next = pNext; } - SHashEntry *pNewEntry = pHashObj->hashList[j]; + SHashEntry *pNewEntry = pHashObj->hashList[newIdx]; pushfrontNodeInEntryList(pNewEntry, pNode); } else { - break; - } - } - - if (pNode != NULL) { - while ((pNext = pNode->next) != NULL) { - int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - - pNode->next = pNext->next; - pNext->next = NULL; - - // added into new slot - SHashEntry *pNewEntry = pHashObj->hashList[j]; - - if (pNewEntry->num == 0) { - assert(pNewEntry->next == NULL); - } else { - assert(pNewEntry->next != NULL); - } - - pushfrontNodeInEntryList(pNewEntry, pNext); - } else { - pNode = pNext; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); + pPrev = pNode; } + pNode = pNext; } } int64_t et = taosGetTimestampUs(); - uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, - ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, +// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; pNewNode->dataLen = (uint32_t)dsize; - pNewNode->count = 1; + pNewNode->refCount= 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -800,51 +760,32 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + - sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } -FORCE_INLINE int32_t taosHashGetKey(void *data, void **key, size_t *keyLen) { - if (NULL == data || NULL == key) { - return -1; - } - - SHashNode *node = GET_HASH_PNODE(data); - *key = GET_HASH_NODE_KEY(node); - if (keyLen) { +void *taosHashGetKey(void *data, size_t* keyLen) { + SHashNode * node = GET_HASH_PNODE(data); + if (keyLen != NULL) { *keyLen = node->keyLen; } - return 0; -} - -FORCE_INLINE int32_t taosHashGetDataLen(void *data) { - SHashNode *node = GET_HASH_PNODE(data); - return node->keyLen; -} - -FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { - SHashNode *node = GET_HASH_PNODE(data); - return node->keyLen; + return GET_HASH_NODE_KEY(node); } // release the pNode, return next pNode, and lock the current entry -static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); SHashNode *prevNode = NULL; *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[*slot]; - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; - while (pNode) { - if (pNode == pOld) break; + if (pNode == pOld) + break; prevNode = pNode; pNode = pNode->next; @@ -857,8 +798,8 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { pNode = pNode->next; } - pOld->count--; - if (pOld->count <= 0) { + atomic_sub_fetch_32(&pOld->refCount, 1); + if (pOld->refCount <=0) { if (prevNode) { prevNode->next = pOld->next; } else { @@ -866,11 +807,11 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { } pe->num--; - atomic_sub_fetch_32(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pOld); + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pOld); } } else { - uError("pNode:%p data:%p is not there!!!", pNode, p); +// uError("pNode:%p data:%p is not there!!!", pNode, p); } return pNode; @@ -879,20 +820,18 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL) return NULL; - int32_t slot = 0; + int slot = 0; char *data = NULL; // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); SHashNode *pNode = NULL; if (p) { pNode = taosHashReleaseNode(pHashObj, p, &slot); if (pNode == NULL) { SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); slot = slot + 1; } @@ -902,10 +841,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { for (; slot < pHashObj->capacity; ++slot) { SHashEntry *pe = pHashObj->hashList[slot]; - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); pNode = pe->next; while (pNode) { @@ -915,23 +851,22 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pNode) break; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); } } if (pNode) { SHashEntry *pe = pHashObj->hashList[slot]; - uint16_t prevRef = atomic_load_16(&pNode->count); - uint16_t afterRef = atomic_add_fetch_16(&pNode->count, 1); + uint16_t prevRef = atomic_load_16(&pNode->refCount); + uint16_t afterRef = atomic_add_fetch_16(&pNode->refCount, 1); + ASSERT(prevRef < afterRef); // the reference count value is overflow, which will cause the delete node operation immediately. if (prevRef > afterRef) { uError("hash entry ref count overflow, prev ref:%d, current ref:%d", prevRef, afterRef); // restore the value - atomic_sub_fetch_16(&pNode->count, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); data = NULL; } else { data = GET_HASH_NODE_DATA(pNode); @@ -941,12 +876,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { uWarn("hash entry ref count is abnormally high: %d", afterRef); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); } - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashRUnlock(pHashObj); return data; } @@ -954,17 +887,20 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL || p == NULL) return; // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); - int32_t slot; + int slot; taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); +} + +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { + void* p = NULL; + return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); } void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 45c38165a1..fe32afb2f4 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -42,8 +42,8 @@ struct SDiskbasedBuf { bool comp; // compressed before flushed to disk uint64_t nextPos; // next page flush position - uint64_t qId; // for debug purpose - bool printStatis; // Print statistics info when closing this buffer. + char* id; // for debug purpose + bool printStatis; // Print statistics info when closing this buffer. SDiskbasedBufStatis statis; }; @@ -356,7 +356,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) { return ppi; } -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { *pBuf = calloc(1, sizeof(SDiskbasedBuf)); @@ -366,13 +366,13 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem } pPBuf->pageSize = pagesize; - pPBuf->numOfPages = 0; // all pages are in buffer in the first place + pPBuf->numOfPages = 0; // all pages are in buffer in the first place pPBuf->totalBufSize = 0; pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. pPBuf->allocateId = -1; - pPBuf->comp = true; - pPBuf->pFile = NULL; - pPBuf->qId = qId; + pPBuf->comp = true; + pPBuf->pFile = NULL; + pPBuf->id = strdup(id); pPBuf->fileSize = 0; pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); pPBuf->freePgList = tdListNew(POINTER_BYTES); @@ -540,13 +540,13 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (pBuf->pFile != NULL) { uDebug( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " - "size:%.2f Kb, %" PRIx64 "\n", + "size:%.2f Kb, %s\n", pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id); taosCloseFile(&pBuf->pFile); } else { - uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId); + uDebug("Paged buffer closed, total:%.2f Kb, no file created, %s", pBuf->totalBufSize / 1024.0, pBuf->id); } // print the statistics information @@ -584,6 +584,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); + tfree(pBuf->id); tfree(pBuf->assistBuf); tfree(pBuf); } @@ -639,9 +640,9 @@ void dBufPrintStatis(const SDiskbasedBuf* pBuf) { printf( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f " - "Kb, %" PRIx64 "\n", + "Kb, %s\n", pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id); printf( "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n", diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 4b9d8c5f51..8fbec31dd2 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -13,7 +13,7 @@ namespace { // simple test void simpleTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, "", "/tmp/"); int32_t pageId = 0; int32_t groupId = 0; @@ -55,7 +55,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; @@ -102,7 +102,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; From c6fb4e928a6e3b1efeae9da89626e150981d07a2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Mar 2022 17:34:55 +0800 Subject: [PATCH 11/82] Also start taosd when the config file does not exis --- include/os/osDir.h | 1 + include/util/tconfig.h | 2 +- source/common/src/tglobal.c | 22 ++++++++++++---------- source/os/src/osDir.c | 9 +++++++++ source/util/src/tconfig.c | 7 +++++-- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index e328c42063..eda83ea0ea 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -26,6 +26,7 @@ int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); +bool taosIsDir(const char *dirname); #ifdef __cplusplus } diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 8275054a64..0e1d352f9a 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -82,7 +82,7 @@ typedef struct SConfig { SConfig *cfgInit(); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); -int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index db84f87b66..4202c1fe14 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -177,24 +177,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { - uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); return -1; } - if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { - if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { - uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); - return -1; + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + uError("failed to load from config file:%s since %s", cfgFile, terrstr()); + return 0; } } if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { - uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + uError("failed to load from env file:%s since %s", envFile, terrstr()); return -1; } if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) { - uError("failed to load from global env variables since %s\n", terrstr()); + uError("failed to load from global env variables since %s", terrstr()); return -1; } @@ -438,8 +438,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (pCfg == NULL) return -1; if (tsc) { + tscEmbeddedInUtil = 0; if (taosAddClientLogCfg(pCfg) != 0) return -1; } else { + tscEmbeddedInUtil = 1; if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1; } @@ -450,7 +452,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return -1; } - if (cfgLoadArray(pCfg, pArgs) != 0) { + if (cfgLoadFromArray(pCfg, pArgs) != 0) { uError("failed to load cfg from array since %s", terrstr()); cfgCleanup(pCfg); return -1; @@ -466,7 +468,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); if (taosInitLog(logname, logFileNum) != 0) { - printf("failed to init log file since %s\n", terrstr()); + uError("failed to init log file since %s", terrstr()); cfgCleanup(pCfg); return -1; } @@ -497,7 +499,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU return -1; } - if (cfgLoadArray(tsCfg, pArgs) != 0) { + if (cfgLoadFromArray(tsCfg, pArgs) != 0) { uError("failed to load cfg from array since %s", terrstr()); cfgCleanup(tsCfg); return -1; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index c464073e5f..91ef97e66b 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -140,4 +140,13 @@ int32_t taosRealPath(char *dirname, int32_t maxlen) { return 0; } +bool taosIsDir(const char *dirname) { + DIR *dir = opendir(dirname); + if (dir != NULL) { + closedir(dir); + return true; + } + return false; +} + #endif diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8330c10ec7..84af091fb3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -60,7 +60,7 @@ int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { } } -int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs) { +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { int32_t size = taosArrayGetSize(pArgs); for (int32_t i = 0; i < size; ++i) { SConfigPair *pPair = taosArrayGet(pArgs, i); @@ -608,7 +608,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t olen, vlen, vlen2, vlen3; ssize_t _bytes = 0; - // FILE *fp = fopen(filepath, "r"); + if (taosIsDir(filepath)) { + return -1; + } + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); From 12c202aa316ad0f7288060b25a49047f676f76ee Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 1 Mar 2022 18:35:07 +0800 Subject: [PATCH 12/82] sync encode test --- source/libs/sync/inc/syncMessage.h | 46 +++++-- source/libs/sync/src/syncMessage.c | 166 +++++++++++++++-------- source/libs/sync/test/syncEncodeTest.cpp | 137 +++++++++++++++---- 3 files changed, 251 insertions(+), 98 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 121314c589..a7de7b9019 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -58,6 +58,20 @@ typedef struct SyncPing { #define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) +SyncPing* syncPingBuild(uint32_t dataLen); + +void syncPingDestroy(SyncPing* pMsg); + +void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); + +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); + +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); + +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); + +cJSON* syncPing2Json(const SyncPing* pMsg); + typedef struct SyncPingReply { uint32_t bytes; uint32_t msgType; @@ -67,6 +81,23 @@ typedef struct SyncPingReply { char data[]; } SyncPingReply; +#define SYNC_PING_REPLY_FIX_LEN \ + (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) + +SyncPingReply* syncPingReplyBuild(uint32_t dataLen); + +void syncPingReplyDestroy(SyncPingReply* pMsg); + +void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); + +void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); + +void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); + +void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); + +cJSON* syncPingReply2Json(const SyncPingReply* pMsg); + typedef struct SyncClientRequest { ESyncMessageType msgType; char* data; @@ -118,21 +149,6 @@ typedef struct SyncAppendEntriesReply { SyncIndex matchIndex; } SyncAppendEntriesReply; -// ---- message build ---- -SyncPing* syncPingBuild(uint32_t dataLen); - -void syncPingDestroy(SyncPing* pSyncPing); - -void syncPingSerialize(const SyncPing* pSyncPing, char* buf, uint32_t bufLen); - -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing); - -void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg); - -void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pSyncPing); - -cJSON* syncPing2Json(const SyncPing* pSyncPing); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 80dd184018..3122aa66bb 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -19,136 +19,194 @@ void onMessage(SRaft* pRaft, void* pMsg) {} -// ---- message build ---- +// ---- message process SyncPing---- SyncPing* syncPingBuild(uint32_t dataLen) { uint32_t bytes = SYNC_PING_FIX_LEN + dataLen; - SyncPing* pSyncPing = malloc(bytes); - memset(pSyncPing, 0, bytes); - pSyncPing->bytes = bytes; - pSyncPing->msgType = SYNC_PING; - pSyncPing->dataLen = dataLen; + SyncPing* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_PING; + pMsg->dataLen = dataLen; } -void syncPingDestroy(SyncPing* pSyncPing) { - if (pSyncPing != NULL) { - free(pSyncPing); +void syncPingDestroy(SyncPing* pMsg) { + if (pMsg != NULL) { + free(pMsg); } } -void syncPingSerialize(const SyncPing* pSyncPing, char* buf, uint32_t bufLen) { - assert(pSyncPing->bytes <= bufLen); - memcpy(buf, pSyncPing, pSyncPing->bytes); +void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); } -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { - /* - uint32_t* pU32 = (uint32_t*)buf; - uint32_t bytes = *pU32; - pSyncPing = (SyncPing*)malloc(bytes); - */ - memcpy(pSyncPing, buf, len); - assert(len == pSyncPing->bytes); - assert(pSyncPing->bytes == SYNC_PING_FIX_LEN + pSyncPing->dataLen); +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == SYNC_PING_FIX_LEN + pMsg->dataLen); } -void syncPing2RpcMsg(const SyncPing* pSyncPing, SRpcMsg* pRpcMsg) { - pRpcMsg->msgType = pSyncPing->msgType; - pRpcMsg->contLen = pSyncPing->bytes; +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) { + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - syncPingSerialize(pSyncPing, pRpcMsg->pCont, pRpcMsg->contLen); + syncPingSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); } -void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pSyncPing) { - syncPingDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pSyncPing); +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg) { + syncPingDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); } -cJSON* syncPing2Json(const SyncPing* pSyncPing) { +cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pSyncPing->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pSyncPing->msgType); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pSrcId, "addr", pSyncPing->srcId.addr); - cJSON_AddNumberToObject(pSrcId, "vgId", pSyncPing->srcId.vgId); + cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pSyncPing->destId.addr); - cJSON_AddNumberToObject(pDestId, "vgId", pSyncPing->destId.vgId); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "srcId", pDestId); - cJSON_AddNumberToObject(pRoot, "dataLen", pSyncPing->dataLen); - cJSON_AddStringToObject(pRoot, "data", pSyncPing->data); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", pMsg->data); - return pRoot; + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncPing", pRoot); + return pJson; +} + +// ---- message process SyncPingReply---- +SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { + uint32_t bytes = SYNC_PING_REPLY_FIX_LEN + dataLen; + SyncPingReply* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_PING; + pMsg->dataLen = dataLen; +} + +void syncPingReplyDestroy(SyncPingReply* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == SYNC_PING_FIX_LEN + pMsg->dataLen); +} + +void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg) { + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncPingReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg) { + syncPingReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pDestId); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", pMsg->data); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncPingReply", pRoot); + return pJson; } #if 0 -void syncPingSerialize(const SyncPing* pSyncPing, char** ppBuf, uint32_t* bufLen) { - *bufLen = sizeof(SyncPing) + pSyncPing->dataLen; +void syncPingSerialize(const SyncPing* pMsg, char** ppBuf, uint32_t* bufLen) { + *bufLen = sizeof(SyncPing) + pMsg->dataLen; *ppBuf = (char*)malloc(*bufLen); void* pStart = *ppBuf; uint32_t allBytes = *bufLen; int len = 0; - len = taosEncodeFixedU32(&pStart, pSyncPing->msgType); + len = taosEncodeFixedU32(&pStart, pMsg->msgType); allBytes -= len; assert(len > 0); pStart += len; - len = taosEncodeFixedU64(&pStart, pSyncPing->srcId.addr); + len = taosEncodeFixedU64(&pStart, pMsg->srcId.addr); allBytes -= len; assert(len > 0); pStart += len; - len = taosEncodeFixedI32(&pStart, pSyncPing->srcId.vgId); + len = taosEncodeFixedI32(&pStart, pMsg->srcId.vgId); allBytes -= len; assert(len > 0); pStart += len; - len = taosEncodeFixedU64(&pStart, pSyncPing->destId.addr); + len = taosEncodeFixedU64(&pStart, pMsg->destId.addr); allBytes -= len; assert(len > 0); pStart += len; - len = taosEncodeFixedI32(&pStart, pSyncPing->destId.vgId); + len = taosEncodeFixedI32(&pStart, pMsg->destId.vgId); allBytes -= len; assert(len > 0); pStart += len; - len = taosEncodeFixedU32(&pStart, pSyncPing->dataLen); + len = taosEncodeFixedU32(&pStart, pMsg->dataLen); allBytes -= len; assert(len > 0); pStart += len; - memcpy(pStart, pSyncPing->data, pSyncPing->dataLen); - allBytes -= pSyncPing->dataLen; + memcpy(pStart, pMsg->data, pMsg->dataLen); + allBytes -= pMsg->dataLen; assert(allBytes == 0); } -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pSyncPing) { +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { void* pStart = (void*)buf; uint64_t u64; int32_t i32; uint32_t u32; pStart = taosDecodeFixedU64(pStart, &u64); - pSyncPing->msgType = u64; + pMsg->msgType = u64; pStart = taosDecodeFixedU64(pStart, &u64); - pSyncPing->srcId.addr = u64; + pMsg->srcId.addr = u64; pStart = taosDecodeFixedI32(pStart, &i32); - pSyncPing->srcId.vgId = i32; + pMsg->srcId.vgId = i32; pStart = taosDecodeFixedU64(pStart, &u64); - pSyncPing->destId.addr = u64; + pMsg->destId.addr = u64; pStart = taosDecodeFixedI32(pStart, &i32); - pSyncPing->destId.vgId = i32; + pMsg->destId.vgId = i32; pStart = taosDecodeFixedU32(pStart, &u32); - pSyncPing->dataLen = u32; + pMsg->dataLen = u32; } #endif \ No newline at end of file diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 029e2bd496..4deceb603e 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -16,59 +16,59 @@ void logTest() { #define PING_MSG_LEN 20 void test1() { - sTrace("test1: ----"); + sTrace("test1: ---- syncPingSerialize, syncPingDeserialize"); char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); - SyncPing* pSyncPing = syncPingBuild(PING_MSG_LEN); - pSyncPing->srcId.addr = 1; - pSyncPing->srcId.vgId = 2; - pSyncPing->destId.addr = 3; - pSyncPing->destId.vgId = 4; - memcpy(pSyncPing->data, msg, PING_MSG_LEN); + SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); + pMsg->srcId.addr = 1; + pMsg->srcId.vgId = 2; + pMsg->destId.addr = 3; + pMsg->destId.vgId = 4; + memcpy(pMsg->data, msg, PING_MSG_LEN); { - cJSON* pJson = syncPing2Json(pSyncPing); + cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); printf("SyncPing: \n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } - uint32_t bufLen = pSyncPing->bytes; + uint32_t bufLen = pMsg->bytes; char* buf = (char*)malloc(bufLen); - syncPingSerialize(pSyncPing, buf, bufLen); + syncPingSerialize(pMsg, buf, bufLen); - SyncPing* pSyncPing2 = (SyncPing*)malloc(pSyncPing->bytes); - syncPingDeserialize(buf, bufLen, pSyncPing2); + SyncPing* pMsg2 = (SyncPing*)malloc(pMsg->bytes); + syncPingDeserialize(buf, bufLen, pMsg2); { - cJSON* pJson = syncPing2Json(pSyncPing2); + cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); printf("SyncPing2: \n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } - syncPingDestroy(pSyncPing); - syncPingDestroy(pSyncPing2); + syncPingDestroy(pMsg); + syncPingDestroy(pMsg2); free(buf); } void test2() { - sTrace("test2: ----"); + sTrace("test2: ---- syncPing2RpcMsg, syncPingFromRpcMsg"); char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "hello raft"); - SyncPing* pSyncPing = syncPingBuild(PING_MSG_LEN); - pSyncPing->srcId.addr = 100; - pSyncPing->srcId.vgId = 200; - pSyncPing->destId.addr = 300; - pSyncPing->destId.vgId = 400; - memcpy(pSyncPing->data, msg, PING_MSG_LEN); + SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); + pMsg->srcId.addr = 100; + pMsg->srcId.vgId = 200; + pMsg->destId.addr = 300; + pMsg->destId.vgId = 400; + memcpy(pMsg->data, msg, PING_MSG_LEN); { - cJSON* pJson = syncPing2Json(pSyncPing); + cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); printf("SyncPing: \n%s\n\n", serialized); free(serialized); @@ -76,23 +76,100 @@ void test2() { } SRpcMsg rpcMsg; - syncPing2RpcMsg(pSyncPing, &rpcMsg); - SyncPing* pSyncPing2 = (SyncPing*)malloc(pSyncPing->bytes); - syncPingFromRpcMsg(&rpcMsg, pSyncPing2); + syncPing2RpcMsg(pMsg, &rpcMsg); + SyncPing* pMsg2 = (SyncPing*)malloc(pMsg->bytes); + syncPingFromRpcMsg(&rpcMsg, pMsg2); rpcFreeCont(rpcMsg.pCont); { - cJSON* pJson = syncPing2Json(pSyncPing2); + cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); printf("SyncPing2: \n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } - syncPingDestroy(pSyncPing); - syncPingDestroy(pSyncPing2); + syncPingDestroy(pMsg); + syncPingDestroy(pMsg2); } +void test3() { + sTrace("test3: ---- syncPingReplySerialize, syncPingReplyDeserialize"); + + char msg[PING_MSG_LEN]; + snprintf(msg, sizeof(msg), "%s", "test ping"); + SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); + pMsg->srcId.addr = 19; + pMsg->srcId.vgId = 29; + pMsg->destId.addr = 39; + pMsg->destId.vgId = 49; + memcpy(pMsg->data, msg, PING_MSG_LEN); + + { + cJSON* pJson = syncPingReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("SyncPingReply: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncPingReplySerialize(pMsg, buf, bufLen); + + SyncPingReply* pMsg2 = (SyncPingReply*)malloc(pMsg->bytes); + syncPingReplyDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncPingReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("SyncPingReply2: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncPingReplyDestroy(pMsg); + syncPingReplyDestroy(pMsg2); + free(buf); +} + +void test4() { + sTrace("test4: ---- syncPingReply2RpcMsg, syncPingReplyFromRpcMsg"); + + char msg[PING_MSG_LEN]; + snprintf(msg, sizeof(msg), "%s", "hello raft"); + SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); + pMsg->srcId.addr = 66; + pMsg->srcId.vgId = 77; + pMsg->destId.addr = 88; + pMsg->destId.vgId = 99; + memcpy(pMsg->data, msg, PING_MSG_LEN); + + { + cJSON* pJson = syncPingReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("SyncPingReply: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pMsg, &rpcMsg); + SyncPingReply* pMsg2 = (SyncPingReply*)malloc(pMsg->bytes); + syncPingReplyFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncPingReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("SyncPingReply2: \n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncPingReplyDestroy(pMsg); + syncPingReplyDestroy(pMsg2); +} int main() { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; @@ -100,6 +177,8 @@ int main() { test1(); test2(); + test3(); + test4(); return 0; } From e697f6301356c9b5f402b8805ce222345f4516c6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 1 Mar 2022 19:34:24 +0800 Subject: [PATCH 13/82] [TD-13736]: console exit input error. --- tools/shell/src/shellMain.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 8a1763c4fc..f62c43773d 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -41,9 +41,11 @@ void *cancelHandler(void *arg) { taosReleaseRef(tscObjRef, rid); #endif #else + reset_terminal_mode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); #endif + reset_terminal_mode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); } From 639b5102ce1d2230e43806a2652f488c0a977bcf Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Mar 2022 19:48:21 +0800 Subject: [PATCH 14/82] feature/qnode --- include/common/tmsg.h | 2 + include/libs/catalog/catalog.h | 2 +- include/libs/qcom/query.h | 2 + source/client/src/clientMsgHandler.c | 38 ++++++++ source/common/src/tmsg.c | 2 + source/dnode/mnode/impl/src/mndDb.c | 53 ++++++----- source/dnode/mnode/impl/src/mnode.c | 2 +- source/libs/catalog/src/catalog.c | 109 +++++++++++++++------- source/libs/catalog/test/catalogTests.cpp | 5 +- source/libs/parser/src/dCDAstProcess.c | 1 + source/libs/qcom/src/querymsg.c | 56 +++++------ 11 files changed, 184 insertions(+), 88 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a739e51d6b..e7b2dfed89 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -167,6 +167,7 @@ typedef struct { typedef struct { char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; int32_t vgVersion; } SBuildUseDBInput; @@ -563,6 +564,7 @@ int32_t tDeserializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; int32_t vgVersion; } SUseDbReq; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ebe7c92d31..c6183780f9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); */ void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId); /** * Get a DB's all vgroup info. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 76f23da1e3..6d3f97fc4e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -156,6 +156,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); */ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); +int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp); + void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 454fb7f1fe..9534c11646 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -19,6 +19,7 @@ #include "clientInt.h" #include "clientLog.h" #include "catalog.h" +#include "query.h" int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -243,6 +244,23 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; + if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + struct SCatalog *pCatalog = NULL; + + if (usedbRsp.vgVersion >= 0) { + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); + } else { + catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid); + } + } + + tFreeSUsedbRsp(&usedbRsp); + } + if (code != TSDB_CODE_SUCCESS) { free(pMsg->pData); setErrno(pRequest, code); @@ -256,6 +274,26 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SName name = {0}; tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + SUseDbOutput output = {0}; + code = queryBuildUseDbOutput(&output, &usedbRsp); + + if (code != 0) { + terrno = code; + if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); + tfree(output.dbVgroup); + + tscError("failed to build use db output since %s", terrstr()); + } else { + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); + } else { + catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup); + } + } + tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8a3cd0a718..8de3906388 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1421,6 +1421,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; tEndEncode(&encoder); @@ -1435,6 +1436,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9165fa2264..974f5fc982 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -940,43 +940,51 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { char *p = strchr(usedbReq.db, '.'); if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + code = 0; } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto USE_DB_OVER; - } - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto USE_DB_OVER; - } + memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + usedbRsp.uid = usedbReq.dbId; + usedbRsp.vgVersion = usedbReq.vgVersion; - if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; - } + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); + } else { + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; - } + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } - if (usedbReq.vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - } + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } + + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + code = 0; + } } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; goto USE_DB_OVER; } @@ -984,7 +992,6 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { pReq->pCont = pRsp; pReq->contLen = contLen; - code = 0; USE_DB_OVER: if (code != 0) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index ab5d0d794b..90d66a80fd 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ PROCESS_RPC_END: if (code == TSDB_CODE_APP_NOT_READY) { mndSendRedirectRsp(pMnode, &pMsg->rpcMsg); } else if (code != 0) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code}; + SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code}; rpcSendResponse(&rpcRsp); } else { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont}; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2b6caa2f8f..f0ea51c2f9 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -166,6 +166,28 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { taosHashGetKey(dbCache, (void **)&dbFName, &len); CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); + + CTG_CACHE_DEBUG("deleted: %d", dbCache->deleted); + if (dbCache->vgInfo) { + CTG_CACHE_DEBUG("vgVersion: %d", dbCache->vgInfo->vgVersion); + CTG_CACHE_DEBUG("hashMethod: %d", dbCache->vgInfo->hashMethod); + if (dbCache->vgInfo->vgHash) { + CTG_CACHE_DEBUG("vgNum: %d", taosHashGetSize(dbCache->vgInfo->vgHash)); + //TODO + } else { + CTG_CACHE_DEBUG("vgHash: %p", dbCache->vgInfo->vgHash); + } + } else { + CTG_CACHE_DEBUG("vgInfo: %p", dbCache->vgInfo); + } + + if (dbCache->tbCache.metaCache) { + CTG_CACHE_DEBUG("metaNum: %d", taosHashGetSize(dbCache->tbCache.metaCache)); + } + + if (dbCache->tbCache.stbCache) { + CTG_CACHE_DEBUG("stbNum: %d", taosHashGetSize(dbCache->tbCache.stbCache)); + } pIter = taosHashIterate(dbHash, pIter); } @@ -242,6 +264,34 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { } +int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { @@ -452,12 +502,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_DB_NOT_EXIST(rpcRsp.code)) { - ctgDebug("db not exist in mnode, dbFName:%s", input->db); - return rpcRsp.code; - } - - ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); + ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -1365,20 +1410,33 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; int32_t code = 0; - if (!forceUpdate) { - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); - if (inCache) { - return TSDB_CODE_SUCCESS; - } + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); + + if (inCache && !forceUpdate) { + return TSDB_CODE_SUCCESS; } SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + if (inCache) { + input.dbId = (*dbCache)->dbId; + input.vgVersion = (*dbCache)->vgInfo->vgVersion; + } else { + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + } - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); + code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); + if (code) { + if (CTG_DB_NOT_EXIST(code) && input.vgVersion > CTG_DEFAULT_INVALID_VERSION) { + ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); + ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); + } + + CTG_ERR_RET(code); + } CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); @@ -1772,7 +1830,6 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { } - void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -1964,10 +2021,10 @@ void catalogFreeHandle(SCatalog* pCtg) { ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1994,6 +2051,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers } *version = dbCache->vgInfo->vgVersion; + *dbId = dbCache->dbId; ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -2099,29 +2157,12 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); - CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: - tfree(action.data); - CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index f3aecadd3f..177592ff38 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -128,6 +128,7 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); ctgDbgEnableDebug("api"); @@ -827,7 +828,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } -#if 0 +#if 1 TEST(tableMeta, normalTable) { @@ -1498,7 +1499,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM)) { usleep(10000); } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6131d50c12..713847807d 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -762,6 +762,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch SUseDbReq usedbReq = {0}; tNameExtractFullName(&n, usedbReq.db); + catalogGetDBVgVersion(pCtx->pCatalog, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId); int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void* pBuf = malloc(bufLen); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 8bc29656d4..2a52e01dc1 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -24,6 +24,32 @@ int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; +int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { + memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN); + pOut->dbId = usedbRsp->uid; + pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); + if (NULL == pOut->dbVgroup) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; + pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; + pOut->dbVgroup->vgHash = + taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup->vgHash) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp->pVgroupInfos, i); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { SBuildTableMetaInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { @@ -57,6 +83,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db)); usedbReq.db[sizeof(usedbReq.db) - 1] = 0; usedbReq.vgVersion = pInput->vgVersion; + usedbReq.dbId = pInput->dbId; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = rpcMallocCont(bufLen); @@ -90,35 +117,10 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_USEDB_OVER; } - memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN); - pOut->dbId = usedbRsp.uid; - pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); - if (NULL == pOut->dbVgroup) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - - pOut->dbVgroup->vgVersion = usedbRsp.vgVersion; - pOut->dbVgroup->hashMethod = usedbRsp.hashMethod; - pOut->dbVgroup->vgHash = - taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pOut->dbVgroup->vgHash) { - tfree(pOut->dbVgroup); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - - for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { - SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - } - - code = 0; + code = queryBuildUseDbOutput(pOut, &usedbRsp); PROCESS_USEDB_OVER: + if (code != 0) { if (pOut) { if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); From aaf5e20fdc6b84538532e306e68793f08b117fe6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 1 Mar 2022 20:29:49 +0800 Subject: [PATCH 15/82] sync encode test --- source/libs/sync/inc/syncInt.h | 18 ++++++------ source/libs/sync/src/syncIO.c | 20 ++++++++----- source/libs/sync/src/syncMain.c | 51 +++++++++++++++++++++++---------- source/libs/sync/src/syncUtil.c | 2 +- 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 82108acf7b..d75c5424ea 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -97,14 +97,17 @@ typedef struct SRaftId { } SRaftId; typedef struct SSyncNode { - int8_t replica; - int8_t quorum; - int32_t refCount; - int64_t rid; - SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + SSyncFSM* pFsm; + + // passed from outside + void* rpcClient; + int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); + + int32_t refCount; + int64_t rid; SNodeInfo me; SNodeInfo peers[TSDB_MAX_REPLICA]; @@ -112,7 +115,6 @@ typedef struct SSyncNode { ESyncRole role; SRaftId raftId; - SSyncFSM* pFsm; tmr_h pPingTimer; int32_t pingTimerMS; @@ -145,10 +147,6 @@ typedef struct SSyncNode { int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); - // passed from outside - void* rpcClient; - int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); - } SSyncNode; SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 5c27f9ec3a..c035ad5d6b 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,19 +30,26 @@ static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, S static int32_t doSyncIODestroy(SSyncIO *io); static SSyncIO *syncIOCreate(); -static void * syncIOConsumer(void *param); +static void *syncIOConsumer(void *param); static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); static void syncIODoReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIODoRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOTick(void *param, void *tmrId); // ---------------------------- -int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { + pMsg->handle = NULL; + rpcSendRequest(handle, pEpSet, pMsg, NULL); + return 0; +} int32_t syncIOStart() { gSyncIO = syncIOCreate(); assert(gSyncIO != NULL); + int32_t ret = doSyncIOStart(gSyncIO); + assert(ret == 0); + return 0; } @@ -67,15 +74,14 @@ static void syncIOTick(void *param, void *tmrId) { taosWriteQitem(io->pMsgQ, pTemp); - bool b = taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, io->syncTimer); - assert(b); + taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, io->syncTimer); } static void *syncIOConsumer(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); @@ -215,8 +221,8 @@ static int32_t doSyncIOStart(SSyncIO *io) { } // start tmr thread - io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); + // io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + // io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4d9e5887cd..0aa3e56062 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -69,15 +69,34 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->vgId = pSyncInfo->vgId; + pSyncNode->syncCfg = pSyncInfo->syncCfg; + memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); + pSyncNode->pFsm = pSyncInfo->pFsm; + + pSyncNode->rpcClient = pSyncInfo->rpcClient; + pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; + + pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; + pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; + + int j = 0; + for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { + if (i != pSyncInfo->syncCfg.myIndex) { + pSyncNode->peers[j] = pSyncInfo->syncCfg.nodeInfo[i]; + j++; + } + } + + pSyncNode->role = TAOS_SYNC_STATE_FOLLOWER; + syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId); + pSyncNode->pPingTimer = NULL; pSyncNode->pingTimerMS = 1000; atomic_store_8(&pSyncNode->pingTimerStart, 0); pSyncNode->FpPingTimer = syncNodePingTimerCb; pSyncNode->pingTimerCounter = 0; - pSyncNode->rpcClient = pSyncInfo->rpcClient; - pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; - pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; @@ -97,10 +116,11 @@ void syncNodePingAll(SSyncNode* pSyncNode) { sTrace("syncNodePingAll %p ", pSyncNode); int32_t ret = 0; for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { - SyncPing* pSyncPing; - SRaftId raftId; - syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &raftId); - ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + SyncPing* pMsg = syncPingBuild(strlen("ping") + 1); + memcpy(pMsg->data, "ping", strlen("ping") + 1); + syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &pMsg->destId); + pMsg->srcId = pSyncNode->raftId; + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); } } @@ -118,10 +138,11 @@ void syncNodePingPeers(SSyncNode* pSyncNode) { void syncNodePingSelf(SSyncNode* pSyncNode) { int32_t ret = 0; - SyncPing* pSyncPing; - SRaftId raftId; - syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &raftId); - ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + SyncPing* pMsg = syncPingBuild(strlen("ping") + 1); + memcpy(pMsg->data, "ping", strlen("ping") + 1); + pMsg->destId = pSyncNode->raftId; + pMsg->srcId = pSyncNode->raftId; + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); } @@ -146,10 +167,10 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { // ------ local funciton --------- static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - int32_t ret = 0; - SRpcMsg* rpcMsg; - syncPing2RpcMsg(pMsg, rpcMsg); - syncNodeSendMsgById(destRaftId, pSyncNode, rpcMsg); + int32_t ret = 0; + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); return ret; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 080840bbf6..0066ab783e 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -22,7 +22,7 @@ uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint64_t u64; uint32_t hostU32 = (uint32_t)inet_addr(host); - assert(hostU32 != (uint32_t)-1); + // assert(hostU32 != (uint32_t)-1); u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); return u64; } From 539f1238ba777b086e25d39987b1e680a901d49a Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 1 Mar 2022 22:11:38 +0800 Subject: [PATCH 16/82] [TD-13736]: console exit input error. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index db84f87b66..00cf770e6b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -271,7 +271,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 0, 1) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; From f5c13c13c6d1a6ddfd3a3ff18ae979cba342bd9a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 1 Mar 2022 22:36:19 +0800 Subject: [PATCH 17/82] support json --- include/libs/index/index.h | 114 +++++++++++++++++++++++++--- source/libs/index/inc/indexInt.h | 57 +++++++++----- source/libs/index/src/index.c | 49 +++++++++--- source/libs/index/src/index_cache.c | 4 +- source/libs/index/src/index_json.c | 44 +++++++++++ source/libs/index/src/index_tfile.c | 13 ++-- source/libs/index/test/fstUT.cc | 10 +-- source/libs/index/test/jsonDemo.cc | 0 8 files changed, 241 insertions(+), 50 deletions(-) create mode 100644 source/libs/index/src/index_json.c create mode 100644 source/libs/index/test/jsonDemo.cc diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 47eb97cc3a..9424eae1b7 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -29,6 +29,12 @@ typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef struct SIndex SIndexJson; +typedef struct SIndexTerm SIndexJsonTerm; +typedef struct SIndexOpts SIndexJsonOpts; +typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery; +typedef struct SArray SIndexJsonMultiTerm; + typedef enum { ADD_VALUE, // add index colume value DEL_VALUE, // delete index column value @@ -39,24 +45,108 @@ typedef enum { } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; + /* - * @param: oper - * + * create multi query + * @param oper (input, relation between querys) */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); + /* - * @param: - * @param: + * destroy multi query + * @param pQuery (input, multi-query-object to be destory) + */ + +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +/* + * add query to multi query + * @param pQuery (input, multi-query-object) + * @param term (input, single query term) + * @param type (input, single query type) + * @return error code + */ +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); +/* + * open index + * @param opt (input, index opt) + * @param path (input, index path) + * @param index (output, index object) + * @return error code + */ +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code */ -int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); void indexClose(SIndex* index); -int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); -int indexDelete(SIndex* index, SIndexMultiTermQuery* query); -int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); -int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +/* + * delete terms that meet query condition + * @param index (input, index object) + * @param query (input, condition query to deleted) + * @return error code + */ + +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +/* + * rebuild index + * @param index (input, index object) + * @parma opt (input, rebuild index opts) + * @return error code + */ +int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * open index + * @param opt (input,index json opt) + * @param path (input, index json path) + * @param index (output, index json object) + * @return error code + */ +int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code + */ + +int tIndexJsonClose(SIndexJson* index); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ + +int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result); /* * @param * @param diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 90ad1e15f4..57d587d297 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -120,29 +120,50 @@ int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ + do { \ + uint8_t oldTy = ty; \ + ty = (ty >> 4) | exTy; \ + ty = (ty << 4) | oldTy; \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5147734a85..b518df2ea2 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -2,8 +2,8 @@ * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -30,6 +30,8 @@ void* indexQhandle = NULL; +static char JSON_COLUMN[] = "JSON"; + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -63,6 +65,9 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); static void indexMergeSameKey(SArray* result, TFileValue* tv); +static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); +int32_t indexSerialKey(ICacheKey* key, char* buf); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); @@ -147,9 +152,8 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { @@ -162,9 +166,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + // ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -554,7 +558,24 @@ END: return -1; } -int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +int32_t indexSerialTermKeyOfTag(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + return indexSerialKey(&key, buf); +} +int32_t indexSerilaTermKeyOfJson(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = JSON_COLUMN, .nColName = strlen(JSON_COLUMN)}; + return indexSerialKey(&key, buf); +} + +int32_t indexSerialTermKey(SIndexTerm* itm, char* buf) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(itm->colType, TSDB_DATA_TYPE_JSON); + if (hasJson) { + return indexSerilaTermKeyOfJson(itm, buf); + } else { + return indexSerialTermKeyOfTag(itm, buf); + } +} +int32_t indexSerialKey(ICacheKey* key, char* buf) { char* p = buf; SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_VAR_TO_BUF(buf, '_', char); @@ -563,3 +584,13 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); return buf - p; } + +// int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +// char* p = buf; +// SERIALIZE_MEM_TO_BUF(buf, key, suid); +// SERIALIZE_VAR_TO_BUF(buf, '_', char); +// // SERIALIZE_MEM_TO_BUF(buf, key, colType); +// // SERIALIZE_VAR_TO_BUF(buf, '_', char); +// SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +// return buf - p; +//} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index d6a7141825..bb29a78cb1 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -24,6 +24,8 @@ #define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 +static char JSON_COLUMN[] = "JSON"; + static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -45,7 +47,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in return NULL; }; cache->mem = indexInternalCacheCreate(type); - cache->colName = tstrdup(colName); + cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c new file mode 100644 index 0000000000..f4d28b45b0 --- /dev/null +++ b/source/libs/index/src/index_json.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "indeInt.h" +#include "index.h" + +int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { + // handle + return tIndexOpen(opts, path, index); +} +// k +int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexPut(index, terms, uid); + // handle put +} + +int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *query, SArray *result) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexSearch(index, query, result); + // handle search +} + +int tIndexJsonClose(SIndexJson *index) { + return tIndexClose(index); + // handle close +} diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index e44f8fc1c3..a05884b790 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -205,7 +205,11 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul } else if (qtype == QUERY_PREFIX) { // handle later // - } else { + } else if (qtype == QUERY_SUFFIX) { + // handle later + } else if (qtype == QUERY_REGEX) { + // handle later + } else if (qtype == QUERY_RANGE) { // handle later } tfileReaderUnRef(reader); @@ -586,11 +590,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, + reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d59a3428da..b34fd71e9c 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -216,21 +216,21 @@ class FstEnv : public ::testing::Test { TEST_F(FstEnv, writeNormal) { fst->CreateWriter(); - std::string str("aa"); + std::string str("11"); for (int i = 0; i < 10; i++) { - str[0] = 'a' + i; + str[0] = '1' + i; str.resize(2); assert(fst->Put(str, i) == true); } // order failed - assert(fst->Put("aa", 1) == false); + assert(fst->Put("11", 1) == false); fst->DestroyWriter(); fst->CreateReader(); uint64_t val; - assert(fst->Get("a", &val) == false); - assert(fst->Get("aa", &val) == true); + assert(fst->Get("1", &val) == false); + assert(fst->Get("11", &val) == true); assert(val == 0); std::vector rlt; diff --git a/source/libs/index/test/jsonDemo.cc b/source/libs/index/test/jsonDemo.cc new file mode 100644 index 0000000000..e69de29bb2 From 16b598f6c37c83e7d547bfeca392cf554eb7eb95 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 2 Mar 2022 08:31:17 +0800 Subject: [PATCH 18/82] feature/qnode --- source/libs/catalog/test/catalogTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 177592ff38..b7432429f4 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -828,7 +828,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } -#if 1 +#if 0 TEST(tableMeta, normalTable) { From c5a74d6f993cf7783dc6c0e05e2b99979bb26c4a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 09:22:22 +0800 Subject: [PATCH 19/82] do not create default dataDir on startup --- source/common/src/tglobal.c | 16 ++++++++++++++++ source/util/src/tconfig.c | 2 ++ 2 files changed, 18 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5745a70ca2..d66895888b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -467,6 +467,12 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + if (taosMkDir(tsLogDir) != 0) { + uError("failed to create dir:%s since %s", tsLogDir, terrstr()); + cfgCleanup(pCfg); + return -1; + } + if (taosInitLog(logname, logFileNum) != 0) { uError("failed to init log file since %s", terrstr()); cfgCleanup(pCfg); @@ -514,6 +520,16 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU } taosSetSystemCfg(tsCfg); + if (taosMkDir(tsTempDir) != 0) { + uError("failed to create dir:%s since %s", tsTempDir, terrstr()); + return -1; + } + + if (!tsc && taosMkDir(tsDataDir) != 0) { + uError("failed to create dir:%s since %s", tsDataDir, terrstr()); + return -1; + } + cfgDumpCfg(tsCfg, tsc, false); return 0; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 84af091fb3..0f22d15cd4 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -145,10 +145,12 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } +#if 0 if (taosMkDir(fullDir) != 0) { uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); return -1; } +#endif cfgFreeItem(pItem); pItem->str = strdup(fullDir); From d7c3415ae0a5806f3a8710dc6fb13f4816197c25 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 09:24:17 +0800 Subject: [PATCH 20/82] support json --- source/libs/index/src/index_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c index f4d28b45b0..9ffab3fad1 100644 --- a/source/libs/index/src/index_json.c +++ b/source/libs/index/src/index_json.c @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "indeInt.h" #include "index.h" +#include "indexInt.h" int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { // handle From 29422c659811ce922d9e0af7e390b4a71637f56d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 09:55:21 +0800 Subject: [PATCH 21/82] telemetry --- include/util/thttp.h | 31 +++++++++++ source/dnode/mnode/impl/src/mndTelem.c | 61 ++------------------- source/dnode/mnode/impl/src/mnode.c | 2 +- source/util/src/thttp.c | 75 ++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 59 deletions(-) create mode 100644 include/util/thttp.h create mode 100644 source/util/src/thttp.c diff --git a/include/util/thttp.h b/include/util/thttp.h new file mode 100644 index 0000000000..f211b2615d --- /dev/null +++ b/include/util/thttp.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_HTTP_H_ +#define _TD_UTIL_HTTP_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_UTIL_H_*/ diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 744cefcb4f..1cd7593846 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -19,8 +19,9 @@ #include "mndSync.h" #include "tbuffer.h" #include "tjson.h" +#include "thttp.h" -#define TELEMETRY_SERVER "localhost" +#define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { @@ -81,62 +82,6 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { return pCont; } -static void mndSendTelemetryReport(const char* pCont) { - int32_t code = -1; - char buf[128] = {0}; - SOCKET fd = 0; - int32_t contLen = strlen(pCont); - - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mError("failed to get telemetry server ip"); - goto SEND_OVER; - } - - fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mError("failed to create telemetry socket"); - goto SEND_OVER; - } - - const char* header = - "POST /report HTTP/1.1\n" - "Host: " TELEMETRY_SERVER - "\n" - "Content-Type: application/json\n" - "Content-Length: "; - if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { - mError("failed to send telemetry header"); - goto SEND_OVER; - } - - snprintf(buf, sizeof(buf), "%d\n\n", contLen); - if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { - mError("failed to send telemetry contlen"); - goto SEND_OVER; - } - - if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { - mError("failed to send telemetry content"); - goto SEND_OVER; - } - - // read something to avoid nginx error 499 - if (taosReadSocket(fd, buf, 10) < 0) { - mError("failed to receive telemetry response"); - goto SEND_OVER; - } - - mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); - code = 0; - -SEND_OVER: - if (code != 0) { - mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); - } - taosCloseSocket(fd); -} - static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; @@ -145,7 +90,7 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { taosWLockLatch(&pMgmt->lock); char* pCont = mndBuildTelemetryReport(pMnode); if (pCont != NULL) { - mndSendTelemetryReport(pCont); + taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont)); free(pCont); } taosWUnLockLatch(&pMgmt->lock); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 879f866214..ab5d0d794b 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -132,7 +132,7 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTelem, 300, pMnode, pMnode->timer, &pMnode->telemTimer)) { + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c new file mode 100644 index 0000000000..14c39d3f03 --- /dev/null +++ b/source/util/src/thttp.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "thttp.h" +#include "taoserror.h" +#include "tlog.h" + +int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen) { + int32_t code = -1; + SOCKET fd = 0; + + uint32_t ip = taosGetIpv4FromFqdn(server); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get http server:%s ip since %s", server, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, port, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create http socket since %s", terrstr()); + goto SEND_OVER; + } + + char header[4096] = {0}; + int32_t headLen = snprintf(header, sizeof(header), + "POST /report HTTP/1.1\n" + "Host: %s\n" + "Content-Type: application/json\n" + "Content-Length: %d\n\n", + server, contLen); + + if (taosWriteSocket(fd, (void*)header, headLen) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to send http header since %s", terrstr()); + goto SEND_OVER; + } + + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to send http content since %s", terrstr()); + goto SEND_OVER; + } + + // read something to avoid nginx error 499 + if (taosReadSocket(fd, header, 10) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to receive response since %s", terrstr()); + goto SEND_OVER; + } + + uInfo("send http to %s:%d, len:%d content: %s", server, port, contLen, pCont); + code = 0; + +SEND_OVER: + if (fd != 0) { + taosCloseSocket(fd); + } + + return code; +} From 804e52569793849020f29bff828d927ec1714d8b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Mar 2022 10:18:42 +0800 Subject: [PATCH 22/82] [td-13039] add new hash. --- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/tsimplehash.h | 104 +++++++++ source/libs/executor/src/tsimplehash.c | 309 +++++++++++++++++++++++++ source/util/src/thash.c | 3 +- 4 files changed, 415 insertions(+), 3 deletions(-) create mode 100644 source/libs/executor/inc/tsimplehash.h create mode 100644 source/libs/executor/src/tsimplehash.c diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index cff7546a7e..17f457e991 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -68,7 +68,7 @@ typedef struct SResultRow { } SResultRow; typedef struct SResultRowInfo { - SResultRow *pCurResult; // current active result row info + SList* pRows; SResultRow** pResult; // result list // int16_t type:8; // data type for hash key int32_t size; // number of result set diff --git a/source/libs/executor/inc/tsimplehash.h b/source/libs/executor/inc/tsimplehash.h new file mode 100644 index 0000000000..a1ba70c702 --- /dev/null +++ b/source/libs/executor/inc/tsimplehash.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSIMPLEHASH_H +#define TDENGINE_TSIMPLEHASH_H + +#include "tarray.h" +#include "tlockfree.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); +typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); +typedef void (*_hash_free_fn_t)(void *); + +typedef struct SSHashObj SSHashObj; + +/** + * init the hash table + * + * @param capacity initial capacity of the hash table + * @param fn hash function to generate the hash value + * @return + */ +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen); + +/** + * return the size of hash table + * @param pHashObj + * @return + */ +int32_t tSimpleHashGetSize(const SSHashObj *pHashObj); + +/** + * put element into hash table, if the element with the same key exists, update it + * @param pHashObj + * @param key + * @param data + * @return + */ +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data); + +/** + * return the payload data with the specified key + * + * @param pHashObj + * @param key + * @return + */ +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key); + +/** + * remove item with the specified key + * @param pHashObj + * @param key + * @param keyLen + */ +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key); + +/** + * Clear the hash table. + * @param pHashObj + */ +void tSimpleHashClear(SSHashObj *pHashObj); + +/** + * Clean up hash table and release all allocated resources. + * @param handle + */ +void tSimpleHashCleanup(SSHashObj *pHashObj); + +/** + * Get the hash table size + * @param pHashObj + * @return + */ +size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj); + +/** + * Get the corresponding key information for a given data in hash table + * @param data + * @param keyLen + * @return + */ +void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen); + +#ifdef __cplusplus +} +#endif +#endif // TDENGINE_TSIMPLEHASH_H diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c new file mode 100644 index 0000000000..4012c3926e --- /dev/null +++ b/source/libs/executor/src/tsimplehash.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tsimplehash.h" +#include "taoserror.h" + +#define SHASH_DEFAULT_LOAD_FACTOR 0.75 +#define HASH_MAX_CAPACITY (1024*1024*16) +#define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) + +#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl)) +#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode)) + +#define HASH_INDEX(v, c) ((v) & ((c)-1)) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) + +#define FREE_HASH_NODE(_n) \ + do { \ + tfree(_n); \ + } while (0); + +typedef struct SHNode { + struct SHNode *next; + char data[]; +} SHNode; + +typedef struct SSHashObj { + SHNode **hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + int32_t keyLen; + int32_t dataLen; +} SSHashObj; + +static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { + int32_t len = MIN(length, HASH_MAX_CAPACITY); + + int32_t i = 4; + while (i < len) i = (i << 1u); + return i; +} + +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) { + ASSERT(fn != NULL); + + if (capacity == 0) { + capacity = 4; + } + + SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj)); + if (pHashObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // the max slots is not defined by user + pHashObj->capacity = taosHashCapacity((int32_t)capacity); + + pHashObj->equalFp = memcmp; + pHashObj->hashFp = fn; + ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + + pHashObj->keyLen = keyLen; + pHashObj->dataLen = dataLen; + + pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *)); + if (pHashObj->hashList == NULL) { + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + return pHashObj; +} + +int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { + SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize); + if (pNewNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pNewNode->next = NULL; + memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen); + return pNewNode; +} + +void taosHashTableResize(SSHashObj *pHashObj) { + if (!HASH_NEED_RESIZE(pHashObj)) { + return; + } + + int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); + if (newCapacity > HASH_MAX_CAPACITY) { +// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", +// pHashObj->capacity, HASH_MAX_CAPACITY); + return; + } + + int64_t st = taosGetTimestampUs(); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + if (pNewEntryList == NULL) { +// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + return; + } + + size_t inc = newCapacity - pHashObj->capacity; + memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + + pHashObj->hashList = pNewEntryList; + pHashObj->capacity = newCapacity; + + for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { + SHNode* pNode = pHashObj->hashList[idx]; + SHNode *pNext; + SHNode *pPrev = NULL; + + if (pNode == NULL) { + continue; + } + + while (pNode != NULL) { + void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen); + + int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); + pNext = pNode->next; + if (newIdx != idx) { + if (pPrev == NULL) { + pHashObj->hashList[idx] = pNext; + } else { + pPrev->next = pNext; + } + + pNode->next = pHashObj->hashList[newIdx]; + pHashObj->hashList[newIdx] = pNode; + } else { + pPrev = pNode; + } + + pNode = pNext; + } + } + + int64_t et = taosGetTimestampUs(); + +// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, +// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +} + +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { + if (pHashObj == NULL || key == NULL) { + return -1; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + + // need the resize process, write lock applied + if (SHASH_NEED_RESIZE(pHashObj)) { + taosHashTableResize(pHashObj); + } + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + + SHNode *pNode = pHashObj->hashList[slot]; + if (pNode == NULL) { + SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal); + if (pNewNode == NULL) { + return -1; + } + + pHashObj->hashList[slot] = pNewNode; + return 0; + } + + while (pNode) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + break; + } + pNode = pNode->next; + } + + if (pNode == NULL) { + SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal); + if (pNewNode == NULL) { + return -1; + } + pNewNode->next = pHashObj->hashList[slot]; + pHashObj->hashList[slot] = pNewNode; + atomic_add_fetch_64(&pHashObj->size, 1); + } else { //update data + memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); + } + + return 0; +} + +static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) { + SHNode *pNode = pHashObj->hashList[index]; + while (pNode) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + break; + } + + pNode = pNode->next; + } + + return pNode; +} + +static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { + return tSimpleHashGetSize(pHashObj) == 0; +} + +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHNode *pNode = pHashObj->hashList[slot]; + if (pNode == NULL) { + return NULL; + } + + char *data = NULL; + pNode = doSearchInEntryList(pHashObj, key, slot); + if (pNode != NULL) { + data = GET_SHASH_NODE_DATA(pNode); + } + + return data; +} + +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) { + // todo +} + +void tSimpleHashClear(SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + SHNode *pNode, *pNext; + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pNode = pHashObj->hashList[i]; + if (pNode == NULL) { + continue; + } + + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pNode); + pNode = pNext; + } + } + pHashObj->size = 0; +} + +void tSimpleHashCleanup(SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + tSimpleHashClear(pHashObj); + tfree(pHashObj->hashList); +} + +size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); +} + +void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { + int32_t offset = offsetof(SHNode, data); + SHNode *node = data - offset; + if (keyLen != NULL) { + *keyLen = pHashObj->keyLen; + } + + return GET_SHASH_NODE_KEY(node, pHashObj->dataLen); +} \ No newline at end of file diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 05bc94caef..87ae48a3b3 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -274,7 +274,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp } taosArrayPush(pHashObj->pMemBlock, &p); - return pHashObj; } @@ -302,7 +301,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - if (pHashObj == NULL || key == NULL || keyLen == 0 || data == NULL || size == 0) { + if (pHashObj == NULL || key == NULL || keyLen == 0) { return -1; } From 3d12401119b73740401f985d134e8dcb2ff92774 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Mar 2022 10:34:29 +0800 Subject: [PATCH 23/82] [td-13039] refactor. --- source/libs/executor/inc/executorimpl.h | 30 +++--- source/libs/executor/src/executorMain.c | 2 +- source/libs/executor/src/executorimpl.c | 112 ++++++++++---------- source/libs/executor/test/executorTests.cpp | 12 +-- 4 files changed, 77 insertions(+), 79 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0ae54f7ad1..10ee4d157f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -234,8 +234,8 @@ typedef struct STaskAttr { } STaskAttr; typedef int32_t (*__optr_open_fn_t)(void* param); -typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); -typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); +typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup); +typedef void (*__optr_close_fn_t)(void* param, int32_t num); struct SOperatorInfo; @@ -306,21 +306,21 @@ enum { }; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char* name; // name, used to show the query execution plan - void* info; // extension attribution - SExprInfo* pExpr; - STaskRuntimeEnv* pRuntimeEnv; // todo remove it - SExecTaskInfo* pTaskInfo; + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char* name; // name, used to show the query execution plan + void* info; // extension attribution + SExprInfo* pExpr; + STaskRuntimeEnv* pRuntimeEnv; // todo remove it + SExecTaskInfo* pTaskInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_open_fn_t prepareFn; - __operator_fn_t exec; - __optr_cleanup_fn_t cleanupFn; + __optr_open_fn_t openFn; + __optr_fn_t nextDataFn; + __optr_close_fn_t closeFn; } SOperatorInfo; typedef struct { @@ -654,8 +654,6 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); // SSDataBlock* doSLimit(void* param, bool* newgroup); // int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 1642120d15..461235f2ab 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int64_t st = 0; st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a8109e7363..18cfe86553 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5281,7 +5281,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = size; - pOperator->exec = doLoadRemoteData; + pOperator->nextDataFn = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; #if 1 @@ -5361,7 +5361,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->exec = doTableScan; + pOperator->nextDataFn = doTableScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -5386,7 +5386,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pOperator->info = pInfo; pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScanImpl; + pOperator->nextDataFn = doTableScanImpl; return pOperator; } @@ -5410,7 +5410,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; // pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->exec = doBlockInfoScan; + pOperator->nextDataFn = doBlockInfoScan; return pOperator; } @@ -5452,7 +5452,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->exec = doStreamBlockScan; + pOperator->nextDataFn = doStreamBlockScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -5663,7 +5663,7 @@ SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*) param; bool newgroup = false; - return pOperator->exec(pOperator, &newgroup); + return pOperator->nextDataFn(pOperator, &newgroup); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) { @@ -5983,8 +5983,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doSortedMerge; - pOperator->cleanupFn = destroySortedMergeOperatorInfo; + pOperator->nextDataFn = doSortedMerge; + pOperator->closeFn = destroySortedMergeOperatorInfo; code = appendDownstream(pOperator, downstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -6079,8 +6079,8 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doSort; - pOperator->cleanupFn = destroyOrderOperatorInfo; + pOperator->nextDataFn = doSort; + pOperator->closeFn = destroyOrderOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6105,7 +6105,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6155,7 +6155,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6241,7 +6241,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6299,7 +6299,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while (1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6350,7 +6350,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { while (1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6393,7 +6393,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6453,7 +6453,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6516,7 +6516,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6571,7 +6571,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6706,7 +6706,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6768,7 +6768,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -6821,7 +6821,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -6906,7 +6906,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -6979,8 +6979,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->cleanupFn != NULL) { - pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput); + if (pOperator->closeFn != NULL) { + pOperator->closeFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -7067,8 +7067,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doAggregate; - pOperator->cleanupFn = destroyAggOperatorInfo; + pOperator->nextDataFn = doAggregate; + pOperator->closeFn = destroyAggOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7164,8 +7164,8 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->numOfOutput = numOfOutput; - pOperator->exec = doMultiTableAggregate; - pOperator->cleanupFn = destroyAggOperatorInfo; + pOperator->nextDataFn = doMultiTableAggregate; + pOperator->closeFn = destroyAggOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7194,8 +7194,8 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doProjectOperation; - pOperator->cleanupFn = destroyProjectOperatorInfo; + pOperator->nextDataFn = doProjectOperation; + pOperator->closeFn = destroyProjectOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7250,10 +7250,10 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->status = OP_IN_EXECUTING; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doFilter; + pOperator->nextDataFn = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroyConditionOperatorInfo; + pOperator->closeFn = destroyConditionOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7269,7 +7269,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn // pOperator->operatorType = OP_Limit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doLimit; + pOperator->nextDataFn = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7311,8 +7311,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; - pOperator->exec = doIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7336,8 +7336,8 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doAllIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7360,8 +7360,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doStateWindowAgg; - pOperator->cleanupFn = destroyStateWindowOperatorInfo; + pOperator->nextDataFn = doStateWindowAgg; + pOperator->closeFn = destroyStateWindowOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7385,8 +7385,8 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSessionWindowAgg; - pOperator->cleanupFn = destroySWindowOperatorInfo; + pOperator->nextDataFn = doSessionWindowAgg; + pOperator->closeFn = destroySWindowOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7409,8 +7409,8 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSTableIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doSTableIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7433,8 +7433,8 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doAllSTableIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7465,8 +7465,8 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashGroupbyAggregate; - pOperator->cleanupFn = destroyGroupbyOperatorInfo; + pOperator->nextDataFn = hashGroupbyAggregate; + pOperator->closeFn = destroyGroupbyOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7504,8 +7504,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doFill; - pOperator->cleanupFn = destroySFillOperatorInfo; + pOperator->nextDataFn = doFill; + pOperator->closeFn = destroySFillOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7553,7 +7553,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI // pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroySlimitOperatorInfo; + pOperator->closeFn = destroySlimitOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7707,11 +7707,11 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->exec = doTagScan; + pOperator->nextDataFn = doTagScan; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroyTagScanOperatorInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; return pOperator; } @@ -7777,7 +7777,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -7849,9 +7849,9 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashDistinct; + pOperator->nextDataFn = hashDistinct; pOperator->pExpr = pExpr; - pOperator->cleanupFn = destroyDistinctOperatorInfo; + pOperator->closeFn = destroyDistinctOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 262232ceb1..85f644335e 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -201,9 +201,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { - pOperator->exec = getDummyBlock; + pOperator->nextDataFn = getDummyBlock; } else { - pOperator->exec = get2ColsDummyBlock; + pOperator->nextDataFn = get2ColsDummyBlock; } SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); @@ -435,7 +435,7 @@ TEST(testCase, external_sort_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); @@ -507,7 +507,7 @@ TEST(testCase, sorted_merge_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); @@ -560,7 +560,7 @@ TEST(testCase, time_interval_Operator_Test) { while(1) { int64_t s = taosGetTimestampUs(); - pRes = pOperator->exec(pOperator, &newgroup); + pRes = pOperator->nextDataFn(pOperator, &newgroup); int64_t e = taosGetTimestampUs(); if (t++ == 1) { @@ -583,7 +583,7 @@ TEST(testCase, time_interval_Operator_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); From ee43a70c4dde610857508312228306d1c5260928 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 10:43:59 +0800 Subject: [PATCH 24/82] sync modify timer --- source/libs/sync/inc/syncInt.h | 30 ++++++++-------- source/libs/sync/inc/syncMessage.h | 4 +++ source/libs/sync/src/syncIO.c | 11 +++--- source/libs/sync/src/syncMain.c | 49 ++++++++++++++++++-------- source/libs/sync/src/syncMessage.c | 14 ++++++++ source/libs/sync/test/syncPingTest.cpp | 4 ++- 6 files changed, 76 insertions(+), 36 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index d75c5424ea..0901330488 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -48,23 +48,23 @@ extern int32_t sDebugFlag; taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \ } \ } -#define sInfo(...) \ - { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sInfo(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SYN INFO ", sDebugFlag, __VA_ARGS__); \ + } \ } -#define sDebug(...) \ - { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sDebug(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SYN DEBUG ", sDebugFlag, __VA_ARGS__); \ + } \ } -#define sTrace(...) \ - { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sTrace(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SYN TRACE ", sDebugFlag, __VA_ARGS__); \ + } \ } struct SRaft; diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index a7de7b9019..603ef00517 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -72,6 +72,10 @@ void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); cJSON* syncPing2Json(const SyncPing* pMsg); +SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); + +SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId); + typedef struct SyncPingReply { uint32_t bytes; uint32_t msgType; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c035ad5d6b..27434a4029 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -38,6 +38,7 @@ static void syncIOTick(void *param, void *tmrId); // ---------------------------- int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { + sTrace("syncIOSendMsg ... "); pMsg->handle = NULL; rpcSendRequest(handle, pEpSet, pMsg, NULL); return 0; @@ -74,7 +75,7 @@ static void syncIOTick(void *param, void *tmrId) { taosWriteQitem(io->pMsgQ, pTemp); - taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, io->syncTimer); + taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, &io->syncTimer); } static void *syncIOConsumer(void *param) { @@ -191,7 +192,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 38000; + rpcInit.localPort = 7010; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; rpcInit.cfp = syncIODoRequest; @@ -209,7 +210,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { } io->epSet.inUse = 0; - addEpIntoEpSet(&io->epSet, "127.0.0.1", 38000); + addEpIntoEpSet(&io->epSet, "127.0.0.1", 7010); // start consumer thread { @@ -221,8 +222,8 @@ static int32_t doSyncIOStart(SSyncIO *io) { } // start tmr thread - // io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - // io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); + io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0aa3e56062..9e335c5e55 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -113,37 +113,36 @@ void syncNodeClose(SSyncNode* pSyncNode) { } void syncNodePingAll(SSyncNode* pSyncNode) { - sTrace("syncNodePingAll %p ", pSyncNode); + sTrace("syncNodePingAll pSyncNode:%p ", pSyncNode); int32_t ret = 0; for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { - SyncPing* pMsg = syncPingBuild(strlen("ping") + 1); - memcpy(pMsg->data, "ping", strlen("ping") + 1); - syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &pMsg->destId); - pMsg->srcId = pSyncNode->raftId; - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); + syncPingDestroy(pMsg); } } void syncNodePingPeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { - SyncPing* pSyncPing; - SRaftId raftId; - syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &raftId); - ret = syncNodePing(pSyncNode, &raftId, pSyncPing); + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); + syncPingDestroy(pMsg); } } void syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret = 0; - SyncPing* pMsg = syncPingBuild(strlen("ping") + 1); - memcpy(pMsg->data, "ping", strlen("ping") + 1); - pMsg->destId = pSyncNode->raftId; - pMsg->srcId = pSyncNode->raftId; + int32_t ret; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &pSyncNode->raftId); ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); + syncPingDestroy(pMsg); } int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { @@ -167,10 +166,29 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { // ------ local funciton --------- static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + sTrace("syncNodePing pSyncNode:%p ", pSyncNode); int32_t ret = 0; SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + + { + cJSON* pJson = syncPing2Json(pMsg); + char* serialized = cJSON_Print(pJson); + sTrace("syncNodePing pMsg:%s ", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + { + SyncPing* pMsg2 = rpcMsg.pCont; + cJSON* pJson = syncPing2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + sTrace("syncNodePing pMsg2:%s ", serialized); + free(serialized); + cJSON_Delete(pJson); + } + return ret; } @@ -185,6 +203,7 @@ static int32_t syncNodeAppendEntries(SSyncNode* ths, const SyncAppendEntries* pM } static int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { + sTrace("syncNodeSendMsgById pSyncNode:%p ", pSyncNode); SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 3122aa66bb..e0975fb3c1 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -80,6 +80,20 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { return pJson; } +SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) { + uint32_t dataLen = strlen(str) + 1; + SyncPing* pMsg = syncPingBuild(dataLen); + pMsg->srcId = *srcId; + pMsg->destId = *destId; + snprintf(pMsg->data, pMsg->dataLen, "%s", str); + return pMsg; +} + +SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) { + SyncPing* pMsg = syncPingBuild2(srcId, destId, "ping"); + return pMsg; +} + // ---- message process SyncPingReply---- SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { uint32_t bytes = SYNC_PING_REPLY_FIX_LEN + dataLen; diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 43196bdd1f..b57f6c5259 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -25,7 +25,7 @@ SSyncNode* doSync() { SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = 0; - pCfg->replicaNum = 3; + pCfg->replicaNum = 1; pCfg->nodeInfo[0].nodePort = 7010; taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); @@ -63,6 +63,7 @@ int main() { ret = syncEnvStart(); assert(ret == 0); +/* SSyncNode* pSyncNode = doSync(); ret = syncNodeStartPingTimer(pSyncNode); @@ -72,6 +73,7 @@ int main() { ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); +*/ while (1) { taosMsleep(1000); From 1bb958732d7131695433fbc17a74960718d9840b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Mar 2022 10:48:07 +0800 Subject: [PATCH 25/82] [td-13039] refactor. --- source/libs/executor/inc/executorimpl.h | 7 ++----- source/libs/executor/src/executorimpl.c | 26 +++++++++++-------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 10ee4d157f..6f7055ad65 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -479,9 +479,6 @@ typedef struct SAggOperatorInfo { typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; - int32_t bufCapacity; - uint32_t seed; - SSDataBlock* existDataBlock; } SProjectOperatorInfo; @@ -615,8 +612,8 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 18cfe86553..de56a12a5d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7171,31 +7171,27 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray return pOperator; } -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - pInfo->seed = rand(); - pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; + int32_t numOfRows = 4096; + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); - SOptrBasicInfo* pBInfo = &pInfo->binfo; - pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); - pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - - initResultRowInfo(&pBInfo->resultRowInfo, 8); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); +// initResultRowInfo(&pBInfo->resultRowInfo, 8); +// setDefaultOutputBuf_rv(pBInfo, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "ProjectOperator"; -// pOperator->operatorType = OP_Project; + pOperator->operatorType = OP_Project; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->pExpr = exprArrayDup(pExprInfo); + pOperator->numOfOutput = taosArrayGetSize(pExprInfo); - pOperator->nextDataFn = doProjectOperation; - pOperator->closeFn = destroyProjectOperatorInfo; + pOperator->nextDataFn = doProjectOperation; + pOperator->closeFn = destroyProjectOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; From fd8fb4daa024500da5991c7afeba736ca1135e60 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 2 Mar 2022 13:52:09 +0800 Subject: [PATCH 26/82] update tests (#10496) --- tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests b/tests index 904e6f0e15..08ed39f0a5 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c +Subproject commit 08ed39f0a5fcbbfb5a630b945ab3d1998d4b4136 From 29d56b0a6455d9dc296a9cca5f97dc7864932568 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 14:00:56 +0800 Subject: [PATCH 27/82] init monitor module --- include/libs/monitor/monitor.h | 139 ++++++++++++++++++++++++ source/libs/CMakeLists.txt | 1 + source/libs/monitor/CMakeLists.txt | 13 +++ source/libs/monitor/inc/monInt.h | 25 +++++ source/libs/monitor/src/monitor.c | 18 +++ source/libs/monitor/test/CMakeLists.txt | 14 +++ source/libs/monitor/test/monTest.cpp | 33 ++++++ 7 files changed, 243 insertions(+) create mode 100644 include/libs/monitor/monitor.h create mode 100644 source/libs/monitor/CMakeLists.txt create mode 100644 source/libs/monitor/inc/monInt.h create mode 100644 source/libs/monitor/src/monitor.c create mode 100644 source/libs/monitor/test/CMakeLists.txt create mode 100644 source/libs/monitor/test/monTest.cpp diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h new file mode 100644 index 0000000000..e4c8137ea4 --- /dev/null +++ b/include/libs/monitor/monitor.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MONITOR_H_ +#define _TD_MONITOR_H_ + +#include "tarray.h" +#include "tdef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMonitor SMonitor; + +typedef struct { + int32_t dnode_id; + char dnode_ep[TSDB_EP_LEN]; +} SMonBasicInfo; + +typedef struct { + int32_t dnode_id; + char dnode_ep[TSDB_EP_LEN]; + char status[8]; +} SMonDnodeDesc; + +typedef struct { + int32_t mnode_id; + char mnode_ep[TSDB_EP_LEN]; + char role[8]; +} SMonMnodeDesc; + +typedef struct { + char first_ep[TSDB_EP_LEN]; + int32_t first_ep_dnode_id; + char version[12]; + float master_uptime; // day + int32_t monitor_interval; // sec + int32_t vgroups_total; + int32_t vgroups_alive; + int32_t vnodes_total; + int32_t vnodes_alive; + int32_t connections_total; + SArray *dnodes; // array of SMonDnodeDesc + SArray *mnodes; // array of SMonMnodeDesc +} SMonClusterInfo; + +typedef struct { + float uptime; // day + float cpu_engine; + float cpu_system; + float cpu_cores; + float mem_engine; // MB + float mem_system; // MB + float mem_total; // MB + float disk_engine; // GB + float disk_used; // GB + float disk_total; // GB + float net_in; // Kb/s + float net_out; // Kb/s + float io_read; // Mb/s + float io_write; // Mb/s + float io_read_disk; // Mb/s + float io_write_disk; // Mb/s + int32_t req_select; + float req_select_rate; + int32_t req_insert; + int32_t req_insert_success; + float req_insert_rate; + int32_t req_insert_batch; + int32_t req_insert_batch_success; + float req_insert_batch_rate; + int32_t errors; + int32_t vnodes_num; + int32_t masters; + int32_t has_mnode; +} SMonDnodeInfo; + +typedef struct { + char name[TSDB_FILENAME_LEN]; + int32_t level; + SDiskSize size; +} SMonDiskDesc; + +typedef struct { + SArray *disks; // array of SMonDiskDesc +} SMonDiskInfo; + +typedef struct { + int32_t dnode_id; + int8_t vnode_online; + char vnode_role[8]; +} SMonVnodeDesc; + +typedef struct { + int32_t vgroup_id; + SMonVnodeDesc vnodes[TSDB_MAX_REPLICA]; +} SMonVgroupDesc; + +typedef struct { + char database_name[TSDB_DB_NAME_LEN]; + int32_t tables_num; + int8_t status; + SArray *vgroups; // array of SMonVgroupDesc +} SMonVgroupInfo; + +typedef struct { + int64_t ts; + int8_t level; + char content[1024]; +} SMonLogItem; + +typedef struct { + SArray *logs; // array of SMonLogItem +} SMonLogInfo; + +typedef struct { + int32_t expire_time; + int32_t timeseries_used; + int32_t timeseries_total; +} SMonGrantInfo; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MONITOR_H_*/ diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index e07e46948f..78625d1eed 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -14,5 +14,6 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) +add_subdirectory(monitor) add_subdirectory(nodes) add_subdirectory(scalar) diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt new file mode 100644 index 0000000000..309d63691c --- /dev/null +++ b/source/libs/monitor/CMakeLists.txt @@ -0,0 +1,13 @@ +aux_source_directory(src MONITOR_SRC) +add_library(monitor STATIC ${MONITOR_SRC}) +target_include_directories( + monitor + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/monitor" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(monitor os util common) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h new file mode 100644 index 0000000000..5798f44a4d --- /dev/null +++ b/source/libs/monitor/inc/monInt.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MONITOR_INT_H_ +#define _TD_MONITOR_INT_H_ + +#include "monitor.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MONITOR_INT_H_*/ diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c new file mode 100644 index 0000000000..b372240bd5 --- /dev/null +++ b/source/libs/monitor/src/monitor.c @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "monInt.h" + diff --git a/source/libs/monitor/test/CMakeLists.txt b/source/libs/monitor/test/CMakeLists.txt new file mode 100644 index 0000000000..e3ab7e7337 --- /dev/null +++ b/source/libs/monitor/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. MONITOR_TEST_SRC) +add_executable(monitor_test ${MONITOR_TEST_SRC}) +target_link_libraries( + monitor_test + PUBLIC monitor + PUBLIC gtest_main +) + +add_test( + NAME monitor_test + COMMAND monitor_test +) diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp new file mode 100644 index 0000000000..a1805d0c9c --- /dev/null +++ b/source/libs/monitor/test/monTest.cpp @@ -0,0 +1,33 @@ +/** + * @file monTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief monitor module tests + * @version 1.0 + * @date 2022-03-05 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "os.h" + +#include "monitor.h" + +class MonitorTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { root = "/tmp/monTest"; } + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + + static const char *root; +}; + +const char *MonitorTest::root; + +TEST_F(MonitorTest, 01_Open_Close) { + +} From 10fd86209d97edf2254d0caec646a908d4e4d8e6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 14:10:27 +0800 Subject: [PATCH 28/82] minor changes --- include/os/osSysinfo.h | 1 - source/dnode/mgmt/impl/src/dndEnv.c | 21 +++------------------ source/dnode/vnode/inc/vnode.h | 4 ---- source/os/src/osSysinfo.c | 3 --- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 35ac032a8a..bf9d7662e6 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -42,7 +42,6 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetTotalSysMemoryKB(uint64_t *kb); bool taosGetProcMemory(float *memoryUsedMB); // bool taosGetSysMemory(float *memoryUsedMB); // -void taosGetDisk(); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); bool taosReadProcIO(int64_t *rchars, int64_t *wchars); bool taosGetProcIO(float *readKB, float *writeKB); diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 9467b56e6b..9f6c22067d 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -288,12 +288,9 @@ int32_t dndInit() { return -1; } - SVnodeOpt vnodeOpt = { - .sver = tsVersion, - .nthreads = tsNumOfCommitThreads, - .putReqToVQueryQFp = dndPutReqToVQueryQ, - .sendReqToDnodeFp = dndSendReqToDnode - }; + SVnodeOpt vnodeOpt = {.nthreads = tsNumOfCommitThreads, + .putReqToVQueryQFp = dndPutReqToVQueryQ, + .sendReqToDnodeFp = dndSendReqToDnode}; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode since %s", terrstr()); @@ -318,15 +315,3 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } - -// OTHER FUNCTIONS =================================== -void taosGetDisk() { -#if 0 - const double unit = 1024 * 1024 * 1024; - - SDiskSize diskSize = tfsGetSize(pTfs); - - tfsUpdateSize(&fsMeta); - -#endif -} \ No newline at end of file diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 31f04e840a..b5b53d9361 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -60,10 +60,6 @@ typedef struct { } SVnodeCfg; typedef struct { - int32_t sver; - const char *timezone; - const char *locale; - const char *charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; SendReqToDnodeFp sendReqToDnodeFp; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 163fad803f..3e761c6d91 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -167,11 +167,9 @@ void taosGetSystemInfo() { tsTotalMemoryMB = taosGetTotalMemory(); float tmp1, tmp2; - // taosGetDisk(); taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - } void taosKillSystem() { @@ -712,7 +710,6 @@ void taosGetSystemInfo() { float tmp1, tmp2; taosGetSysMemory(&tmp1); taosGetProcMemory(&tmp2); - // taosGetDisk(); taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); From 4cf8c0acf44095185f5b554d42339d64dd6e6687 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 14:44:04 +0800 Subject: [PATCH 29/82] monitor --- include/common/tglobal.h | 6 ++++++ source/common/src/tglobal.c | 19 ++++++++++++++++++- source/dnode/mgmt/impl/src/dndEnv.c | 9 ++++----- source/dnode/mgmt/impl/src/dndMgmt.c | 26 +++++++++++++++++++++++--- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e09f6d11bd..a0f83200c9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -52,6 +52,12 @@ extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerDay[3]; +// monitor +extern bool tsEnableMonitor; +extern int32_t tsMonitorInterval; +extern char tsMonitorFqdn[]; +extern uint16_t tsMonitorPort; + // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d66895888b..9905cf9833 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -46,6 +46,12 @@ int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = 1; bool tsPrintAuth = 0; +// monitor +bool tsEnableMonitor = 1; +int32_t tsMonitorInterval = 5; +char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; +uint16_t tsMonitorPort = 6043; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -314,6 +320,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; + if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; + return 0; } @@ -345,7 +357,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { } static void taosSetClientCfg(SConfig *pCfg) { - tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN); + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); @@ -425,6 +437,11 @@ static void taosSetServerCfg(SConfig *pCfg) { tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; + tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); + tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; + if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 9f6c22067d..da75b4524a 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -140,7 +140,7 @@ static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { return 0; } -static void dndCloseImp(SDnode *pDnode) { +static void dndCloseDir(SDnode *pDnode) { tfree(pDnode->dir.mnode); tfree(pDnode->dir.vnodes); tfree(pDnode->dir.dnode); @@ -260,7 +260,7 @@ void dndClose(SDnode *pDnode) { dndCleanupMgmt(pDnode); tfsClose(pDnode->pTfs); - dndCloseImp(pDnode); + dndCloseDir(pDnode); free(pDnode); dInfo("dnode object is closed, data:%p", pDnode); } @@ -288,9 +288,8 @@ int32_t dndInit() { return -1; } - SVnodeOpt vnodeOpt = {.nthreads = tsNumOfCommitThreads, - .putReqToVQueryQFp = dndPutReqToVQueryQ, - .sendReqToDnodeFp = dndSendReqToDnode}; + SVnodeOpt vnodeOpt = { + .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode}; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode since %s", terrstr()); diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 8f18222ab6..ac9f53a935 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -473,19 +473,39 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } +static void dndSendMonitorReport(SDnode *pDnode) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return; + + dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); +} + static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = tsStatusInterval * 1000; + int64_t lastStatusTime = taosGetTimestampMs(); + int64_t lastMonitorTime = lastStatusTime; setThreadName("dnode-hb"); while (true) { pthread_testcancel(); - taosMsleep(ms); + taosMsleep(200); + if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) { + continue; + } - if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent && !pMgmt->dropped) { + int64_t curTime = taosGetTimestampMs(); + + float statusInterval = (curTime - lastStatusTime) / 1000.0f; + if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { dndSendStatusReq(pDnode); + lastStatusTime = curTime; + } + + float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; + if (monitorInterval >= tsMonitorInterval) { + dndSendMonitorReport(pDnode); + lastMonitorTime = curTime; } } } From 4be79789ca9d044df45cd465b85fad63146b647e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 15:05:16 +0800 Subject: [PATCH 30/82] support json --- include/libs/index/index.h | 4 +-- source/libs/index/inc/indexInt.h | 2 ++ source/libs/index/src/index.c | 55 ++++++++++------------------- source/libs/index/src/index_cache.c | 46 +++++++++++++++++++----- source/libs/index/src/index_json.c | 12 +++---- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 9424eae1b7..453b49e4c6 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -125,10 +125,10 @@ int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index); /* * close index * @param index (input, index to be closed) - * @return error code + * @return void */ -int tIndexJsonClose(SIndexJson* index); +void tIndexJsonClose(SIndexJson* index); /* * insert terms into index diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 57d587d297..928e8b6102 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -119,6 +119,8 @@ typedef struct TFileCacheKey { int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); +// int32_t indexSerialKey(ICacheKey* key, char* buf); +// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); #define indexFatal(...) \ do { \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b518df2ea2..168e819073 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -65,8 +65,8 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); static void indexMergeSameKey(SArray* result, TFileValue* tv); -static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -int32_t indexSerialKey(ICacheKey* key, char* buf); +// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); +// int32_t indexSerialKey(ICacheKey* key, char* buf); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); @@ -152,8 +152,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - int32_t sz = indexSerialTermKey(p, buf); + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { @@ -166,9 +167,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - // ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialTermKey(p, buf); + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -334,8 +335,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result IndexCache* cache = NULL; char buf[128] = {0}; - ICacheKey key = {.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + ICacheKey key = { + .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); pthread_mutex_lock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); @@ -558,39 +560,18 @@ END: return -1; } -int32_t indexSerialTermKeyOfTag(SIndexTerm* p, char* buf) { - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - return indexSerialKey(&key, buf); -} -int32_t indexSerilaTermKeyOfJson(SIndexTerm* p, char* buf) { - ICacheKey key = {.suid = p->suid, .colName = JSON_COLUMN, .nColName = strlen(JSON_COLUMN)}; - return indexSerialKey(&key, buf); -} +int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); -int32_t indexSerialTermKey(SIndexTerm* itm, char* buf) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(itm->colType, TSDB_DATA_TYPE_JSON); - if (hasJson) { - return indexSerilaTermKeyOfJson(itm, buf); - } else { - return indexSerialTermKeyOfTag(itm, buf); - } -} -int32_t indexSerialKey(ICacheKey* key, char* buf) { char* p = buf; SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_VAR_TO_BUF(buf, '_', char); // SERIALIZE_MEM_TO_BUF(buf, key, colType); // SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + if (hasJson) { + SERIALIZE_STR_VAR_TO_BUF(buf, JSON_COLUMN, strlen(JSON_COLUMN)); + } else { + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + } return buf - p; } - -// int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { -// char* p = buf; -// SERIALIZE_MEM_TO_BUF(buf, key, suid); -// SERIALIZE_VAR_TO_BUF(buf, '_', char); -// // SERIALIZE_MEM_TO_BUF(buf, key, colType); -// // SERIALIZE_VAR_TO_BUF(buf, '_', char); -// SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -// return buf - p; -//} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index bb29a78cb1..a1f074feae 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -25,6 +25,7 @@ #define MEM_ESTIMATE_RADIO 1.5 static char JSON_COLUMN[] = "JSON"; +static char JSON_VALUE_DELIM = '&'; static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -46,6 +47,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in indexError("failed to create index cache"); return NULL; }; + cache->mem = indexInternalCacheCreate(type); cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; @@ -209,11 +211,38 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } } } +static char* indexCachePackJsonData(SIndexTerm* itm) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)calloc(1, sz); + char* p = buf; + + memcpy(p, itm->colVal, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, itm->colVal, itm->nColVal); + + return buf; +} int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); IndexCache* pCache = cache; indexCacheRef(pCache); @@ -224,8 +253,12 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { } // set up key ct->colType = term->colType; - ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); - memcpy(ct->colVal, term->colVal, term->nColVal); + if (hasJson) { + ct->colVal = indexCachePackJsonData(term); + } else { + ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); + memcpy(ct->colVal, term->colVal, term->nColVal); + } ct->version = atomic_add_fetch_32(&pCache->version, 1); // set value ct->uid = uid; @@ -369,6 +402,8 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { } static MemTable* indexInternalCacheCreate(int8_t type) { + type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type; + MemTable* tbl = calloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -391,9 +426,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); - // IterateValue* iv = &itera->val; - // IterateValue tIterVal = {.colVal = NULL, .val = taosArrayInit(1, sizeof(uint64_t))}; - bool next = tSkipListIterNext(iter); if (next) { SSkipListNode* node = tSkipListIterGet(iter); @@ -413,10 +445,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { taosArrayPush(iv->val, &ct->uid); } - // IterateValue* iv = &itera->val; - // iterateValueDestroy(iv, true); - //*iv = tIterVal; - return next; } diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c index 9ffab3fad1..de88ff3c8a 100644 --- a/source/libs/index/src/index_json.c +++ b/source/libs/index/src/index_json.c @@ -17,9 +17,8 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { // handle - return tIndexOpen(opts, path, index); + return indexOpen(opts, path, index); } -// k int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(terms); i++) { SIndexJsonTerm *p = taosArrayGetP(terms, i); @@ -29,16 +28,17 @@ int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { // handle put } -int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *query, SArray *result) { +int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) { + SArray *terms = tq->query; for (int i = 0; i < taosArrayGetSize(terms); i++) { SIndexJsonTerm *p = taosArrayGetP(terms, i); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } - return indexSearch(index, query, result); + return indexSearch(index, tq, result); // handle search } -int tIndexJsonClose(SIndexJson *index) { - return tIndexClose(index); +void tIndexJsonClose(SIndexJson *index) { + return indexClose(index); // handle close } From 5a4fbcf2621f54aec1fd4fd9db2429c3c32228bb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 15:11:54 +0800 Subject: [PATCH 31/82] sync io --- source/libs/sync/inc/syncIO.h | 38 +-- source/libs/sync/src/syncEnv.c | 1 - source/libs/sync/src/syncIO.c | 327 +++++++++++++---------- source/libs/sync/test/CMakeLists.txt | 28 ++ source/libs/sync/test/syncIOTickPing.cpp | 35 +++ source/libs/sync/test/syncIOTickQ.cpp | 35 +++ source/libs/sync/test/syncPingTest.cpp | 20 +- source/libs/sync/test/syncTest.cpp | 2 +- 8 files changed, 310 insertions(+), 176 deletions(-) create mode 100644 source/libs/sync/test/syncIOTickPing.cpp create mode 100644 source/libs/sync/test/syncIOTickQ.cpp diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 6723fa66c5..e5c55baff4 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -30,35 +30,37 @@ extern "C" { #include "trpc.h" typedef struct SSyncIO { - void * serverRpc; - void * clientRpc; STaosQueue *pMsgQ; - STaosQset * pQset; - pthread_t tid; - int8_t isStart; + STaosQset *pQset; + pthread_t consumerTid; - SEpSet epSet; + void *serverRpc; + void *clientRpc; + SEpSet myAddr; - void *syncTimer; - void *syncTimerManager; - - int32_t (*start)(struct SSyncIO *ths); - int32_t (*stop)(struct SSyncIO *ths); - int32_t (*ping)(struct SSyncIO *ths); - - int32_t (*onMsg)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); - int32_t (*destroy)(struct SSyncIO *ths); + void *ioTimerTickQ; + void *ioTimerTickPing; + void *ioTimerManager; void *pSyncNode; - int32_t (*FpOnPing)(struct SSyncNode *ths, SyncPing *pMsg); + int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); + int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); + int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); + int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); + int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); + int32_t (*FpOnSyncAppendEntriesReply)(SSyncNode *pSyncNode, SyncAppendEntriesReply *pMsg); + + int8_t isStart; } SSyncIO; extern SSyncIO *gSyncIO; -int32_t syncIOStart(); +int32_t syncIOStart(char *host, uint16_t port); int32_t syncIOStop(); -int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); +int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg); +int32_t syncIOTickQ(); +int32_t syncIOTickPing(); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index bfdf2a9c5e..a9cf035650 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -76,4 +76,3 @@ static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int m } static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer) {} -// -------------------------------- \ No newline at end of file diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 27434a4029..932c27d64a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -23,146 +23,66 @@ SSyncIO *gSyncIO = NULL; // local function ------------ -static int32_t doSyncIOStart(SSyncIO *io); -static int32_t doSyncIOStop(SSyncIO *io); -static int32_t doSyncIOPing(SSyncIO *io); -static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t doSyncIODestroy(SSyncIO *io); +static int32_t syncIOStartInternal(SSyncIO *io); +static int32_t syncIOStopInternal(SSyncIO *io); +static SSyncIO *syncIOCreate(char *host, uint16_t port); +static int32_t syncIODestroy(SSyncIO *io); -static SSyncIO *syncIOCreate(); -static void *syncIOConsumer(void *param); -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static void syncIODoReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncIODoRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncIOTick(void *param, void *tmrId); +static void *syncIOConsumerFunc(void *param); +static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); +static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); + +static int32_t syncIOTickQInternal(SSyncIO *io); +static void syncIOTickQFunc(void *param, void *tmrId); +static int32_t syncIOTickPingInternal(SSyncIO *io); +static void syncIOTickPingFunc(void *param, void *tmrId); // ---------------------------- -int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { +// public function ------------ +int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { sTrace("syncIOSendMsg ... "); pMsg->handle = NULL; - rpcSendRequest(handle, pEpSet, pMsg, NULL); + rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return 0; } -int32_t syncIOStart() { - gSyncIO = syncIOCreate(); +int32_t syncIOStart(char *host, uint16_t port) { + gSyncIO = syncIOCreate(host, port); assert(gSyncIO != NULL); - int32_t ret = doSyncIOStart(gSyncIO); + int32_t ret = syncIOStartInternal(gSyncIO); assert(ret == 0); return 0; } -int32_t syncIOStop() { return 0; } +int32_t syncIOStop() { + int32_t ret = syncIOStopInternal(gSyncIO); + assert(ret == 0); -// local function ------------ -static void syncIOTick(void *param, void *tmrId) { - SSyncIO *io = (SSyncIO *)param; - sDebug("syncIOTick ... "); - - SRpcMsg rpcMsg; - rpcMsg.pCont = rpcMallocCont(10); - snprintf(rpcMsg.pCont, 10, "TICK"); - rpcMsg.contLen = 10; - rpcMsg.handle = NULL; - rpcMsg.msgType = 2; - - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); - - taosWriteQitem(io->pMsgQ, pTemp); - - taosTmrReset(syncIOTick, 1000, io, io->syncTimerManager, &io->syncTimer); -} - -static void *syncIOConsumer(void *param) { - SSyncIO *io = param; - - STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; - int type; - - qall = taosAllocateQall(); - - while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); - sDebug("%d sync-io msgs are received", numOfMsgs); - if (numOfMsgs <= 0) break; - - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - sDebug("sync-io recv type:%d msg:%s", pRpcMsg->msgType, (char *)(pRpcMsg->pCont)); - } - - taosResetQitems(qall); - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - rpcFreeCont(pRpcMsg->pCont); - - if (pRpcMsg->handle != NULL) { - int msgSize = 128; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - rpcSendResponse(&rpcMsg); - } - - taosFreeQitem(pRpcMsg); - } - } - - taosFreeQall(qall); - return NULL; -} - -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; + ret = syncIODestroy(gSyncIO); + assert(ret == 0); return ret; } -static void syncIODoReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sDebug("syncIODoReply ... "); - rpcFreeCont(pMsg->pCont); +int32_t syncIOTickQ() { + int32_t ret = syncIOTickQInternal(gSyncIO); + assert(ret == 0); + return ret; } -static void syncIODoRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - SSyncIO *io = pParent; - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - - sDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(io->pMsgQ, pTemp); +int32_t syncIOTickPing() { + int32_t ret = syncIOTickPingInternal(gSyncIO); + assert(ret == 0); + return ret; } -static SSyncIO *syncIOCreate() { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->start = doSyncIOStart; - io->stop = doSyncIOStop; - io->ping = doSyncIOPing; - io->onMsg = doSyncIOOnMsg; - io->destroy = doSyncIODestroy; - - return io; -} - -static int32_t doSyncIOStart(SSyncIO *io) { +// local function ------------ +static int32_t syncIOStartInternal(SSyncIO *io) { taosBlockSIGPIPE(); + rpcInit(); tsRpcForceTcp = 1; // cient rpc init @@ -172,7 +92,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { rpcInit.localPort = 0; rpcInit.label = "SYNC-IO-CLIENT"; rpcInit.numOfThreads = 1; - rpcInit.cfp = syncIODoReply; + rpcInit.cfp = syncIOProcessReply; rpcInit.sessions = 100; rpcInit.idleTime = 100; rpcInit.user = "sync-io"; @@ -195,7 +115,7 @@ static int32_t doSyncIOStart(SSyncIO *io) { rpcInit.localPort = 7010; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; - rpcInit.cfp = syncIODoRequest; + rpcInit.cfp = syncIOProcessRequest; rpcInit.sessions = 1000; rpcInit.idleTime = 2 * 1500; rpcInit.afp = syncIOAuth; @@ -209,12 +129,9 @@ static int32_t doSyncIOStart(SSyncIO *io) { } } - io->epSet.inUse = 0; - addEpIntoEpSet(&io->epSet, "127.0.0.1", 7010); - // start consumer thread { - if (pthread_create(&io->tid, NULL, syncIOConsumer, io) != 0) { + if (pthread_create(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -222,35 +139,32 @@ static int32_t doSyncIOStart(SSyncIO *io) { } // start tmr thread - io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - io->syncTimer = taosTmrStart(syncIOTick, 1000, io, io->syncTimerManager); + io->ioTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); return 0; } -static int32_t doSyncIOStop(SSyncIO *io) { +static int32_t syncIOStopInternal(SSyncIO *io) { atomic_store_8(&io->isStart, 0); - pthread_join(io->tid, NULL); + pthread_join(io->consumerTid, NULL); return 0; } -static int32_t doSyncIOPing(SSyncIO *io) { - SRpcMsg rpcMsg, rspMsg; +static SSyncIO *syncIOCreate(char *host, uint16_t port) { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); - rpcMsg.pCont = rpcMallocCont(10); - snprintf(rpcMsg.pCont, 10, "ping"); - rpcMsg.contLen = 10; - rpcMsg.handle = NULL; - rpcMsg.msgType = 1; + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); + io->myAddr.inUse = 0; + addEpIntoEpSet(&io->myAddr, host, port); - return 0; + return io; } -static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } - -static int32_t doSyncIODestroy(SSyncIO *io) { +static int32_t syncIODestroy(SSyncIO *io) { int8_t start = atomic_load_8(&io->isStart); assert(start == 0); @@ -264,15 +178,136 @@ static int32_t doSyncIODestroy(SSyncIO *io) { io->clientRpc = NULL; } - if (io->pMsgQ != NULL) { - free(io->pMsgQ); - io->pMsgQ = NULL; - } - - if (io->pQset != NULL) { - free(io->pQset); - io->pQset = NULL; - } + taosCloseQueue(io->pMsgQ); + taosCloseQset(io->pQset); return 0; } + +static void *syncIOConsumerFunc(void *param) { + SSyncIO *io = param; + + STaosQall *qall; + SRpcMsg *pRpcMsg, rpcMsg; + int type; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); + sTrace("syncIOConsumerFunc %d msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen, + (char *)(pRpcMsg->pCont)); + + if (pRpcMsg->msgType == SYNC_PING) { + if (io->FpOnSyncPing != NULL) { + SyncPing *pSyncMsg = syncPingBuild(pRpcMsg->contLen); + syncPingFromRpcMsg(pRpcMsg, pSyncMsg); + io->FpOnSyncPing(io->pSyncNode, pSyncMsg); + } + } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); + syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); + io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); + } else { + ; + } + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + if (pRpcMsg->handle != NULL) { + int msgSize = 128; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + + sTrace("syncIOConsumerFunc rpcSendResponse ... msgType:%d contLen:%d", pRpcMsg->msgType, rpcMsg.contLen); + rpcSendResponse(&rpcMsg); + } + + taosFreeQitem(pRpcMsg); + } + } + + taosFreeQall(qall); + return NULL; +} + +static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + return ret; +} + +static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + sTrace("syncIOProcessRequest: type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont); + + SSyncIO *io = pParent; + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + taosWriteQitem(io->pMsgQ, pTemp); +} + +static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + sTrace("syncIOProcessReply: type:%d, contLen:%d msg:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont); + rpcFreeCont(pMsg->pCont); +} + +static int32_t syncIOTickQInternal(SSyncIO *io) { + io->ioTimerTickQ = taosTmrStart(syncIOTickQFunc, 1000, io, io->ioTimerManager); + return 0; +} + +static void syncIOTickQFunc(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + sTrace("<-- syncIOTickQFunc -->"); + + SRpcMsg rpcMsg; + rpcMsg.contLen = 64; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickQ"); + rpcMsg.handle = NULL; + rpcMsg.msgType = 55; + + SRpcMsg *pTemp; + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); + + taosWriteQitem(io->pMsgQ, pTemp); + taosTmrReset(syncIOTickQFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickQ); +} + +static int32_t syncIOTickPingInternal(SSyncIO *io) { + io->ioTimerTickPing = taosTmrStart(syncIOTickPingFunc, 1000, io, io->ioTimerManager); + return 0; +} + +static void syncIOTickPingFunc(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + sTrace("<-- syncIOTickPingFunc -->"); + + SRpcMsg rpcMsg; + rpcMsg.contLen = 64; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickPing"); + rpcMsg.handle = NULL; + rpcMsg.msgType = 77; + + rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); + taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); +} diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 8e1c99d179..9abe3f232e 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -2,6 +2,8 @@ add_executable(syncTest "") add_executable(syncEnvTest "") add_executable(syncPingTest "") add_executable(syncEncodeTest "") +add_executable(syncIOTickQ "") +add_executable(syncIOTickPing "") target_sources(syncTest @@ -20,6 +22,14 @@ target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" ) +target_sources(syncIOTickQ + PRIVATE + "syncIOTickQ.cpp" +) +target_sources(syncIOTickPing + PRIVATE + "syncIOTickPing.cpp" +) target_include_directories(syncTest @@ -42,6 +52,16 @@ target_include_directories(syncEncodeTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncIOTickQ + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncIOTickPing + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -60,6 +80,14 @@ target_link_libraries(syncEncodeTest sync gtest_main ) +target_link_libraries(syncIOTickQ + sync + gtest_main +) +target_link_libraries(syncIOTickPing + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncIOTickPing.cpp b/source/libs/sync/test/syncIOTickPing.cpp new file mode 100644 index 0000000000..1559b57585 --- /dev/null +++ b/source/libs/sync/test/syncIOTickPing.cpp @@ -0,0 +1,35 @@ +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret; + + ret = syncIOStart((char*)"127.0.0.1", 7010); + assert(ret == 0); + + ret = syncIOTickPing(); + assert(ret == 0); + + while (1) { + sleep(1); + } + return 0; +} diff --git a/source/libs/sync/test/syncIOTickQ.cpp b/source/libs/sync/test/syncIOTickQ.cpp new file mode 100644 index 0000000000..90304079e3 --- /dev/null +++ b/source/libs/sync/test/syncIOTickQ.cpp @@ -0,0 +1,35 @@ +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret; + + ret = syncIOStart((char*)"127.0.0.1", 7010); + assert(ret == 0); + + ret = syncIOTickQ(); + assert(ret == 0); + + while (1) { + sleep(1); + } + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index b57f6c5259..bca634765e 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -39,7 +39,7 @@ SSyncNode* doSync() { SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); - gSyncIO->FpOnPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; @@ -57,23 +57,23 @@ int main() { logTest(); - int32_t ret = syncIOStart(); + int32_t ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); -/* - SSyncNode* pSyncNode = doSync(); + /* + SSyncNode* pSyncNode = doSync(); - ret = syncNodeStartPingTimer(pSyncNode); - assert(ret == 0); + ret = syncNodeStartPingTimer(pSyncNode); + assert(ret == 0); - taosMsleep(5000); + taosMsleep(5000); - ret = syncNodeStopPingTimer(pSyncNode); - assert(ret == 0); -*/ + ret = syncNodeStopPingTimer(pSyncNode); + assert(ret == 0); + */ while (1) { taosMsleep(1000); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 0843a48bb8..0b397ef921 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -8,7 +8,7 @@ void *pingFunc(void *param) { SSyncIO *io = (SSyncIO *)param; while (1) { sDebug("io->ping"); - io->ping(io); + // io->ping(io); sleep(1); } return NULL; From 3e591a022ce0775565642a855f7fcaf8874b1d53 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 15:20:49 +0800 Subject: [PATCH 32/82] sync io --- source/libs/sync/inc/syncIO.h | 6 +++--- source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/test/CMakeLists.txt | 20 +++++++++---------- ...cIOTickPing.cpp => syncIOTickPingTest.cpp} | 0 .../{syncIOTickQ.cpp => syncIOTickQTest.cpp} | 0 source/libs/sync/test/syncPingTest.cpp | 14 ++++++------- 6 files changed, 20 insertions(+), 22 deletions(-) rename source/libs/sync/test/{syncIOTickPing.cpp => syncIOTickPingTest.cpp} (100%) rename source/libs/sync/test/{syncIOTickQ.cpp => syncIOTickQTest.cpp} (100%) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index e5c55baff4..238948b403 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -31,11 +31,11 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; - STaosQset *pQset; + STaosQset * pQset; pthread_t consumerTid; - void *serverRpc; - void *clientRpc; + void * serverRpc; + void * clientRpc; SEpSet myAddr; void *ioTimerTickQ; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 932c27d64a..bb71b1e73d 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -188,7 +188,7 @@ static void *syncIOConsumerFunc(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 9abe3f232e..c089987c19 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -2,8 +2,8 @@ add_executable(syncTest "") add_executable(syncEnvTest "") add_executable(syncPingTest "") add_executable(syncEncodeTest "") -add_executable(syncIOTickQ "") -add_executable(syncIOTickPing "") +add_executable(syncIOTickQTest "") +add_executable(syncIOTickPingTest "") target_sources(syncTest @@ -22,13 +22,13 @@ target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" ) -target_sources(syncIOTickQ +target_sources(syncIOTickQTest PRIVATE - "syncIOTickQ.cpp" + "syncIOTickQTest.cpp" ) -target_sources(syncIOTickPing +target_sources(syncIOTickPingTest PRIVATE - "syncIOTickPing.cpp" + "syncIOTickPingTest.cpp" ) @@ -52,12 +52,12 @@ target_include_directories(syncEncodeTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOTickQ +target_include_directories(syncIOTickQTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOTickPing +target_include_directories(syncIOTickPingTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" @@ -80,11 +80,11 @@ target_link_libraries(syncEncodeTest sync gtest_main ) -target_link_libraries(syncIOTickQ +target_link_libraries(syncIOTickQTest sync gtest_main ) -target_link_libraries(syncIOTickPing +target_link_libraries(syncIOTickPingTest sync gtest_main ) diff --git a/source/libs/sync/test/syncIOTickPing.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp similarity index 100% rename from source/libs/sync/test/syncIOTickPing.cpp rename to source/libs/sync/test/syncIOTickPingTest.cpp diff --git a/source/libs/sync/test/syncIOTickQ.cpp b/source/libs/sync/test/syncIOTickQTest.cpp similarity index 100% rename from source/libs/sync/test/syncIOTickQ.cpp rename to source/libs/sync/test/syncIOTickQTest.cpp diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index bca634765e..b80b0f2e20 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -63,17 +63,15 @@ int main() { ret = syncEnvStart(); assert(ret == 0); - /* - SSyncNode* pSyncNode = doSync(); + SSyncNode* pSyncNode = doSync(); - ret = syncNodeStartPingTimer(pSyncNode); - assert(ret == 0); + ret = syncNodeStartPingTimer(pSyncNode); + assert(ret == 0); - taosMsleep(5000); + taosMsleep(5000); - ret = syncNodeStopPingTimer(pSyncNode); - assert(ret == 0); - */ + ret = syncNodeStopPingTimer(pSyncNode); + assert(ret == 0); while (1) { taosMsleep(1000); From 149cf9422315cd25ebfffef1b31888ddc97dee2e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 2 Mar 2022 15:28:40 +0800 Subject: [PATCH 33/82] [TD-13756]: file system write fsync error --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 136afb9a15..4b10a179a9 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -326,7 +326,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { nleft -= nwritten; tbuf += nwritten; } - fsync(pFile->fd); + return count; } From 4f4100f9c6d602a06554fb0fa967f7a0d5a68137 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 15:41:19 +0800 Subject: [PATCH 34/82] monitor --- include/common/tglobal.h | 1 + include/libs/monitor/monitor.h | 28 ++++++++-- source/common/src/tglobal.c | 3 + source/dnode/mgmt/impl/CMakeLists.txt | 1 + source/dnode/mgmt/impl/src/dndEnv.c | 9 +++ source/dnode/mgmt/impl/src/dndMgmt.c | 10 ++++ source/libs/monitor/inc/monInt.h | 17 ++++++ source/libs/monitor/src/monitor.c | 79 +++++++++++++++++++++++++++ source/util/src/tjson.c | 6 +- 9 files changed, 147 insertions(+), 7 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a0f83200c9..2261170e63 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -57,6 +57,7 @@ extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; extern char tsMonitorFqdn[]; extern uint16_t tsMonitorPort; +extern int32_t tsMonitorMaxLogs; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index e4c8137ea4..747c46c0c6 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -23,8 +23,6 @@ extern "C" { #endif -typedef struct SMonitor SMonitor; - typedef struct { int32_t dnode_id; char dnode_ep[TSDB_EP_LEN]; @@ -122,16 +120,34 @@ typedef struct { char content[1024]; } SMonLogItem; -typedef struct { - SArray *logs; // array of SMonLogItem -} SMonLogInfo; - typedef struct { int32_t expire_time; int32_t timeseries_used; int32_t timeseries_total; } SMonGrantInfo; +typedef struct SMonInfo SMonInfo; + +typedef struct { + const char *server; + uint16_t port; + int32_t maxLogs; +} SMonCfg; + +int32_t monInit(const SMonCfg *pCfg); +void monCleanup(); +void monAddLogItem(SMonLogItem *pItem); + +SMonInfo *monCreateMonitorInfo(); +void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); +void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); +void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); +void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); +void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); +void monSetGrantInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); +void monSendReport(SMonInfo *pMonitor); +void monCleanupMonitorInfo(SMonInfo *pMonitor); + #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9905cf9833..5998b7c9ce 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -51,6 +51,7 @@ bool tsEnableMonitor = 1; int32_t tsMonitorInterval = 5; char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsMonitorPort = 6043; +int32_t tsMonitorMaxLogs = 100; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -325,6 +326,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; return 0; } @@ -441,6 +443,7 @@ static void taosSetServerCfg(SConfig *pCfg) { tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; + tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt index 171f9649fb..ec9e1be02d 100644 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ b/source/dnode/mgmt/impl/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC sync PUBLIC taos PUBLIC tfs + PUBLIC monitor ) target_include_directories( dnode diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index da75b4524a..2539443b41 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -21,6 +21,7 @@ #include "dndSnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "monitor.h" #include "sync.h" #include "tfs.h" #include "wal.h" @@ -297,6 +298,13 @@ int32_t dndInit() { return -1; } + SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn}; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + dndCleanup(); + return -1; + } + dInfo("dnode env is initialized"); return 0; } @@ -310,6 +318,7 @@ void dndCleanup() { walCleanUp(); vnodeCleanup(); rpcCleanup(); + monCleanup(); taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index ac9f53a935..171bdc792c 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -22,6 +22,7 @@ #include "dndTransport.h" #include "dndVnodes.h" #include "dndWorker.h" +#include "monitor.h" static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); @@ -475,8 +476,17 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return; + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); + + SMonBasicInfo basicInfo = {.dnode_id = dndGetDnodeId(pDnode)}; + tstrncpy(basicInfo.dnode_ep, tsLocalEp, TSDB_EP_LEN); + monSetBasicInfo(pMonitor, &basicInfo); + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); } static void *dnodeThreadRoutine(void *param) { diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 5798f44a4d..61f9980e4e 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -18,6 +18,23 @@ #include "monitor.h" +#include "tarray.h" +#include "tlockfree.h" +#include "tjson.h" + +typedef struct { + SRWLatch lock; + SArray *logs; // array of SMonLogItem + int32_t maxLogs; + const char *server; + uint16_t port; +} SMonitor; + +typedef struct SMonInfo { + SArray *logs; // array of SMonLogItem + SJson *pJson; +} SMonInfo; + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index b372240bd5..0bb1775135 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -15,4 +15,83 @@ #define _DEFAULT_SOURCE #include "monInt.h" +#include "taoserror.h" +#include "thttp.h" +#include "tlog.h" +static SMonitor tsMonitor = {0}; + +int32_t monInit(const SMonCfg *pCfg) { + tsMonitor.logs = taosArrayInit(16, sizeof(SMonInfo)); + if (tsMonitor.logs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tsMonitor.maxLogs = pCfg->maxLogs; + tsMonitor.server = pCfg->server; + tsMonitor.port = pCfg->port; + taosInitRWLatch(&tsMonitor.lock); + return 0; +} + +void monCleanup() { + taosArrayDestroy(tsMonitor.logs); + tsMonitor.logs = NULL; +} + +void monAddLogItem(SMonLogItem *pItem) { + taosWLockLatch(&tsMonitor.lock); + int32_t size = taosArrayGetSize(tsMonitor.logs); + if (size > tsMonitor.maxLogs) { + uInfo("too many logs for monitor"); + } else { + taosArrayPush(tsMonitor.logs, pItem); + } + taosWUnLockLatch(&tsMonitor.lock); +} + +SMonInfo *monCreateMonitorInfo() { + SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo)); + if (pMonitor == NULL) return NULL; + + taosWLockLatch(&tsMonitor.lock); + pMonitor->logs = taosArrayDup(pMonitor->logs); + taosArrayClear(tsMonitor.logs); + taosWUnLockLatch(&tsMonitor.lock); + + pMonitor->pJson = tjsonCreateObject(); + if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + monCleanupMonitorInfo(pMonitor); + return NULL; + } + + return pMonitor; +} + +void monCleanupMonitorInfo(SMonInfo *pMonitor) { + taosArrayDestroy(pMonitor->logs); + tjsonDelete(pMonitor->pJson); + free(pMonitor); +} + +void monSendReport(SMonInfo *pMonitor) { + char *pCont = tjsonToString(pMonitor->pJson); + if (pCont != NULL) { + taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont)); + free(pCont); + } +} + +void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { + SJson *pJson = pMonitor->pJson; + tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); + tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); +} + +void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); +void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); +void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); +void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); +void monSetGrantInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 2c117771b1..27848c4e23 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -26,7 +26,11 @@ SJson* tjsonCreateObject() { return pJson; } -void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); } +void tjsonDelete(SJson* pJson) { + if (pJson != NULL) { + cJSON_Delete((cJSON*)pJson); + } +} int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) { char tmp[40] = {0}; From a2d9cd4403c484b234ddea0ea3fcdd5d09ddbf19 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 16:19:15 +0800 Subject: [PATCH 35/82] sync io --- source/libs/sync/src/syncIO.c | 4 ++-- source/libs/sync/src/syncUtil.c | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index bb71b1e73d..96bb305e76 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -188,7 +188,7 @@ static void *syncIOConsumerFunc(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); @@ -310,4 +310,4 @@ static void syncIOTickPingFunc(void *param, void *tmrId) { rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); -} +} \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 0066ab783e..9c134e2d08 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -46,12 +46,25 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { uint16_t port; syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); + + /* + pEpSet->numOfEps = 1; + pEpSet->inUse = 0; + pEpSet->eps[0].port = port; + snprintf(epSet.eps[0].fqdn, sizeof(epSet.eps[0].fqdn), host); + */ + pEpSet->inUse = 0; + pEpSet->numOfEps = 1; addEpIntoEpSet(pEpSet, host, port); } void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { - raftId->addr = syncUtilAddr2U64(pNodeInfo->nodeFqdn, pNodeInfo->nodePort); + uint32_t ipv4 = taosGetIpv4FromFqdn(pNodeInfo->nodeFqdn); + assert(ipv4 != 0xFFFFFFFF); + char ipbuf[128]; + tinet_ntoa(ipbuf, ipv4); + raftId->addr = syncUtilAddr2U64(ipbuf, pNodeInfo->nodePort); raftId->vgId = vgId; } From 398d5ec3a7219feed9557d3efcc1108eb35af1bd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 16:37:04 +0800 Subject: [PATCH 36/82] sync io --- source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/test/CMakeLists.txt | 42 ++++++++++++++++ .../sync/test/syncIOSendMsgClientTest.cpp | 50 +++++++++++++++++++ .../sync/test/syncIOSendMsgServerTest.cpp | 33 ++++++++++++ source/libs/sync/test/syncIOSendMsgTest.cpp | 50 +++++++++++++++++++ source/libs/sync/test/syncPingTest.cpp | 5 +- 6 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 source/libs/sync/test/syncIOSendMsgClientTest.cpp create mode 100644 source/libs/sync/test/syncIOSendMsgServerTest.cpp create mode 100644 source/libs/sync/test/syncIOSendMsgTest.cpp diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 96bb305e76..88fba6429d 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -112,7 +112,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7010; + rpcInit.localPort = io->myAddr.eps[0].port; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; rpcInit.cfp = syncIOProcessRequest; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index c089987c19..f28bbf46cc 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -4,6 +4,9 @@ add_executable(syncPingTest "") add_executable(syncEncodeTest "") add_executable(syncIOTickQTest "") add_executable(syncIOTickPingTest "") +add_executable(syncIOSendMsgTest "") +add_executable(syncIOSendMsgClientTest "") +add_executable(syncIOSendMsgServerTest "") target_sources(syncTest @@ -30,6 +33,18 @@ target_sources(syncIOTickPingTest PRIVATE "syncIOTickPingTest.cpp" ) +target_sources(syncIOSendMsgTest + PRIVATE + "syncIOSendMsgTest.cpp" +) +target_sources(syncIOSendMsgClientTest + PRIVATE + "syncIOSendMsgClientTest.cpp" +) +target_sources(syncIOSendMsgServerTest + PRIVATE + "syncIOSendMsgServerTest.cpp" +) target_include_directories(syncTest @@ -62,6 +77,21 @@ target_include_directories(syncIOTickPingTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncIOSendMsgTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncIOSendMsgClientTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncIOSendMsgServerTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -88,6 +118,18 @@ target_link_libraries(syncIOTickPingTest sync gtest_main ) +target_link_libraries(syncIOSendMsgTest + sync + gtest_main +) +target_link_libraries(syncIOSendMsgClientTest + sync + gtest_main +) +target_link_libraries(syncIOSendMsgServerTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp new file mode 100644 index 0000000000..09dc9bbb42 --- /dev/null +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -0,0 +1,50 @@ +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret; + + ret = syncIOStart((char *)"127.0.0.1", 7010); + assert(ret == 0); + + for (int i = 0; i < 10; ++i) { + SEpSet epSet; + epSet.inUse = 0; + epSet.numOfEps = 1; + addEpIntoEpSet(&epSet, "127.0.0.1", 7030); + + SRpcMsg rpcMsg; + rpcMsg.contLen = 64; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); + rpcMsg.handle = NULL; + rpcMsg.msgType = 77; + + syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); + sleep(1); + } + + while (1) { + sleep(1); + } + + return 0; +} diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOSendMsgServerTest.cpp new file mode 100644 index 0000000000..8af9344ed6 --- /dev/null +++ b/source/libs/sync/test/syncIOSendMsgServerTest.cpp @@ -0,0 +1,33 @@ +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret; + + ret = syncIOStart((char *)"127.0.0.1", 7030); + assert(ret == 0); + + while (1) { + sleep(1); + } + + return 0; +} diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp new file mode 100644 index 0000000000..70fbe55b67 --- /dev/null +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -0,0 +1,50 @@ +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret; + + ret = syncIOStart((char *)"127.0.0.1", 7010); + assert(ret == 0); + + for (int i = 0; i < 10; ++i) { + SEpSet epSet; + epSet.inUse = 0; + epSet.numOfEps = 1; + addEpIntoEpSet(&epSet, "127.0.0.1", 7010); + + SRpcMsg rpcMsg; + rpcMsg.contLen = 64; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); + rpcMsg.handle = NULL; + rpcMsg.msgType = 77; + + syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); + sleep(1); + } + + while (1) { + sleep(1); + } + + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index b80b0f2e20..01e5731dde 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -68,10 +68,11 @@ int main() { ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); - taosMsleep(5000); - + /* + taosMsleep(10000); ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); + */ while (1) { taosMsleep(1000); From 8b1a991186a9f8afd6a345dd4be8bd1d8c408c91 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 16:45:37 +0800 Subject: [PATCH 37/82] send the basic information of the monitor --- include/common/ttime.h | 2 ++ source/common/src/ttime.c | 47 +++++++++++++++++++++++++++++++ source/libs/monitor/src/monitor.c | 8 +++++- source/util/src/thttp.c | 10 +++---- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index a4b2aa6673..b71426e312 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -52,6 +52,8 @@ void deltaToUtcInitOnce(); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); +void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); + #ifdef __cplusplus } #endif diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 460c4a6fc0..7a318c2eb8 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -627,3 +627,50 @@ const char* fmtts(int64_t ts) { return buf; } + +void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { + char ts[40] = {0}; + struct tm* ptm; + + int32_t fractionLen; + char* format = NULL; + time_t quot = 0; + long mod = 0; + + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: { + quot = t / 1000; + fractionLen = 5; + format = ".%03" PRId64; + mod = t % 1000; + break; + } + + case TSDB_TIME_PRECISION_MICRO: { + quot = t / 1000000; + fractionLen = 8; + format = ".%06" PRId64; + mod = t % 1000000; + break; + } + + case TSDB_TIME_PRECISION_NANO: { + quot = t / 1000000000; + fractionLen = 11; + format = ".%09" PRId64; + mod = t % 1000000000; + break; + } + + default: + fractionLen = 0; + assert(false); + } + + ptm = localtime("); + int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); + length += snprintf(ts + length, fractionLen, format, mod); + length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); + + tstrncpy(buf, ts, bufLen); +} \ No newline at end of file diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 0bb1775135..f8a9039f48 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "thttp.h" #include "tlog.h" +#include "ttime.h" static SMonitor tsMonitor = {0}; @@ -56,7 +57,7 @@ SMonInfo *monCreateMonitorInfo() { if (pMonitor == NULL) return NULL; taosWLockLatch(&tsMonitor.lock); - pMonitor->logs = taosArrayDup(pMonitor->logs); + pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); taosWUnLockLatch(&tsMonitor.lock); @@ -88,6 +89,11 @@ void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { SJson *pJson = pMonitor->pJson; tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + + int64_t ms = taosGetTimestampMs(); + char buf[40] = {0}; + taosFormatUtcTime(buf, sizeof(buf), ms, TSDB_TIME_PRECISION_MILLI); + tjsonAddStringToObject(pJson, "ts", buf); } void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c index 14c39d3f03..aa4c795744 100644 --- a/source/util/src/thttp.c +++ b/source/util/src/thttp.c @@ -32,7 +32,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, fd = taosOpenTcpClientSocket(ip, port, 0); if (fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create http socket since %s", terrstr()); + uError("failed to create socket to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } @@ -46,24 +46,24 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, if (taosWriteSocket(fd, (void*)header, headLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to send http header since %s", terrstr()); + uError("failed to send http header to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to send http content since %s", terrstr()); + uError("failed to send http content to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } // read something to avoid nginx error 499 if (taosReadSocket(fd, header, 10) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to receive response since %s", terrstr()); + uError("failed to receive response from %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } - uInfo("send http to %s:%d, len:%d content: %s", server, port, contLen, pCont); + uInfo("send http to %s:%u, len:%d content: %s", server, port, contLen, pCont); code = 0; SEND_OVER: From 6a6f31c4a75dd30d916b7b31512f7620b6af1d02 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 17:09:22 +0800 Subject: [PATCH 38/82] support json --- source/libs/index/src/index_cache.c | 13 +++- source/libs/index/test/CMakeLists.txt | 18 ++++++ source/libs/index/test/jsonUT.cc | 92 +++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 source/libs/index/test/jsonUT.cc diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index a1f074feae..2742830d3b 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -222,7 +222,7 @@ static char* indexCachePackJsonData(SIndexTerm* itm) { char* buf = (char*)calloc(1, sz); char* p = buf; - memcpy(p, itm->colVal, itm->nColName); + memcpy(p, itm->colName, itm->nColName); p += itm->nColName; memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); @@ -329,13 +329,22 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + char* p = term->colVal; + if (hasJson) { + p = indexCachePackJsonData(term); + } + CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; int ret = indexQueryMem(mem, &ct, qtype, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm ret = indexQueryMem(imm, &ct, qtype, result, s); } + if (hasJson) { + tfree(p); + } indexMemUnRef(mem); indexMemUnRef(imm); diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index bed42be3e5..1ebf85368b 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") add_executable(UtilUT "") +add_executable(jsonUT "") target_sources(indexTest PRIVATE @@ -21,6 +22,10 @@ target_sources(UtilUT "utilUT.cc" ) +target_sources(jsonUT + PRIVATE + "jsonUT.cc" +) target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -43,6 +48,12 @@ target_include_directories ( UtilUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories (jsonUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -73,6 +84,13 @@ target_link_libraries (UtilUT index ) +target_link_libraries (jsonUT + os + util + common + gtest_main + index +) #add_test( # NAME index_test diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc new file mode 100644 index 0000000000..20b59add9f --- /dev/null +++ b/source/libs/index/test/jsonUT.cc @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +static std::string dir = "/tmp/json"; +class JsonEnv : public ::testing::Test { + protected: + virtual void SetUp() { + taosRemoveDir(dir.c_str()); + opts = indexOptsCreate(); + int ret = tIndexJsonOpen(opts, dir.c_str(), &index); + assert(ret == 0); + } + virtual void TearDown() { + tIndexJsonClose(index); + indexOptsDestroy(opts); + } + SIndexJsonOpts* opts; + SIndexJson* index; +}; + +TEST_F(JsonEnv, testWrite) { + { + std::string colName("voltage"); + std::string colVal("ab"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab1"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("123"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab"); + + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SArray* result = taosArrayInit(1, sizeof(uint64_t)); + indexMultiTermQueryAdd(mq, q, QUERY_TERM); + tIndexJsonSearch(index, mq, result); + assert(100 == taosArrayGetSize(result)); + indexMultiTermQueryDestroy(mq); + } + + // SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; +} From 8057e44d1702af275db0cf30d085692c766b7fa5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 17:40:22 +0800 Subject: [PATCH 39/82] sync modify timer --- source/libs/sync/src/syncIO.c | 4 +++- source/libs/sync/src/syncMain.c | 14 +++++++++----- source/libs/sync/src/syncMessage.c | 4 ++-- source/libs/sync/src/syncUtil.c | 5 ++--- source/libs/sync/test/syncIOSendMsgClientTest.cpp | 2 +- source/libs/sync/test/syncIOSendMsgTest.cpp | 2 +- source/libs/sync/test/syncPingTest.cpp | 9 ++++++--- 7 files changed, 24 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 88fba6429d..74f848338f 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -41,7 +41,8 @@ static void syncIOTickPingFunc(void *param, void *tmrId); // public function ------------ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { - sTrace("syncIOSendMsg ... "); + sTrace("<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u", clientRpc, pEpSet->numOfEps, + pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port); pMsg->handle = NULL; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return 0; @@ -54,6 +55,7 @@ int32_t syncIOStart(char *host, uint16_t port) { int32_t ret = syncIOStartInternal(gSyncIO); assert(ret == 0); + sTrace("syncIOStart ok, gSyncIO:%p gSyncIO->clientRpc:%p", gSyncIO, gSyncIO->clientRpc); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9e335c5e55..d19c51900f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -184,7 +184,7 @@ static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, Syn SyncPing* pMsg2 = rpcMsg.pCont; cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); - sTrace("syncNodePing pMsg2:%s ", serialized); + sTrace("syncNodePing rpcMsg.pCont:%s ", serialized); free(serialized); cJSON_Delete(pJson); } @@ -253,12 +253,16 @@ static void syncNodePingTimerCb(void* param, void* tmrId) { ++(pSyncNode->pingTimerCounter); // pSyncNode->pingTimerMS += 100; - sTrace("pSyncNode->pingTimerCounter:%lu, pSyncNode->pingTimerMS:%d, pSyncNode->pPingTimer:%p, tmrId:%p ", - pSyncNode->pingTimerCounter, pSyncNode->pingTimerMS, pSyncNode->pPingTimer, tmrId); + sTrace( + "syncNodePingTimerCb: pSyncNode->pingTimerCounter:%lu, pSyncNode->pingTimerMS:%d, pSyncNode->pPingTimer:%p, " + "tmrId:%p ", + pSyncNode->pingTimerCounter, pSyncNode->pingTimerMS, pSyncNode->pPingTimer, tmrId); + + syncNodePingAll(pSyncNode); taosTmrReset(syncNodePingTimerCb, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); - - syncNodePingAll(pSyncNode); + } else { + sTrace("syncNodePingTimerCb: pingTimerStart:%u ", pSyncNode->pingTimerStart); } } \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index e0975fb3c1..8e7dc8ba61 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -70,7 +70,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON* pDestId = cJSON_CreateObject(); cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pDestId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", pMsg->data); @@ -145,7 +145,7 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON* pDestId = cJSON_CreateObject(); cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pDestId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", pMsg->data); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9c134e2d08..a3fbfaf905 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -51,11 +51,10 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { pEpSet->numOfEps = 1; pEpSet->inUse = 0; pEpSet->eps[0].port = port; - snprintf(epSet.eps[0].fqdn, sizeof(epSet.eps[0].fqdn), host); + snprintf(pEpSet->eps[0].fqdn, sizeof(pEpSet->eps[0].fqdn), "%s", host); */ - pEpSet->inUse = 0; - pEpSet->numOfEps = 1; + pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, host, port); } diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 09dc9bbb42..8adaff1a2f 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -34,7 +34,7 @@ int main() { SRpcMsg rpcMsg; rpcMsg.contLen = 64; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); + snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); rpcMsg.handle = NULL; rpcMsg.msgType = 77; diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 70fbe55b67..33af51f23d 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -34,7 +34,7 @@ int main() { SRpcMsg rpcMsg; rpcMsg.contLen = 64; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); + snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); rpcMsg.handle = NULL; rpcMsg.msgType = 77; diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 01e5731dde..902513f7cb 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -28,13 +28,16 @@ SSyncNode* doSync() { pCfg->replicaNum = 1; pCfg->nodeInfo[0].nodePort = 7010; - taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); pCfg->nodeInfo[1].nodePort = 7110; - taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); pCfg->nodeInfo[2].nodePort = 7210; - taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); From dc30a83f2494a1adba21435c31e176e435dda13c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 17:48:39 +0800 Subject: [PATCH 40/82] monitor --- include/dnode/mnode/mnode.h | 25 +++------ include/libs/monitor/monitor.h | 52 +++++++++---------- source/common/CMakeLists.txt | 2 +- source/common/src/tname.c | 3 +- source/dnode/bnode/CMakeLists.txt | 2 +- source/dnode/mgmt/daemon/CMakeLists.txt | 2 +- source/dnode/mgmt/impl/CMakeLists.txt | 13 +---- source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/inc/dndMnode.h | 3 ++ source/dnode/mgmt/impl/src/dndMgmt.c | 18 ++++++- source/dnode/mgmt/impl/src/dndMnode.c | 10 ++++ .../dnode/mgmt/impl/test/sut/CMakeLists.txt | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 10 +--- source/dnode/mnode/impl/inc/mndInt.h | 22 ++++++-- source/dnode/mnode/impl/src/mnode.c | 10 ++-- source/dnode/mnode/sdb/CMakeLists.txt | 7 +-- source/dnode/qnode/CMakeLists.txt | 2 +- source/dnode/snode/CMakeLists.txt | 2 +- source/libs/cache/CMakeLists.txt | 2 +- source/libs/catalog/CMakeLists.txt | 2 +- source/libs/function/CMakeLists.txt | 2 +- source/libs/index/CMakeLists.txt | 2 +- source/libs/monitor/src/monitor.c | 24 +++++++-- source/libs/parser/CMakeLists.txt | 2 +- source/libs/planner/CMakeLists.txt | 2 +- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/sync/CMakeLists.txt | 2 +- source/libs/transport/CMakeLists.txt | 2 +- source/libs/wal/CMakeLists.txt | 2 +- source/os/CMakeLists.txt | 8 +-- source/util/src/thttp.c | 2 +- 31 files changed, 135 insertions(+), 105 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 2680bb83ed..d295e772e8 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -16,6 +16,8 @@ #ifndef _TD_MND_H_ #define _TD_MND_H_ +#include "monitor.h" + #ifdef __cplusplus extern "C" { #endif @@ -30,20 +32,6 @@ typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef struct SMnodeLoad { - int64_t numOfDnode; - int64_t numOfMnode; - int64_t numOfVgroup; - int64_t numOfDatabase; - int64_t numOfSuperTable; - int64_t numOfChildTable; - int64_t numOfNormalTable; - int64_t numOfColumn; - int64_t totalPoints; - int64_t totalStorage; - int64_t compStorage; -} SMnodeLoad; - typedef struct { int32_t dnodeId; int64_t clusterId; @@ -92,13 +80,16 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption); void mndDestroy(const char *path); /** - * @brief Get mnode statistics info. + * @brief Get mnode monitor info. * * @param pMnode The mnode object. - * @param pLoad Statistics of the mnode. + * @param pClusterInfo + * @param pVgroupInfo + * @param pGrantInfo * @return int32_t 0 for success, -1 for failure. */ -int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); /** * @brief Get user authentication info. diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 747c46c0c6..4b7c6a9bc9 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -55,6 +55,30 @@ typedef struct { SArray *mnodes; // array of SMonMnodeDesc } SMonClusterInfo; +typedef struct { + int32_t dnode_id; + int8_t vnode_online; + char vnode_role[8]; +} SMonVnodeDesc; + +typedef struct { + int32_t vgroup_id; + SMonVnodeDesc vnodes[TSDB_MAX_REPLICA]; +} SMonVgroupDesc; + +typedef struct { + char database_name[TSDB_DB_NAME_LEN]; + int32_t tables_num; + int8_t status; + SArray *vgroups; // array of SMonVgroupDesc +} SMonVgroupInfo; + +typedef struct { + int32_t expire_time; + int32_t timeseries_used; + int32_t timeseries_total; +} SMonGrantInfo; + typedef struct { float uptime; // day float cpu_engine; @@ -96,36 +120,12 @@ typedef struct { SArray *disks; // array of SMonDiskDesc } SMonDiskInfo; -typedef struct { - int32_t dnode_id; - int8_t vnode_online; - char vnode_role[8]; -} SMonVnodeDesc; - -typedef struct { - int32_t vgroup_id; - SMonVnodeDesc vnodes[TSDB_MAX_REPLICA]; -} SMonVgroupDesc; - -typedef struct { - char database_name[TSDB_DB_NAME_LEN]; - int32_t tables_num; - int8_t status; - SArray *vgroups; // array of SMonVgroupDesc -} SMonVgroupInfo; - typedef struct { int64_t ts; int8_t level; char content[1024]; } SMonLogItem; -typedef struct { - int32_t expire_time; - int32_t timeseries_used; - int32_t timeseries_total; -} SMonGrantInfo; - typedef struct SMonInfo SMonInfo; typedef struct { @@ -141,10 +141,10 @@ void monAddLogItem(SMonLogItem *pItem); SMonInfo *monCreateMonitorInfo(); void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); +void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); +void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo); void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); -void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); -void monSetGrantInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); void monSendReport(SMonInfo *pMonitor); void monCleanupMonitorInfo(SMonInfo *pMonitor); diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 04e5a36e86..1e4ad95504 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src COMMON_SRC) -add_library(common ${COMMON_SRC}) +add_library(common STATIC ${COMMON_SRC}) target_include_directories( common PUBLIC "${CMAKE_SOURCE_DIR}/include/common" diff --git a/source/common/src/tname.c b/source/common/src/tname.c index e061862856..fb77417cac 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -22,6 +22,7 @@ bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; } +#if 0 // TODO refactor SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) { if (numOfFilters == 0 || src == NULL) { @@ -46,7 +47,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil return pFilter; } - +#endif #if 0 int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision) { if (slidingTime == 0) { diff --git a/source/dnode/bnode/CMakeLists.txt b/source/dnode/bnode/CMakeLists.txt index a284437450..f51969f6bf 100644 --- a/source/dnode/bnode/CMakeLists.txt +++ b/source/dnode/bnode/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src BNODE_SRC) -add_library(bnode ${BNODE_SRC}) +add_library(bnode STATIC ${BNODE_SRC}) target_include_directories( bnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/bnode" diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index e07c15c95a..3238bbf3f0 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -6,4 +6,4 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(taosd dnode util os) +target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt index ec9e1be02d..c99a405527 100644 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ b/source/dnode/mgmt/impl/CMakeLists.txt @@ -1,18 +1,7 @@ aux_source_directory(src DNODE_SRC) add_library(dnode STATIC ${DNODE_SRC}) target_link_libraries( - dnode - PUBLIC cjson - PUBLIC mnode - PUBLIC vnode - PUBLIC qnode - PUBLIC snode - PUBLIC bnode - PUBLIC wal - PUBLIC sync - PUBLIC taos - PUBLIC tfs - PUBLIC monitor + dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor ) target_include_directories( dnode diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 417bc1e041..9fabe40186 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -36,6 +36,7 @@ extern "C" { #include "ttime.h" #include "tworker.h" #include "tglobal.h" +#include "monitor.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h index dafbae10ad..0f03bb3832 100644 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -32,6 +32,9 @@ int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 171bdc792c..4bf8b82043 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -474,6 +474,11 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } +void dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->dnode_id = dndGetDnodeId(pDnode); + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return; SMonInfo *pMonitor = monCreateMonitorInfo(); @@ -481,10 +486,19 @@ static void dndSendMonitorReport(SDnode *pDnode) { dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); - SMonBasicInfo basicInfo = {.dnode_id = dndGetDnodeId(pDnode)}; - tstrncpy(basicInfo.dnode_ep, tsLocalEp, TSDB_EP_LEN); + SMonBasicInfo basicInfo = {0}; + dndGetBasicInfo(pDnode, &basicInfo); monSetBasicInfo(pMonitor, &basicInfo); + SMonClusterInfo clusterInfo = {0}; + SMonVgroupInfo vgroupInfo = {0}; + SMonGrantInfo grantInfo = {0}; + if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + monSendReport(pMonitor); monCleanupMonitorInfo(pMonitor); } diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index e1b16a188f..6cb117867f 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -630,3 +630,13 @@ int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *enc dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); return code; } + +int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) return -1; + + int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); + dndReleaseMnode(pDnode, pMnode); + return code; +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt index d20c680fad..3a993986fe 100644 --- a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src SUT_SRC) -add_library(sut STATIC ${SUT_SRC}) +add_library(sut STATIC STATIC ${SUT_SRC}) target_link_libraries( sut PUBLIC dnode diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 3aad002d40..514bba19f4 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -1,18 +1,12 @@ aux_source_directory(src MNODE_SRC) -add_library(mnode ${MNODE_SRC}) +add_library(mnode STATIC ${MNODE_SRC}) target_include_directories( mnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode - PRIVATE scheduler - PRIVATE sdb - PRIVATE wal - PRIVATE transport - PRIVATE cjson - PRIVATE sync + mnode scheduler sdb wal transport cjson sync monitor ) if(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 54595fb105..929d47184b 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -21,11 +21,11 @@ #include "sdb.h" #include "tcache.h" #include "tep.h" +#include "tglobal.h" #include "tqueue.h" #include "ttime.h" -#include "wal.h" #include "version.h" -#include "tglobal.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -38,6 +38,20 @@ typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *p typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); +typedef struct SMnodeLoad { + int64_t numOfDnode; + int64_t numOfMnode; + int64_t numOfVgroup; + int64_t numOfDatabase; + int64_t numOfSuperTable; + int64_t numOfChildTable; + int64_t numOfNormalTable; + int64_t numOfColumn; + int64_t totalPoints; + int64_t totalStorage; + int64_t compStorage; +} SMnodeLoad; + typedef struct { const char *name; MndInitFp initFp; @@ -104,7 +118,9 @@ int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg); void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); -uint64_t mndGenerateUid(char *name, int32_t len) ; +uint64_t mndGenerateUid(char *name, int32_t len); + +int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index ab5d0d794b..9478fd7d5a 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -22,6 +22,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" +#include "mndInfoSchema.h" #include "mndMnode.h" #include "mndOffset.h" #include "mndProfile.h" @@ -36,7 +37,6 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndInfoSchema.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -400,6 +400,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + return 0; + } + SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); if (pMsg == NULL) { @@ -505,10 +510,9 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } } - // Note: uid 0 is reserved uint64_t mndGenerateUid(char *name, int32_t len) { - int32_t hashval = MurmurHash3_32(name, len); + int32_t hashval = MurmurHash3_32(name, len); do { int64_t us = taosGetTimestampUs(); diff --git a/source/dnode/mnode/sdb/CMakeLists.txt b/source/dnode/mnode/sdb/CMakeLists.txt index b6620f8be4..168d5063c2 100644 --- a/source/dnode/mnode/sdb/CMakeLists.txt +++ b/source/dnode/mnode/sdb/CMakeLists.txt @@ -1,13 +1,10 @@ aux_source_directory(src MNODE_SRC) -add_library(sdb ${MNODE_SRC}) +add_library(sdb STATIC ${MNODE_SRC}) target_include_directories( sdb PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode/sdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - sdb - PRIVATE os - PRIVATE common - PRIVATE util + sdb os common util ) \ No newline at end of file diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index f6f78f7357..92cd8fcb5c 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src QNODE_SRC) -add_library(qnode ${QNODE_SRC}) +add_library(qnode STATIC ${QNODE_SRC}) target_include_directories( qnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/qnode" diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index a94dd9edd8..dafd5d6594 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src SNODE_SRC) -add_library(snode ${SNODE_SRC}) +add_library(snode STATIC ${SNODE_SRC}) target_include_directories( snode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/snode" diff --git a/source/libs/cache/CMakeLists.txt b/source/libs/cache/CMakeLists.txt index 5ba59ef160..c99b5602ad 100644 --- a/source/libs/cache/CMakeLists.txt +++ b/source/libs/cache/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src CACHE_SRC) -add_library(cache ${CACHE_SRC}) +add_library(cache STATIC ${CACHE_SRC}) target_include_directories( cache PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/cache" diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index bb9ed686a7..09cd252a89 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src CATALOG_SRC) -add_library(catalog ${CATALOG_SRC}) +add_library(catalog STATIC ${CATALOG_SRC}) target_include_directories( catalog PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog" diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 9f700dbb3c..a10a542b6b 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src FUNCTION_SRC) -add_library(function ${FUNCTION_SRC}) +add_library(function STATIC ${FUNCTION_SRC}) target_include_directories( function PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 50e76abd3f..047fc555a0 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src INDEX_SRC) -add_library(index ${INDEX_SRC}) +add_library(index STATIC ${INDEX_SRC}) target_include_directories( index PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index f8a9039f48..811ee40dc8 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -96,8 +96,22 @@ void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { tjsonAddStringToObject(pJson, "ts", buf); } -void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); -void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); -void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); -void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); -void monSetGrantInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); +void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { + +} + +void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { + +} + +void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { + +} + +void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { + +} + +void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { + +} diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 417c56aba1..5d02868657 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src PARSER_SRC) -add_library(parser ${PARSER_SRC}) +add_library(parser STATIC ${PARSER_SRC}) target_include_directories( parser PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser" diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 14b4488e6a..db5d31f22b 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src PLANNER_SRC) -add_library(planner ${PLANNER_SRC}) +add_library(planner STATIC ${PLANNER_SRC}) target_include_directories( planner PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/planner" diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 1b4aee3ccf..862ae7ccae 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src SCHEDULER_SRC) -add_library(scheduler ${SCHEDULER_SRC}) +add_library(scheduler STATIC ${SCHEDULER_SRC}) target_include_directories( scheduler diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index cb38d7e363..cb196acc02 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src SYNC_SRC) -add_library(sync ${SYNC_SRC}) +add_library(sync STATIC ${SYNC_SRC}) target_link_libraries( sync diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index a2e82201bf..465646ac95 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src TRANSPORT_SRC) -add_library(transport ${TRANSPORT_SRC}) +add_library(transport STATIC ${TRANSPORT_SRC}) target_include_directories( transport PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt index bcf759e04f..4cf7cff818 100644 --- a/source/libs/wal/CMakeLists.txt +++ b/source/libs/wal/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src WAL_SRC) -add_library(wal ${WAL_SRC}) +add_library(wal STATIC ${WAL_SRC}) target_include_directories( wal PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/wal" diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 0eda9d637d..c3bf94e888 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -1,14 +1,10 @@ aux_source_directory(src OS_SRC) -add_library(os ${OS_SRC}) +add_library(os STATIC ${OS_SRC}) target_include_directories( os PUBLIC "${CMAKE_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" ) target_link_libraries( - os - PUBLIC pthread - PUBLIC dl - PUBLIC rt - PUBLIC m + os pthread dl rt m ) \ No newline at end of file diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c index aa4c795744..adf29b1aa9 100644 --- a/source/util/src/thttp.c +++ b/source/util/src/thttp.c @@ -32,7 +32,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, fd = taosOpenTcpClientSocket(ip, port, 0); if (fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create socket to %s:%u since %s", server, port, terrstr()); + uError("failed to create http socket to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } From 926934f52207989653ca3080dc49581d4f402676 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 17:50:05 +0800 Subject: [PATCH 41/82] static library --- contrib/test/craft/CMakeLists.txt | 2 +- contrib/test/tdev/CMakeLists.txt | 2 +- contrib/test/traft/single_node/CMakeLists.txt | 2 +- example/CMakeLists.txt | 6 +----- source/libs/qcom/CMakeLists.txt | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index e0f6ae64bd..d4d5a6365f 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file +target_link_libraries(simulate_vnode craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/tdev/CMakeLists.txt b/contrib/test/tdev/CMakeLists.txt index d173e8d24a..a1657a7a69 100644 --- a/contrib/test/tdev/CMakeLists.txt +++ b/contrib/test/tdev/CMakeLists.txt @@ -1,4 +1,4 @@ aux_source_directory(src TDEV_SRC) add_executable(tdev ${TDEV_SRC}) -target_include_directories(tdev PUBLIC inc) \ No newline at end of file +target_include_directories(tdev inc) \ No newline at end of file diff --git a/contrib/test/traft/single_node/CMakeLists.txt b/contrib/test/traft/single_node/CMakeLists.txt index 666ce271b8..84b65978b9 100644 --- a/contrib/test/traft/single_node/CMakeLists.txt +++ b/contrib/test/traft/single_node/CMakeLists.txt @@ -3,4 +3,4 @@ target_sources(singleNode PRIVATE "singleNode.c" ) -target_link_libraries(singleNode PUBLIC traft lz4 uv_a) +target_link_libraries(singleNode traft lz4 uv_a) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 71a9f9f4c5..51e1e996dc 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -2,11 +2,7 @@ aux_source_directory(src TMQ_DEMO_SRC) add_executable(tmq ${TMQ_DEMO_SRC}) target_link_libraries( - tmq - PUBLIC taos - #PUBLIC util - #PUBLIC common - #PUBLIC os + tmq taos ) target_include_directories( tmq diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index c63e54fb9b..a9bf0f5594 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src QUERY_SRC) -add_library(qcom ${QUERY_SRC}) +add_library(qcom STATIC ${QUERY_SRC}) target_include_directories( qcom PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom" From 5dc2cfffded8e4e6d27ea605ca4b01d7de8fbfb8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Mar 2022 17:53:44 +0800 Subject: [PATCH 42/82] minor changes --- contrib/test/tdev/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/test/tdev/CMakeLists.txt b/contrib/test/tdev/CMakeLists.txt index a1657a7a69..d173e8d24a 100644 --- a/contrib/test/tdev/CMakeLists.txt +++ b/contrib/test/tdev/CMakeLists.txt @@ -1,4 +1,4 @@ aux_source_directory(src TDEV_SRC) add_executable(tdev ${TDEV_SRC}) -target_include_directories(tdev inc) \ No newline at end of file +target_include_directories(tdev PUBLIC inc) \ No newline at end of file From 6d6e7bdf3d854bb6d423ac4cf8d9fa75bc420dbb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 18:30:21 +0800 Subject: [PATCH 43/82] sync modify timer --- source/libs/sync/inc/syncMessage.h | 2 +- source/libs/sync/src/syncIO.c | 7 +++++-- source/libs/sync/src/syncMain.c | 12 ++++++++++++ source/libs/sync/src/syncMessage.c | 1 + source/libs/sync/test/syncIOSendMsgClientTest.cpp | 2 +- source/libs/sync/test/syncIOSendMsgTest.cpp | 2 +- source/libs/sync/test/syncPingTest.cpp | 1 + 7 files changed, 22 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 603ef00517..a4ae910608 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -30,7 +30,7 @@ extern "C" { // encode as uint64 typedef enum ESyncMessageType { - SYNC_PING = 0, + SYNC_PING = 100, SYNC_PING_REPLY, SYNC_CLIENT_REQUEST, SYNC_CLIENT_REQUEST_REPLY, diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 74f848338f..bd49b04375 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -41,8 +41,11 @@ static void syncIOTickPingFunc(void *param, void *tmrId); // public function ------------ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { - sTrace("<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u", clientRpc, pEpSet->numOfEps, - pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + sTrace( + "<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u, pMsg->ahandle:%p, pMsg->handle:%p, " + "pMsg->msgType:%d, pMsg->contLen:%d", + clientRpc, pEpSet->numOfEps, pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->ahandle, pMsg->handle, + pMsg->msgType, pMsg->contLen); pMsg->handle = NULL; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d19c51900f..180a2b13a1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -168,8 +168,19 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { sTrace("syncNodePing pSyncNode:%p ", pSyncNode); int32_t ret = 0; + SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); + + /* + SRpcMsg rpcMsg; + rpcMsg.contLen = 64; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "xxxxxxxxxxxxxx"); + rpcMsg.handle = NULL; + rpcMsg.msgType = 1; + */ + syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); { @@ -219,6 +230,7 @@ static int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSync static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; + sTrace("syncNodeOnPingCb ---- ========="); return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 8e7dc8ba61..1d1b2e6965 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -47,6 +47,7 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { } void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; pRpcMsg->contLen = pMsg->bytes; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 8adaff1a2f..9e8c7c8b65 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -28,7 +28,7 @@ int main() { for (int i = 0; i < 10; ++i) { SEpSet epSet; epSet.inUse = 0; - epSet.numOfEps = 1; + epSet.numOfEps = 0; addEpIntoEpSet(&epSet, "127.0.0.1", 7030); SRpcMsg rpcMsg; diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 33af51f23d..a297981ee5 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -28,7 +28,7 @@ int main() { for (int i = 0; i < 10; ++i) { SEpSet epSet; epSet.inUse = 0; - epSet.numOfEps = 1; + epSet.numOfEps = 0; addEpIntoEpSet(&epSet, "127.0.0.1", 7010); SRpcMsg rpcMsg; diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 902513f7cb..002838b010 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -67,6 +67,7 @@ int main() { assert(ret == 0); SSyncNode* pSyncNode = doSync(); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); From 39062f19a15630581f3945e6d110193e0c9ca4e0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 2 Mar 2022 19:12:25 +0800 Subject: [PATCH 44/82] [TD-13756]: file system with lock. --- source/os/src/osFile.c | 66 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4b10a179a9..c2e9b52f88 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -15,8 +15,6 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#define MAX_FPRINTFLINE_BUFFER_SIZE (1000) - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include @@ -46,10 +44,15 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ typedef int32_t FileFd; +#define FILE_WITH_LOCK 1 + typedef struct TdFile { - int refId; - FileFd fd; - FILE *fp; +#if FILE_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + FileFd fd; + FILE *fp; } * TdFilePtr, TdFile; void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { @@ -238,6 +241,9 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { if (fp != NULL) fclose(fp); return NULL; } +#if FILE_WITH_LOCK + pthread_rwlock_init(&(pFile->rwlock),NULL); +#endif pFile->fd = fd; pFile->fp = fp; pFile->refId = 0; @@ -249,6 +255,12 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else + if (ppFile == NULL || *ppFile == NULL) { + return 0; + } +#if FILE_WITH_LOCK + pthread_rwlock_wrlock(&((*ppFile)->rwlock)); +#endif if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { return 0; } @@ -263,6 +275,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { (*ppFile)->fd = -1; } (*ppFile)->refId = 0; +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&((*ppFile)->rwlock)); + pthread_rwlock_destroy(&((*ppFile)->rwlock)); +#endif free(*ppFile); *ppFile = NULL; return 0; @@ -273,6 +289,9 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { if (pFile == NULL) { return 0; } +#if FILE_WITH_LOCK + pthread_rwlock_rdlock(&(pFile->rwlock)); +#endif assert(pFile->fd >= 0); int64_t leftbytes = count; int64_t readbytes; @@ -284,9 +303,15 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { if (errno == EINTR) { continue; } else { +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif return -1; } } else if (readbytes == 0) { +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif return (int64_t)(count - leftbytes); } @@ -294,6 +319,9 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { tbuf += readbytes; } +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif return count; } @@ -301,14 +329,24 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) if (pFile == NULL) { return 0; } +#if FILE_WITH_LOCK + pthread_rwlock_rdlock(&(pFile->rwlock)); +#endif assert(pFile->fd >= 0); - return pread(pFile->fd, buf, count, offset); + int64_t ret = pread(pFile->fd, buf, count, offset); +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif + return ret; } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { if (pFile == NULL) { return 0; } +#if FILE_WITH_LOCK + pthread_rwlock_wrlock(&(pFile->rwlock)); +#endif assert(pFile->fd >= 0); int64_t nleft = count; @@ -321,12 +359,18 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { if (errno == EINTR) { continue; } +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif return -1; } nleft -= nwritten; tbuf += nwritten; } +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif return count; } @@ -334,8 +378,15 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { if (pFile == NULL) { return 0; } +#if FILE_WITH_LOCK + pthread_rwlock_rdlock(&(pFile->rwlock)); +#endif assert(pFile->fd >= 0); - return (int64_t)lseek(pFile->fd, (long)offset, whence); + int64_t ret = lseek(pFile->fd, (long)offset, whence); +#if FILE_WITH_LOCK + pthread_rwlock_unlock(&(pFile->rwlock)); +#endif + return ret; } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { @@ -637,7 +688,6 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { } assert(pFile->fp != NULL); - char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; va_list ap; va_start(ap, format); vfprintf(pFile->fp, format, ap); From 99bc46e7de6bbcd938c5d413c033d0e560b03b74 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 19:56:15 +0800 Subject: [PATCH 45/82] sync modify timer --- source/libs/sync/src/syncIO.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index bd49b04375..85a9459483 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -210,10 +210,28 @@ static void *syncIOConsumerFunc(void *param) { if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { - SyncPing *pSyncMsg = syncPingBuild(pRpcMsg->contLen); - syncPingFromRpcMsg(pRpcMsg, pSyncMsg); + SyncPing *pSyncMsg; + + SRpcMsg tmpRpcMsg; + memcpy(&tmpRpcMsg, pRpcMsg, sizeof(SRpcMsg)); + pSyncMsg = syncPingBuild(tmpRpcMsg.contLen); + + // syncPingFromRpcMsg(pRpcMsg, pSyncMsg); + + memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); + +#if 0 + { + cJSON *pJson = syncPing2Json(pSyncMsg); + char *serialized = cJSON_Print(pJson); + sTrace("syncIOConsumerFunc syncNodePing pMsg:%s ", serialized); + free(serialized); + cJSON_Delete(pJson); + } +#endif io->FpOnSyncPing(io->pSyncNode, pSyncMsg); } + } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); @@ -257,7 +275,8 @@ static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, cha } static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sTrace("syncIOProcessRequest: type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont); + sTrace("<-- syncIOProcessRequest --> type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen, + (char *)pMsg->pCont); SSyncIO *io = pParent; SRpcMsg *pTemp; From 08df4b42de6823bbdf9231aa68d0e974f86be371 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 20:08:46 +0800 Subject: [PATCH 46/82] sync modify timer --- source/libs/sync/inc/syncMessage.h | 4 ++-- source/libs/sync/src/syncIO.c | 13 ++----------- source/libs/sync/src/syncMain.c | 11 ++++++++++- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index a4ae910608..bb9ec65d3b 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -30,8 +30,8 @@ extern "C" { // encode as uint64 typedef enum ESyncMessageType { - SYNC_PING = 100, - SYNC_PING_REPLY, + SYNC_PING = 101, + SYNC_PING_REPLY = 103, SYNC_CLIENT_REQUEST, SYNC_CLIENT_REQUEST_REPLY, SYNC_REQUEST_VOTE, diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 85a9459483..590466d1cc 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -216,19 +216,10 @@ static void *syncIOConsumerFunc(void *param) { memcpy(&tmpRpcMsg, pRpcMsg, sizeof(SRpcMsg)); pSyncMsg = syncPingBuild(tmpRpcMsg.contLen); - // syncPingFromRpcMsg(pRpcMsg, pSyncMsg); + syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); + // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); -#if 0 - { - cJSON *pJson = syncPing2Json(pSyncMsg); - char *serialized = cJSON_Print(pJson); - sTrace("syncIOConsumerFunc syncNodePing pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } -#endif io->FpOnSyncPing(io->pSyncNode, pSyncMsg); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 180a2b13a1..ce3dba4c0e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -230,7 +230,16 @@ static int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSync static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; - sTrace("syncNodeOnPingCb ---- ========="); + sTrace("<-- syncNodeOnPingCb -->"); + + { + cJSON* pJson = syncPing2Json(pMsg); + char* serialized = cJSON_Print(pJson); + sTrace("syncNodeOnPingCb syncNodePing pMsg:%s ", serialized); + free(serialized); + cJSON_Delete(pJson); + } + return ret; } From fe1f280ce4440c1d71fdda5897eb5ae1c2f161d5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 2 Mar 2022 20:43:03 +0800 Subject: [PATCH 47/82] sync modify timer --- source/libs/sync/inc/syncMessage.h | 4 ++ source/libs/sync/src/syncIO.c | 5 ++- source/libs/sync/src/syncMain.c | 15 +++++++ source/libs/sync/src/syncMessage.c | 54 +++++++++++++++++++++++++- source/libs/sync/src/syncUtil.c | 1 + source/libs/sync/test/syncPingTest.cpp | 3 +- 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index bb9ec65d3b..be53559e8a 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -102,6 +102,10 @@ void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); cJSON* syncPingReply2Json(const SyncPingReply* pMsg); +SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); + +SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId); + typedef struct SyncClientRequest { ESyncMessageType msgType; char* data; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 590466d1cc..abb52b42d0 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -226,7 +226,10 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); - io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); + + if (io->FpOnSyncPingReply != NULL) { + io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); + } } else { ; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ce3dba4c0e..49fac038da 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -240,11 +240,26 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { cJSON_Delete(pJson); } + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->raftId, &pMsg->srcId); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pMsgReply, &rpcMsg); + syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + return ret; } static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { int32_t ret = 0; + sTrace("<-- syncNodeOnPingReplyCb -->"); + + { + cJSON* pJson = syncPingReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + sTrace("syncNodeOnPingReplyCb syncNodePing pMsg:%s ", serialized); + free(serialized); + cJSON_Delete(pJson); + } + return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 1d1b2e6965..a26c8401b2 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -15,6 +15,7 @@ #include "syncMessage.h" #include "syncRaft.h" +#include "syncUtil.h" #include "tcoding.h" void onMessage(SRaft* pRaft, void* pMsg) {} @@ -65,11 +66,29 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON* pSrcId = cJSON_CreateObject(); cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); @@ -101,7 +120,7 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { SyncPingReply* pMsg = malloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_PING; + pMsg->msgType = SYNC_PING_REPLY; pMsg->dataLen = dataLen; } @@ -123,6 +142,7 @@ void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg } void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; pRpcMsg->contLen = pMsg->bytes; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); @@ -140,11 +160,29 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON* pSrcId = cJSON_CreateObject(); cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); @@ -156,6 +194,20 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { return pJson; } +SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) { + uint32_t dataLen = strlen(str) + 1; + SyncPingReply* pMsg = syncPingReplyBuild(dataLen); + pMsg->srcId = *srcId; + pMsg->destId = *destId; + snprintf(pMsg->data, pMsg->dataLen, "%s", str); + return pMsg; +} + +SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) { + SyncPingReply* pMsg = syncPingReplyBuild2(srcId, destId, "pang"); + return pMsg; +} + #if 0 void syncPingSerialize(const SyncPing* pMsg, char** ppBuf, uint32_t* bufLen) { *bufLen = sizeof(SyncPing) + pMsg->dataLen; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index a3fbfaf905..b4959a810b 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -38,6 +38,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port) { void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; + pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, pNodeInfo->nodeFqdn, pNodeInfo->nodePort); } diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 002838b010..24d9ead5e3 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -25,7 +25,7 @@ SSyncNode* doSync() { SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = 0; - pCfg->replicaNum = 1; + pCfg->replicaNum = 2; pCfg->nodeInfo[0].nodePort = 7010; snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); @@ -68,6 +68,7 @@ int main() { SSyncNode* pSyncNode = doSync(); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); From 7ae87c10103c801d06092ea3ddea00b9e9b671bd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 22:06:02 +0800 Subject: [PATCH 48/82] add UT --- source/libs/index/inc/index_comm.h | 32 +++++++++++++++++++ source/libs/index/src/index_cache.c | 35 ++------------------ source/libs/index/src/index_comm.c | 48 ++++++++++++++++++++++++++++ source/libs/index/src/index_tfile.c | 13 +++++++- source/libs/index/test/fstUT.cc | 16 ++++++++++ source/libs/index/test/jsonDemo.cc | 0 source/libs/index/test/jsonUT.cc | 33 +++++++++++++++++-- source/libs/transport/src/transSrv.c | 20 ++++++------ 8 files changed, 153 insertions(+), 44 deletions(-) create mode 100644 source/libs/index/inc/index_comm.h create mode 100644 source/libs/index/src/index_comm.c delete mode 100644 source/libs/index/test/jsonDemo.cc diff --git a/source/libs/index/inc/index_comm.h b/source/libs/index/inc/index_comm.h new file mode 100644 index 0000000000..0d8418ba65 --- /dev/null +++ b/source/libs/index/inc/index_comm.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_INDEX_COMM_H_ +#define _TD_INDEX_COMM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char JSON_COLUMN[]; +extern char JSON_VALUE_DELIM; + +char* indexPackJsonData(SIndexTerm* itm); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 2742830d3b..599bac3fe6 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -14,6 +14,7 @@ */ #include "index_cache.h" +#include "index_comm.h" #include "index_util.h" #include "tcompare.h" #include "tsched.h" @@ -24,9 +25,6 @@ #define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 -static char JSON_COLUMN[] = "JSON"; -static char JSON_VALUE_DELIM = '&'; - static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -211,33 +209,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } } } -static char* indexCachePackJsonData(SIndexTerm* itm) { - /* - * |<-----colname---->|<-----dataType---->|<--------colVal---------->| - * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| - */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); - - int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; - char* buf = (char*)calloc(1, sz); - char* p = buf; - - memcpy(p, itm->colName, itm->nColName); - p += itm->nColName; - - memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); - p += sizeof(JSON_VALUE_DELIM); - - memcpy(p, &ty, sizeof(ty)); - p += sizeof(ty); - - memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); - p += sizeof(JSON_VALUE_DELIM); - - memcpy(p, itm->colVal, itm->nColVal); - - return buf; -} int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; @@ -254,7 +225,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // set up key ct->colType = term->colType; if (hasJson) { - ct->colVal = indexCachePackJsonData(term); + ct->colVal = indexPackJsonData(term); } else { ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal); @@ -333,7 +304,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); char* p = term->colVal; if (hasJson) { - p = indexCachePackJsonData(term); + p = indexPackJsonData(term); } CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; diff --git a/source/libs/index/src/index_comm.c b/source/libs/index/src/index_comm.c new file mode 100644 index 0000000000..4f3cbaa4da --- /dev/null +++ b/source/libs/index/src/index_comm.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index.h" +#include "indexInt.h" + +char JSON_COLUMN[] = "JSON"; +char JSON_VALUE_DELIM = '&'; + +char* indexPackJsonData(SIndexTerm* itm) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)calloc(1, sz); + char* p = buf; + + memcpy(p, itm->colName, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, itm->colVal, itm->nColVal); + + return buf; +} diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a05884b790..a2089e8eee 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -15,6 +15,7 @@ p * #include "index_tfile.h" #include "index.h" +#include "index_comm.h" #include "index_fst.h" #include "index_fst_counting_writer.h" #include "index_util.h" @@ -186,13 +187,20 @@ void tfileReaderDestroy(TFileReader* reader) { int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { SIndexTerm* term = query->term; + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); EIndexQueryType qtype = query->qType; int ret = -1; // refactor to callback later if (qtype == QUERY_TERM) { uint64_t offset; - FstSlice key = fstSliceCreate(term->colVal, term->nColVal); + char* p = term->colVal; + uint64_t sz = term->nColVal; + if (hasJson) { + p = indexPackJsonData(term); + sz = strlen(p); + } + FstSlice key = fstSliceCreate(p, sz); if (fstGet(reader->fst, &key, &offset)) { indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); @@ -202,6 +210,9 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul term->colVal); } fstSliceDestroy(&key); + if (hasJson) { + free(p); + } } else if (qtype == QUERY_PREFIX) { // handle later // diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index b34fd71e9c..3bc3b6da6a 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -238,3 +238,19 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Search(ctx, rlt) == true); } TEST_F(FstEnv, WriteMillonrRecord) {} +TEST_F(FstEnv, writeAbNormal) { + fst->CreateWriter(); + std::string str1("voltage&\b&ab"); + std::string str2("voltbge&\b&ab"); + + fst->Put(str1, 1); + fst->Put(str2, 2); + + fst->DestroyWriter(); + + fst->CreateReader(); + uint64_t val; + assert(fst->Get("1", &val) == false); + assert(fst->Get("voltage&\b&ab", &val) == true); + assert(val == 1); +} diff --git a/source/libs/index/test/jsonDemo.cc b/source/libs/index/test/jsonDemo.cc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 20b59add9f..e184bc49ae 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -21,6 +21,8 @@ class JsonEnv : public ::testing::Test { protected: virtual void SetUp() { taosRemoveDir(dir.c_str()); + taosMkDir(dir.c_str()); + opts = indexOptsCreate(); int ret = tIndexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); @@ -87,6 +89,33 @@ TEST_F(JsonEnv, testWrite) { assert(100 == taosArrayGetSize(result)); indexMultiTermQueryDestroy(mq); } - - // SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; +} +TEST_F(JsonEnv, testWriteMillonData) { + { + std::string colName("voltagefdadfa"); + std::string colVal("abxxxxxxxxxxxx"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab"); + + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SArray* result = taosArrayInit(1, sizeof(uint64_t)); + indexMultiTermQueryAdd(mq, q, QUERY_TERM); + tIndexJsonSearch(index, mq, result); + assert(100 == taosArrayGetSize(result)); + indexMultiTermQueryDestroy(mq); + } } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index f0db054797..c7b6ca2a2c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -286,15 +286,17 @@ void uvOnWriteCb(uv_write_t* req, int status) { transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); - assert(taosArrayGetSize(conn->srvMsgs) >= 1); - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); - taosArrayRemove(conn->srvMsgs, 0); - destroySmsg(msg); + if (conn->srvMsgs != NULL) { + assert(taosArrayGetSize(conn->srvMsgs) >= 1); + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + taosArrayRemove(conn->srvMsgs, 0); + destroySmsg(msg); - // send second data, just use for push - if (taosArrayGetSize(conn->srvMsgs) > 0) { - msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); - uvStartSendRespInternal(msg); + // send second data, just use for push + if (taosArrayGetSize(conn->srvMsgs) > 0) { + msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); + uvStartSendRespInternal(msg); + } } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); @@ -615,7 +617,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); destroySmsg(msg); } - taosArrayDestroy(conn->srvMsgs); + conn->srvMsgs = taosArrayDestroy(conn->srvMsgs); QUEUE_REMOVE(&conn->queue); if (clear) { From 9b5b74b8e1f005b00bd4825cd3d7521f0cb97278 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Mar 2022 23:30:02 +0800 Subject: [PATCH 49/82] [td-13039] fix bug and memory leak in the unit test. --- include/util/tcache.h | 52 +-- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 25 +- source/libs/executor/test/executorTests.cpp | 6 +- source/util/src/tcache.c | 448 ++++++++++++-------- source/util/src/thash.c | 65 +-- source/util/src/tpagedbuf.c | 18 +- source/util/test/arrayTest.cpp | 5 + source/util/test/encodeTest.cpp | 8 +- source/util/test/hashTest.cpp | 9 +- source/util/test/pageBufferTest.cpp | 113 ++--- 11 files changed, 384 insertions(+), 367 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index 0de3ab3a28..7c29ab4f58 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -40,56 +40,10 @@ typedef struct SCacheStatis { int64_t refreshCount; } SCacheStatis; +typedef struct SCacheObj SCacheObj; + struct STrashElem; -typedef struct SCacheDataNode { - uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t lifespan; // life duration when this element should be remove from cache - uint64_t expireTime; // expire time - uint64_t signature; - struct STrashElem *pTNodeHeader; // point to trash node head - uint16_t keySize : 15; // max key size: 32kb - bool inTrashcan : 1; // denote if it is in trash or not - uint32_t size; // allocated size for current SCacheDataNode - T_REF_DECLARE() - char *key; - char data[]; -} SCacheDataNode; - -typedef struct STrashElem { - struct STrashElem *prev; - struct STrashElem *next; - SCacheDataNode *pData; -} STrashElem; - -/* - * to accommodate the old data which has the same key value of new one in hashList - * when an new node is put into cache, if an existed one with the same key: - * 1. if the old one does not be referenced, update it. - * 2. otherwise, move the old one to pTrash, addedTime the new one. - * - * when the node in pTrash does not be referenced, it will be release at the expired expiredTime - */ -typedef struct { - int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. - int64_t refreshTime; - STrashElem *pTrash; - char *name; - SCacheStatis statistics; - SHashObj *pHashTable; - __cache_free_fn_t freeFp; - uint32_t numOfElemsInTrash; // number of element in trash - uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; - bool extendLifespan; // auto extend life span when one item is accessed. - int64_t checkTick; // tick used to record the check times of the refresh threads -#if defined(LINUX) - pthread_rwlock_t lock; -#else - pthread_mutex_t lock; -#endif -} SCacheObj; - /** * initialize the cache object * @param keyType key type @@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data); * @param data * @return */ -void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); +void *taosCacheTransferData(SCacheObj *pCacheObj, void **data); /** * remove data in cache, the data will not be removed immediately. diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6f7055ad65..f75720dc86 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -615,7 +615,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index de56a12a5d..209bd643e5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -6224,7 +6224,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); @@ -6274,7 +6274,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); @@ -7273,24 +7273,18 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn return pOperator; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); initAggSup(&pInfo->aggSup, pExprInfo); - // todo: pInfo->order = TSDB_ORDER_ASC; pInfo->precision = TSDB_TIME_PRECISION_MICRO; - pInfo->win.skey = INT64_MIN; - pInfo->win.ekey = INT64_MAX; - pInfo->interval.intervalUnit = 's'; - pInfo->interval.slidingUnit = 's'; - pInfo->interval.interval = 1000; - pInfo->interval.sliding = 1000; + pInfo->win = pTaskInfo->window; + pInfo->interval = *pInterval; - int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); - int32_t numOfOutput = taosArrayGetSize(pExprInfo); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); @@ -7305,16 +7299,15 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfOutput; + pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->info = pInfo; - pOperator->nextDataFn = doIntervalAgg; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; } - SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 85f644335e..d04d72af85 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -548,7 +548,11 @@ TEST(testCase, time_interval_Operator_Test) { SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2); SExecTaskInfo ti = {0}; - SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti); + SInterval interval = {0}; + interval.sliding = interval.interval = 1000; + interval.slidingUnit = interval.intervalUnit = 'a'; + + SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &interval, &ti); bool newgroup = false; SSDataBlock* pRes = NULL; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 560e5348c2..aec2530f5d 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -15,11 +15,88 @@ #define _DEFAULT_SOURCE #include "tcache.h" +#include "taoserror.h" #include "tlog.h" -#include "ttimer.h" #include "tutil.h" -static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { +static pthread_t cacheRefreshWorker = {0}; +static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static SArray *pCacheArrayList = NULL; +static bool stopRefreshWorker = false; +static bool refreshWorkerNormalStopped = false; +static bool refreshWorkerUnexpectedStopped = false; + +typedef struct SCacheNode { + uint64_t addedTime; // the added time when this element is added or updated into cache + uint64_t lifespan; // life duration when this element should be remove from cache + int64_t expireTime; // expire time + uint64_t signature; + struct STrashElem *pTNodeHeader; // point to trash node head + uint16_t keyLen: 15; // max key size: 32kb + bool inTrashcan : 1; // denote if it is in trash or not + uint32_t size; // allocated size for current SCacheNode + uint32_t dataLen; + T_REF_DECLARE() + struct SCacheNode *pNext; + char *key; + char *data; +} SCacheNode; + +typedef struct SCacheEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SCacheNode *next; +} SCacheEntry; + +typedef struct STrashElem { + struct STrashElem *prev; + struct STrashElem *next; + SCacheNode *pData; +} STrashElem; + +/* + * to accommodate the old data which has the same key value of new one in hashList + * when an new node is put into cache, if an existed one with the same key: + * 1. if the old one does not be referenced, update it. + * 2. otherwise, move the old one to pTrash, addedTime the new one. + * + * when the node in pTrash does not be referenced, it will be release at the expired expiredTime + */ +struct SCacheObj { + int64_t sizeInBytes; // total allocated buffer in this hash table, SCacheObj is not included. + int64_t refreshTime; + char *name; + SCacheStatis statistics; + + SCacheEntry *pEntryList; + size_t capacity; // number of slots + size_t numOfElems; // number of elements in cache + _hash_fn_t hashFp; // hash function + __cache_free_fn_t freeFp; + + uint32_t numOfElemsInTrash; // number of element in trash + STrashElem *pTrash; + + uint8_t deleting; // set the deleting flag to stop refreshing ASAP. + pthread_t refreshWorker; + bool extendLifespan; // auto extend life span when one item is accessed. + int64_t checkTick; // tick used to record the check times of the refresh threads +#if defined(LINUX) + pthread_rwlock_t lock; +#else + pthread_mutex_t lock; +#endif +}; + +typedef struct SCacheObjTravSup { + SCacheObj *pCacheObj; + int64_t time; + __cache_trav_fn_t fp; + void *param1; +} SCacheObjTravSup; + +static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_wrlock(&pCacheObj->lock); #else @@ -27,7 +104,7 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { +static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_unlock(&pCacheObj->lock); #else @@ -35,7 +112,7 @@ static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) { +static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) return pthread_rwlock_init(&pCacheObj->lock, NULL); #else @@ -43,7 +120,7 @@ static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) { +static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_destroy(&pCacheObj->lock); #else @@ -63,14 +140,6 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); */ static void *taosCacheTimedRefresh(void *handle); -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; -static SArray *pCacheArrayList = NULL; -static bool stopRefreshWorker = false; -static bool refreshWorkerNormalStopped = false; -static bool refreshWorkerUnexpectedStopped = false; - static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); @@ -99,9 +168,9 @@ pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { * in pData. Pointer copy causes memory access error. * @param size size of block * @param lifespan total survial expiredTime from now - * @return SCacheDataNode + * @return SCacheNode */ -static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, +static SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration); /** @@ -110,7 +179,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const * @param pCacheObj Cache object * @param pNode Cache slot object */ -static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode); +static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode); /** * remove nodes in trash with refCount == 0 in cache @@ -126,18 +195,16 @@ static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force); * @param pCacheObj cache object * @param pNode data node */ -static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->signature != (uint64_t)pNode) { uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); return; } - atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); - int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); - assert(size > 0); + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, total num:%d size:%" PRId64 "bytes", - pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize); + pCacheObj->name, pNode->key, pNode->data, pNode->size, (int)pCacheObj->numOfElems - 1, pCacheObj->sizeInBytes); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -181,6 +248,45 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem free(pElem); } +static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { + assert(pNode != NULL && pEntry != NULL); + + pNode->pNext = pEntry->next; + pEntry->next = pNode; + pEntry->num += 1; +} + +static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pNode) { + if (prev == NULL) { + ASSERT(pe->next == pNode); + pe->next = pNode->pNext; + } else { + prev->pNext = pNode->pNext; + } + + pe->num -= 1; +} + +static FORCE_INLINE SCacheEntry* doFindEntry(SCacheObj* pCacheObj, const void* key, size_t keyLen) { + uint32_t hashVal = (*pCacheObj->hashFp)(key, keyLen); + int32_t slot = hashVal % pCacheObj->capacity; + return &pCacheObj->pEntryList[slot]; +} + +static FORCE_INLINE SCacheNode * +doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode** prev) { + SCacheNode *pNode = pe->next; + while (pNode) { + if ((pNode->keyLen == keyLen) && memcmp(pNode->key, key, keyLen) == 0) { + break; + } + *prev = pNode; + pNode = pNode->pNext; + } + + return pNode; +} + SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName) { const int32_t SLEEP_DURATION = 500; // 500 ms @@ -195,39 +301,41 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); - pCacheObj->name = strdup(cacheName); - if (pCacheObj->pHashTable == NULL) { + pCacheObj->pEntryList = calloc(4096, sizeof(SCacheEntry)); + if (pCacheObj->pEntryList == NULL) { free(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } // set free cache node callback function - pCacheObj->freeFp = fn; - pCacheObj->refreshTime = refreshTimeInSeconds * 1000; - pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; + pCacheObj->capacity = 4096; // todo refactor + pCacheObj->hashFp = taosGetDefaultHashFunction(keyType); + pCacheObj->freeFp = fn; + pCacheObj->refreshTime = refreshTimeInSeconds * 1000; + pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access - if (__cache_lock_init(pCacheObj) != 0) { - taosHashCleanup(pCacheObj->pHashTable); + if (__trashcan_lock_init(pCacheObj) != 0) { + tfree(pCacheObj->pEntryList); free(pCacheObj); uError("failed to init lock, reason:%s", strerror(errno)); return NULL; } + pCacheObj->name = strdup(cacheName); doRegisterCacheObj(pCacheObj); return pCacheObj; } void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int32_t durationMS) { - if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { + if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); + SCacheNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; @@ -235,87 +343,77 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v T_REF_INC(pNode1); - int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - if (succ == 0) { - atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen); + + taosWLockLatch(&pe->latch); + + SCacheNode *prev = NULL; + SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + + if (pNode == NULL) { + pushfrontNodeInEntryList(pe, pNode1); + atomic_add_fetch_64(&pCacheObj->numOfElems, 1); + atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size); uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 - ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); + ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems, + pCacheObj->sizeInBytes, (int64_t)dataSize); } else { // duplicated key exists - while (1) { - SCacheDataNode *p = NULL; - // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); - int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen); + // move current node to trashcan + removeNodeInEntryList(pe, prev, pNode); - // add to trashcan - if (ret == 0) { - if (T_REF_VAL_GET(p) == 0) { - if (pCacheObj->freeFp) { - pCacheObj->freeFp(p->data); - } - - atomic_sub_fetch_64(&pCacheObj->totalSize, p->size); - tfree(p); - } else { - taosAddToTrashcan(pCacheObj, p); - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); - } + if (T_REF_VAL_GET(pNode) == 0) { + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); } - assert(T_REF_VAL_GET(pNode1) == 1); - - ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - if (ret == 0) { - atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); - - uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 - ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - - return pNode1->data; - - } else { - // failed, try again - } + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); + tfree(pNode); + } else { + taosAddToTrashcan(pCacheObj, pNode); + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data); } + + pushfrontNodeInEntryList(pe, pNode1); + atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size); + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems, + pCacheObj->sizeInBytes, (int64_t)dataSize); } + taosWUnLockLatch(&pe->latch); return pNode1->data; } -static void incRefFn(void *ptNode) { - assert(ptNode != NULL); - - SCacheDataNode **p = (SCacheDataNode **)ptNode; - assert(T_REF_VAL_GET(*p) >= 0); - - int32_t ret = T_REF_INC(*p); - assert(ret > 0); -} - void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { if (pCacheObj == NULL || pCacheObj->deleting == 1) { return NULL; } - if (taosHashGetSize(pCacheObj->pHashTable) == 0) { + if (pCacheObj->numOfElems == 0) { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); return NULL; } - // TODO remove it - SCacheDataNode *ptNode = NULL; - ptNode = taosHashAcquire(pCacheObj->pHashTable, key, keyLen); - // taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); + SCacheNode *prev = NULL; + SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen); - void *pData = (ptNode != NULL) ? ptNode->data : NULL; + taosRLockLatch(&pe->latch); + SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + if (pNode != NULL) { + int32_t ref = T_REF_INC(pNode); + ASSERT(ref > 0); + } + + taosRUnLockLatch(&pe->latch); + + void *pData = (pNode != NULL) ? pNode->data : NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, - T_REF_VAL_GET(ptNode)); + T_REF_VAL_GET(pNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -328,9 +426,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; - size_t offset = offsetof(SCacheDataNode, data); - SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset); - + SCacheNode *ptNode = (SCacheNode *)((char *)data - sizeof(SCacheNode)); if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; @@ -344,24 +440,20 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { return data; } -void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { +void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) { if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; - size_t offset = offsetof(SCacheDataNode, data); - SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); - + SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } assert(T_REF_VAL_GET(ptNode) >= 1); - char *d = *data; // clear its reference to old area *data = NULL; - return d; } @@ -379,9 +471,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // therefore the check for the empty of both the hash table and the trashcan has a race condition. // It happens when there is only one object in the cache, and two threads which has referenced this object // start to free the it simultaneously [TD-1569]. - size_t offset = offsetof(SCacheDataNode, data); - - SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); if (pNode->signature != (uint64_t)pNode) { uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode); return; @@ -420,9 +510,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // destroyed by refresh worker if decrease ref count before removing it from linked-list. assert(pNode->pTNodeHeader->pData == pNode); - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); ref = T_REF_DEC(pNode); assert(ref == 0); @@ -435,36 +525,37 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. - SCacheDataNode *p = NULL; - int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void - // *)); + SCacheNode * prev = NULL; + SCacheEntry *pe = doFindEntry(pCacheObj, pNode->key, pNode->keyLen); + + taosWLockLatch(&pe->latch); ref = T_REF_DEC(pNode); - // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. - // note that the remove operation can be executed only once. - if (ret == 0) { + SCacheNode *p = doSearchInEntryList(pe, pNode->key, pNode->keyLen, &prev); + + if (p != NULL) { + // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. + // note that the remove operation can be executed only once. if (p != pNode) { uDebug( - "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " - "others already", - pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); + "cache:%s, key:%p, a new entry:%p found, refcnt:%d, prev entry:%p, refcnt:%d has been removed by " + "others already, prev must in trashcan", + pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode)); - assert(p->pTNodeHeader == NULL); - taosAddToTrashcan(pCacheObj, p); + assert(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); } else { + removeNodeInEntryList(pe, prev, p); uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 - atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); - int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); + int32_t size = (int32_t)pCacheObj->numOfElems; uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes", - pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); + pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->sizeInBytes); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -473,6 +564,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { free(pNode); } } + + taosWUnLockLatch(&pe->latch); } else { uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); @@ -484,45 +577,15 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char *key = pNode->key; char *p = pNode->data; - // int32_t ref = T_REF_VAL_GET(pNode); - // - // if (ref == 1 && inTrashcan) { - // // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may - // be - // // destroyed by refresh worker if decrease ref count before removing it from linked-list. - // assert(pNode->pTNodeHeader->pData == pNode); - // - // __cache_wr_lock(pCacheObj); - // doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); - // __cache_unlock(pCacheObj); - // - // ref = T_REF_DEC(pNode); - // assert(ref == 0); - // - // doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); - // } else { - // ref = T_REF_DEC(pNode); - // assert(ref >= 0); - // } - int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); } } -typedef struct SHashTravSupp { - SCacheObj *pCacheObj; - int64_t time; - __cache_trav_fn_t fp; - void *param1; -} SHashTravSupp; - -static bool travHashTableEmptyFn(void *param, void *data) { - SHashTravSupp *ps = (SHashTravSupp *)param; +static bool doRemoveNodeFn(void *param, SCacheNode *pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode *pNode = *(SCacheDataNode **)data; - if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); } else { // do add to trashcan @@ -533,10 +596,38 @@ static bool travHashTableEmptyFn(void *param, void *data) { return false; } -void taosCacheEmpty(SCacheObj *pCacheObj) { - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; +void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) { + int32_t numOfEntries = (int32_t)pCacheObj->capacity; + for (int32_t i = 0; i < numOfEntries; ++i) { + SCacheEntry *pEntry = &pCacheObj->pEntryList[i]; + if (pEntry->num == 0) { + continue; + } -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + taosWLockLatch(&pEntry->latch); + + SCacheNode *pNode = pEntry->next; + while (pNode != NULL) { + SCacheNode *next = pNode->pNext; + + if (fp(pSup, pNode)) { + pNode = pNode->pNext; + } else { + pEntry->next = next; + pEntry->num -= 1; + + atomic_sub_fetch_64(&pCacheObj->numOfElems, 1); + pNode = next; + } + } + + taosWUnLockLatch(&pEntry->latch); + } +} + +void taosCacheEmpty(SCacheObj* pCacheObj) { + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + doTraverseElems(pCacheObj, doRemoveNodeFn, &sup); taosTrashcanEmpty(pCacheObj, false); } @@ -559,38 +650,41 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { doCleanupDataCache(pCacheObj); } -SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { - size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; +SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { + size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen; - SCacheDataNode *pNewNode = calloc(1, totalSize); + SCacheNode *pNewNode = calloc(1, sizeInBytes); if (pNewNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } + pNewNode->data = (char*)pNewNode + sizeof(SCacheNode); + pNewNode->dataLen = size; memcpy(pNewNode->data, pData, size); - pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size; - pNewNode->keySize = (uint16_t)keyLen; + pNewNode->key = (char *)pNewNode + sizeof(SCacheNode) + size; + pNewNode->keyLen = (uint16_t)keyLen; memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->lifespan = duration; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)sizeInBytes; return pNewNode; } -void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; pElem->prev = NULL; @@ -605,14 +699,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->pTrash = pElem; pCacheObj->numOfElemsInTrash++; - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); if (pCacheObj->numOfElemsInTrash == 0) { if (pCacheObj->pTrash != NULL) { @@ -621,7 +715,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { pCacheObj->numOfElemsInTrash); } - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); return; } @@ -646,29 +740,27 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { } } - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); } void doCleanupDataCache(SCacheObj *pCacheObj) { - // SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + doTraverseElems(pCacheObj, doRemoveNodeFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? - taosHashCleanup(pCacheObj->pHashTable); taosTrashcanEmpty(pCacheObj, true); - __cache_lock_destroy(pCacheObj); + __trashcan_lock_destroy(pCacheObj); + tfree(pCacheObj->pEntryList); tfree(pCacheObj->name); - memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } -bool travHashTableFn(void *param, void *data) { - SHashTravSupp *ps = (SHashTravSupp *)param; +bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode *pNode = *(SCacheDataNode **)data; if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); @@ -687,8 +779,8 @@ bool travHashTableFn(void *param, void *data) { static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { assert(pCacheObj != NULL); - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; + doTraverseElems(pCacheObj, doRemoveExpiredFn, &sup); } void taosCacheRefreshWorkerUnexpectedStopped(void) { @@ -747,7 +839,7 @@ void *taosCacheTimedRefresh(void *handle) { continue; } - size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable); + size_t elemInHash = pCacheObj->numOfElems; if (elemInHash + pCacheObj->numOfElemsInTrash == 0) { continue; } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 87ae48a3b3..efbd9adddf 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -21,7 +21,6 @@ // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT #define MAX_WARNING_REF_COUNT 10000 -#define EXT_SIZE 1024 #define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_INDEX(v, c) ((v) & ((c)-1)) @@ -211,14 +210,14 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); /** - * initialize a hash table * - * @param capacity initial capacity of the hash table - * @param fn hash function - * @param update whether the hash table allows in place update - * @param type whether the hash table has per entry lock - * @return hash table object + * @param pHashObj + * @return */ +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} + SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (fn == NULL) { assert(0); @@ -296,10 +295,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)atomic_load_64(&pHashObj->size); } -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} - int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { if (pHashObj == NULL || key == NULL || keyLen == 0) { return -1; @@ -318,6 +313,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da taosHashWUnlock(pHashObj); } + // disable resize taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); @@ -326,11 +322,13 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; +#if 0 if (pe->num > 0) { assert(pNode != NULL); } else { assert(pNode == NULL); } +#endif SHashNode* prev = NULL; while (pNode) { @@ -369,7 +367,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize taosHashRUnlock(pHashObj); - return pHashObj->enableUpdate ? 0 : -1; } } @@ -532,49 +529,6 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); } -void taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || fp == NULL) { - return; - } - - // disable the resize process - taosHashRLock(pHashObj); - - int32_t numOfEntries = (int32_t)pHashObj->capacity; - for (int32_t i = 0; i < numOfEntries; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - continue; - } - - taosHashEntryWLock(pHashObj, pEntry); - - SHashNode *pPrevNode = NULL; - SHashNode *pNode = pEntry->next; - while (pNode != NULL) { - if (fp(param, GET_HASH_NODE_DATA(pNode))) { - pPrevNode = pNode; - pNode = pNode->next; - } else { - if (pPrevNode == NULL) { - pEntry->next = pNode->next; - } else { - pPrevNode->next = pNode->next; - } - pEntry->num -= 1; - atomic_sub_fetch_64(&pHashObj->size, 1); - SHashNode *next = pNode->next; - FREE_HASH_NODE(pNode); - pNode = next; - } - } - - taosHashEntryWUnlock(pHashObj, pEntry); - } - - taosHashRUnlock(pHashObj); -} - void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; @@ -897,6 +851,7 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { taosHashRUnlock(pHashObj); } +//TODO remove it void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { void* p = NULL; return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fe32afb2f4..5bc4b81be7 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -269,11 +269,12 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag SPageInfo* ppi = malloc(sizeof(SPageInfo)); ppi->pageId = pageId; - ppi->pData = NULL; + ppi->pData = NULL; ppi->offset = -1; ppi->length = -1; - ppi->used = true; - ppi->pn = NULL; + ppi->used = true; + ppi->pn = NULL; + ppi->dirty = false; return *(SPageInfo**)taosArrayPush(list, &ppi); } @@ -471,7 +472,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { return (void*)(GET_DATA_PAYLOAD(*pi)); } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0); + assert((*pi)->pData == NULL && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1))); char* availablePage = NULL; if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { @@ -493,9 +494,12 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { lruListPushFront(pBuf->lruList, *pi); (*pi)->used = true; - int32_t code = loadPageFromDisk(pBuf, *pi); - if (code != 0) { - return NULL; + // some data has been flushed to disk, and needs to be loaded into buffer again. + if ((*pi)->length > 0 && (*pi)->offset >= 0) { + int32_t code = loadPageFromDisk(pBuf, *pi); + if (code != 0) { + return NULL; + } } return (void*)(GET_DATA_PAYLOAD(*pi)); diff --git a/source/util/test/arrayTest.cpp b/source/util/test/arrayTest.cpp index 939d4a701d..d8ace09392 100644 --- a/source/util/test/arrayTest.cpp +++ b/source/util/test/arrayTest.cpp @@ -43,6 +43,9 @@ static void remove_batch_test() { taosArrayPush(delList, &a); taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); EXPECT_EQ(taosArrayGetSize(pa), 17); + + taosArrayDestroy(pa); + taosArrayDestroy(delList); } } // namespace @@ -79,4 +82,6 @@ TEST(arrayTest, array_search_test) { } } + + taosArrayDestroy(pa); } diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 5505a6207f..ec5ce756ba 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -201,8 +201,8 @@ TEST(td_encode_test, encode_decode_cstr) { } } - delete buf; - delete cstr; + delete[] buf; + delete[] cstr; } typedef struct { @@ -354,7 +354,7 @@ static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { tEndDecode(pCoder); return 0; } - +#if 0 TEST(td_encode_test, compound_struct_encode_test) { SCoder encoder, decoder; uint8_t * buf1; @@ -436,5 +436,5 @@ TEST(td_encode_test, compound_struct_encode_test) { GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b); tCoderClear(&decoder); } - +#endif #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index ac1bae2434..3856129be0 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -106,7 +106,7 @@ void noLockPerformanceTest() { ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; - int32_t num = 5000000; + int32_t num = 5000; int64_t st = taosGetTimestampUs(); @@ -186,10 +186,15 @@ void acquireRleaseTest() { printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); - + + tfree(pdata->p); + taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); + + taosHashCleanup(hashTable); + tfree(data.p); } } diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 8fbec31dd2..f392aac7d1 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -12,145 +12,150 @@ namespace { // simple test void simpleTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, "", "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/"); int32_t pageId = 0; int32_t groupId = 0; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); - ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); + ASSERT_EQ(getTotalBufSize(pBuf), 1024); - SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + SIDList list = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1); + ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); - releaseBufPage(pResultBuf, pBufPage); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); - SFilePage* t = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* t = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t == pBufPage1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage4); - SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + releaseBufPage(pBuf, pBufPage2); + + SFilePage* pBufPage5 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage5); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } void writeDownTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, "1", "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); *(int32_t*)(pBufPage->data) = nx; writePageId = pageId; - releaseBufPage(pResultBuf, pBufPage); + + setBufPageDirty(pBufPage, true); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); + releaseBufPage(pBuf, t4); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); + SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 5); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } void recyclePageTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, "1", "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); - releaseBufPage(pResultBuf, pBufPage); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); + releaseBufPage(pBuf, t4); - SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t5 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t5 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t5 == pBufPage5); ASSERT_TRUE(pageId == 5); + releaseBufPage(pBuf, t5); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); + SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); *(int32_t*)(pBufPagex->data) = nx; writePageId = pageId; // update the data - releaseBufPage(pResultBuf, pBufPagex); + releaseBufPage(pBuf, pBufPagex); - SFilePage* pBufPagex1 = static_cast(getBufPage(pResultBuf, 1)); + SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1)); - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 6); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } } // namespace From 09e6462c5fad9a5612ae64e96b93898e2fbe25a9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Mar 2022 09:09:11 +0800 Subject: [PATCH 50/82] feature/qnode --- include/common/tmsg.h | 4 + source/common/src/tmsg.c | 6 + source/dnode/mgmt/impl/src/dndVnodes.c | 3 + source/dnode/mnode/impl/src/mndVgroup.c | 3 + source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/vnd/vnodeMain.c | 3 + source/libs/catalog/inc/catalogInt.h | 39 +- source/libs/catalog/src/catalog.c | 307 ++++++++---- source/libs/catalog/test/catalogTests.cpp | 580 ++++++++++++++++++++-- 9 files changed, 812 insertions(+), 136 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ab0472a575..25d668e788 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -739,6 +739,9 @@ typedef struct { int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; + uint32_t hashBegin; + uint32_t hashEnd; + int8_t hashMethod; int8_t walLevel; int8_t precision; int8_t compression; @@ -749,6 +752,7 @@ typedef struct { int8_t selfIndex; int8_t streamMode; SReplica replicas[TSDB_MAX_REPLICA]; + } SCreateVnodeReq, SAlterVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d8c850c6af..ae31dff310 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2112,6 +2112,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1; + if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1; + if (tEncodeI8(&encoder, pReq->hashMethod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; @@ -2152,6 +2155,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->hashMethod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index f20493aa7f..2767461f12 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -523,6 +523,9 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.segSize = 128; pCfg->walCfg.vgId = pCreate->vgId; + pCfg->hashBegin = pCreate->hashBegin; + pCfg->hashEnd = pCreate->hashEnd; + pCfg->hashMethod = pCreate->hashMethod; } static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b437b44417..f7b177f170 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -215,6 +215,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.replica = pVgroup->replica; createReq.selfIndex = -1; createReq.streamMode = pVgroup->streamMode; + createReq.hashBegin = pVgroup->hashBegin; + createReq.hashEnd = pVgroup->hashEnd; + createReq.hashMethod = pDb->hashMethod; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 31f04e840a..9a4f920499 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -57,6 +57,9 @@ typedef struct { SMetaCfg metaCfg; STqCfg tqCfg; SWalCfg walCfg; + uint32_t hashBegin; + uint32_t hashEnd; + int8_t hashMethod; } SVnodeCfg; typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index c748907d6c..ba346064ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -30,6 +30,9 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { cfg.pDnode = pVnodeCfg->pDnode; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; + cfg.hashBegin = pVnodeCfg->hashBegin; + cfg.hashEnd = pVnodeCfg->hashEnd; + cfg.hashMethod = pVnodeCfg->hashMethod; } // Validate options diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index e71f559ad9..c4f1a117fe 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -60,6 +60,7 @@ typedef struct SCtgDebug { bool lockDebug; bool cacheDebug; bool apiDebug; + bool metaDebug; uint32_t showCachePeriodSec; } SCtgDebug; @@ -119,6 +120,10 @@ typedef struct SCatalogStat { SCtgCacheStat cache; } SCatalogStat; +typedef struct SCtgUpdateMsgHeader { + SCatalog* pCtg; +} SCtgUpdateMsgHeader; + typedef struct SCtgUpdateVgMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -145,6 +150,14 @@ typedef struct SCtgRemoveStbMsg { uint64_t suid; } SCtgRemoveStbMsg; +typedef struct SCtgRemoveTblMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; +} SCtgRemoveTblMsg; + + typedef struct SCtgMetaAction { int32_t act; void *data; @@ -189,19 +202,21 @@ typedef struct SCtgAction { #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) -#define CTG_FLAG_STB 0x1 -#define CTG_FLAG_NOT_STB 0x2 -#define CTG_FLAG_UNKNOWN_STB 0x4 -#define CTG_FLAG_INF_DB 0x8 +#define CTG_FLAG_STB 0x1 +#define CTG_FLAG_NOT_STB 0x2 +#define CTG_FLAG_UNKNOWN_STB 0x4 +#define CTG_FLAG_INF_DB 0x8 +#define CTG_FLAG_FORCE_UPDATE 0x10 -#define CTG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) -#define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) -#define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) -#define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) -#define CTG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) -#define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) -#define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) -#define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) +#define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) +#define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) +#define CTG_FLAG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_FLAG_IS_FORCE_UPDATE(_flag) ((_flag) & CTG_FLAG_FORCE_UPDATE) +#define CTG_FLAG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) +#define CTG_FLAG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) +#define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) +#define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) #define CTG_IS_INF_DBNAME(_dbname) ((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f0ea51c2f9..5779906761 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -72,6 +72,12 @@ int32_t ctgDbgEnableDebug(char *option) { return TSDB_CODE_SUCCESS; } + if (0 == strcasecmp(option, "meta")) { + gCTGDebug.metaDebug = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + qError("invalid debug option:%s", option); return TSDB_CODE_CTG_INTERNAL_ERROR; @@ -148,9 +154,30 @@ int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { return num; } +void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { + if (!gCTGDebug.metaDebug) { + return; + } -void ctgDbgShowDBCache(SHashObj *dbHash) { - if (NULL == dbHash) { + STableComInfo *c = &p->tableInfo; + + if (TSDB_CHILD_TABLE == p->tableType) { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + return; + } else { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); + } + + int32_t colNum = c->numOfColumns + c->numOfTags; + for (int32_t i = 0; i < colNum; ++i) { + SSchema *s = &p->schema[i]; + ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + } +} + +void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { + if (NULL == dbHash || !gCTGDebug.cacheDebug) { return; } @@ -164,31 +191,24 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { dbCache = (SCtgDBCache *)pIter; taosHashGetKey(dbCache, (void **)&dbFName, &len); - - CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); - CTG_CACHE_DEBUG("deleted: %d", dbCache->deleted); + int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; + int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; + int32_t hashMethod = -1; + int32_t vgNum = 0; + if (dbCache->vgInfo) { - CTG_CACHE_DEBUG("vgVersion: %d", dbCache->vgInfo->vgVersion); - CTG_CACHE_DEBUG("hashMethod: %d", dbCache->vgInfo->hashMethod); + vgVersion = dbCache->vgInfo->vgVersion; + hashMethod = dbCache->vgInfo->hashMethod; if (dbCache->vgInfo->vgHash) { - CTG_CACHE_DEBUG("vgNum: %d", taosHashGetSize(dbCache->vgInfo->vgHash)); - //TODO - } else { - CTG_CACHE_DEBUG("vgHash: %p", dbCache->vgInfo->vgHash); + vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); } - } else { - CTG_CACHE_DEBUG("vgInfo: %p", dbCache->vgInfo); - } - - if (dbCache->tbCache.metaCache) { - CTG_CACHE_DEBUG("metaNum: %d", taosHashGetSize(dbCache->tbCache.metaCache)); - } - - if (dbCache->tbCache.stbCache) { - CTG_CACHE_DEBUG("stbNum: %d", taosHashGetSize(dbCache->tbCache.stbCache)); - } + } + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + pIter = taosHashIterate(dbHash, pIter); } } @@ -197,15 +217,15 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (NULL == pCtg) { + if (!gCTGDebug.cacheDebug || NULL == pCtg) { return; } - CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - ctgDbgShowDBCache(pCtg->dbCache); + ctgDbgShowDBCache(pCtg, pCtg->dbCache); } @@ -292,6 +312,66 @@ _return: } +int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; + SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + + +int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL}; + SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { @@ -554,7 +634,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); @@ -562,7 +642,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); } else { tNameGetFullDbName(pTableName, dbFName); @@ -590,6 +670,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } *exist = 1; + if (dbId) { + *dbId = dbCache->dbId; + } tbMeta = *pTableMeta; @@ -646,7 +729,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); } else { tNameGetFullDbName(pTableName, dbFName); @@ -1304,18 +1387,21 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (orig) { origType = orig->tableType; - origSuid = orig->suid; - if (origType == TSDB_SUPER_TABLE && ((!isStb) || origSuid != meta->suid)) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (origType == TSDB_SUPER_TABLE) { + if ((!isStb) || orig->suid != meta->suid) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); + } + + origSuid = orig->suid; } } @@ -1334,13 +1420,14 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDbgShowTableMeta(pCtg, tbName, meta); if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - if (isStb && origSuid == meta->suid) { + if (origType == TSDB_SUPER_TABLE && origSuid == meta->suid) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; @@ -1506,7 +1593,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - if (!CTG_IS_INF_DB(flag)) { + if (!CTG_FLAG_IS_INF_DB(flag)) { CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); } @@ -1518,11 +1605,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); - } else if (CTG_IS_STB(flag)) { + } else if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode @@ -1538,14 +1625,17 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); tfree(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + } + if (0 == exist) { CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); @@ -1606,35 +1696,40 @@ _return: CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t flag) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; int32_t code = 0; + uint64_t dbId = 0; + uint64_t suid = 0; + STableMetaOutput *output = NULL; if (CTG_IS_INF_DBNAME(pTableName->dbname)) { - CTG_SET_INF_DB(flag); + CTG_FLAG_SET_INF_DB(flag); } - if ((!forceUpdate) || (CTG_IS_INF_DB(flag))) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId)); - if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { - return TSDB_CODE_SUCCESS; + int32_t tbType = 0; + + if (exist) { + if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) { + goto _return; } - tfree(*pTableMeta); - } else if (CTG_IS_UNKNOWN_STB(flag)) { - int32_t tbType = 0; - - CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType, flag)); + tbType = (*pTableMeta)->tableType; + suid = (*pTableMeta)->suid; - CTG_SET_STB(flag, tbType); + tfree(*pTableMeta); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { + CTG_FLAG_SET_STB(flag, tbType); } - STableMetaOutput *output = NULL; while (true) { CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); @@ -1662,7 +1757,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL)); if (0 == exist) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; @@ -1675,10 +1770,26 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons _return: + if (CTG_TABLE_NOT_EXIST(code) && exist) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } + + if (TSDB_SUPER_TABLE == tbType) { + ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid); + } else { + ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname); + } + } + tfree(output); if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); + ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta); } CTG_RET(code); @@ -1694,7 +1805,7 @@ int32_t ctgActUpdateVg(SCtgMetaAction *action) { _return: - tfree(msg->dbInfo); + ctgFreeVgInfo(msg->dbInfo); tfree(msg); CTG_RET(code); @@ -1780,7 +1891,6 @@ _return: int32_t ctgActRemoveStb(SCtgMetaAction *action) { int32_t code = 0; SCtgRemoveStbMsg *msg = action->data; - bool removed = false; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1826,7 +1936,36 @@ _return: } int32_t ctgActRemoveTbl(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + +_return: + + tfree(msg); + + CTG_RET(code); } @@ -1846,12 +1985,15 @@ void* ctgUpdateThreadFunc(void* param) { SCtgMetaAction *action = NULL; ctgPopAction(&action); + SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; - qDebug("process %s action", gCtgAction[action->act].name); + ctgDebug("process [%s] action", gCtgAction[action->act].name); (*gCtgAction[action->act].func)(action); CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + + ctgDbgShowClusterCache(pCtg); } CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); @@ -2121,22 +2263,20 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->dbId = dbId; msg->dbInfo = dbInfo; - dbInfo = NULL; action.data = msg; CTG_ERR_JRET(ctgPushAction(&action)); + dbInfo = NULL; + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); CTG_API_LEAVE(code); _return: - if (dbInfo) { - taosHashCleanup(dbInfo->vgHash); - tfree(dbInfo); - } + ctgFreeVgInfo(dbInfo); tfree(msg); @@ -2179,31 +2319,12 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; - SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); - CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); - msg->dbId = dbId; - msg->suid = suid; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: - tfree(action.data); - CTG_API_LEAVE(code); } @@ -2211,13 +2332,13 @@ _return: int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_STB)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2279,13 +2400,13 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_GEN_STB_FLAG(isSTable), NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL)); } int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, true, pTableMeta, CTG_GEN_STB_FLAG(isSTable))); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); } int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { @@ -2309,7 +2430,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -2441,7 +2562,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, false, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b7432429f4..b417a645be 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist, int32_t flag); + int32_t *exist, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgDbgEnableDebug(char *option); @@ -57,12 +57,14 @@ enum { CTGT_RSP_CTBMETA, CTGT_RSP_STBMETA, CTGT_RSP_MSTBMETA, + CTGT_RSP_TBMETA_NOT_EXIST, }; bool ctgTestStop = false; bool ctgTestEnableSleep = false; +bool ctgTestEnableLog = true; bool ctgTestDeadLoop = false; -int32_t ctgTestPrintNum = 200000; +int32_t ctgTestPrintNum = 10000; int32_t ctgTestMTRunSec = 5; int32_t ctgTestCurrentVgVersion = 0; @@ -74,14 +76,18 @@ int32_t ctgTestSVersion = 1; int32_t ctgTestTVersion = 1; int32_t ctgTestSuid = 2; uint64_t ctgTestDbId = 33; +uint64_t ctgTestNormalTblUid = 1; uint64_t ctgTestClusterId = 0x1; char *ctgTestDbname = "1.db1"; char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; +char *ctgTestCurrentCTableName = NULL; +char *ctgTestCurrentTableName = NULL; +char *ctgTestCurrentSTableName = NULL; -int32_t ctgTestRspFunc[10] = {0}; +int32_t ctgTestRspFunc[100] = {0}; int32_t ctgTestRspIdx = 0; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { @@ -123,6 +129,10 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { } void ctgTestInitLogFile() { + if (!ctgTestEnableLog) { + return; + } + const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -131,6 +141,8 @@ void ctgTestInitLogFile() { strcpy(tsLogDir, "/var/log/taos"); ctgDbgEnableDebug("api"); + ctgDbgEnableDebug("meta"); + ctgDbgEnableDebug("cache"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -321,7 +333,7 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = 0; - metaRsp.tuid = 0x0000000000000001; + metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -349,10 +361,15 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * tFreeSTableMetaRsp(&metaRsp); } +void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST; +} + + void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestCTablename); + strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); strcpy(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; @@ -399,7 +416,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestSTablename); + strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); strcpy(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; @@ -409,7 +426,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid; - metaRsp.tuid = ctgTestSuid; + metaRsp.tuid = ctgTestSuid++; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -511,6 +528,9 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp case CTGT_RSP_MSTBMETA: ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); break; + case CTGT_RSP_TBMETA_NOT_EXIST: + ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp); + break; default: break; } @@ -773,7 +793,7 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0); + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL); if (code || 0 == exist) { assert(0); } @@ -828,7 +848,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } -#if 0 +#if 1 TEST(tableMeta, normalTable) { @@ -860,7 +880,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.epset.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { - usleep(10000); + usleep(50000); } ctgTestSetRspTableMeta(); @@ -870,6 +890,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); @@ -880,7 +901,7 @@ TEST(tableMeta, normalTable) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -975,7 +996,7 @@ TEST(tableMeta, childTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -994,7 +1015,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tableMeta = NULL; + tfree(tableMeta); strcpy(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -1074,8 +1095,8 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1084,7 +1105,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1111,7 +1132,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1199,8 +1220,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1209,21 +1230,21 @@ TEST(tableMeta, rmStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } } - code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid - 1); ASSERT_EQ(code, 0); while (true) { int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { - usleep(10000); + usleep(50000); } else { break; } @@ -1269,8 +1290,8 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1279,7 +1300,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1299,7 +1320,7 @@ TEST(tableMeta, updateStbMeta) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(10000); + usleep(50000); } else { break; } @@ -1330,6 +1351,499 @@ TEST(tableMeta, updateStbMeta) { memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } +TEST(refreshGetMeta, normal2normal) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_TBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); +} + +TEST(refreshGetMeta, normal2notexist) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_TBMETA_NOT_EXIST; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, CTG_ERR_CODE_TABLE_NOT_EXIST); + ASSERT_TRUE(tableMeta == NULL); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); +} + + +TEST(refreshGetMeta, normal2child) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentCTableName = ctgTestTablename; + ctgTestCurrentSTableName = ctgTestSTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + +TEST(refreshGetMeta, stable2child) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentSTableName = ctgTestTablename; + ctgTestCurrentCTableName = ctgTestTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + ctgTestCurrentSTableName = ctgTestSTablename; + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + +TEST(refreshGetMeta, stable2stable) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentSTableName = ctgTestTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + + +TEST(refreshGetMeta, child2stable) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentCTableName = ctgTestTablename; + ctgTestCurrentSTableName = ctgTestSTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + ctgTestCurrentSTableName = ctgTestTablename; + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + + TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1499,11 +2013,15 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM)) { - usleep(10000); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); } - code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); @@ -1525,7 +2043,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(10000); + usleep(50000); } else { break; } @@ -1749,7 +2267,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { - usleep(10000); + usleep(50000); } code = catalogGetExpiredDBs(pCtg, &dbs, &num); From a78d7028ef51edd829922713df6c60e3fb714ecb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 10:46:48 +0800 Subject: [PATCH 51/82] new fileIO --- source/libs/sync/inc/syncRaftStore.h | 3 +- source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/src/syncRaftStore.c | 120 ++++++++++++++++++-- source/libs/sync/test/CMakeLists.txt | 14 +++ source/libs/sync/test/syncRaftStoreTest.cpp | 42 +++++++ 5 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 source/libs/sync/test/syncRaftStoreTest.cpp diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index c54f1a1c83..c480486ff0 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -35,7 +35,8 @@ typedef struct SRaftStore { SyncTerm currentTerm; SRaftId voteFor; // FileFd fd; - char path[RAFT_STORE_PATH_LEN]; + TdFilePtr pFile; + char path[RAFT_STORE_PATH_LEN]; } SRaftStore; SRaftStore *raftStoreOpen(const char *path); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index abb52b42d0..8d04b5bb26 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -193,7 +193,7 @@ static void *syncIOConsumerFunc(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 1c88be72e7..59c85c38de 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -18,21 +18,125 @@ // to complie success: FileIO interface is modified -SRaftStore *raftStoreOpen(const char *path) { return NULL; } +SRaftStore *raftStoreOpen(const char *path) { + int32_t ret; -static int32_t raftStoreInit(SRaftStore *pRaftStore) { return 0; } + SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + if (pRaftStore == NULL) { + sError("raftStoreOpen malloc error"); + return NULL; + } + memset(pRaftStore, 0, sizeof(*pRaftStore)); + snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); -int32_t raftStoreClose(SRaftStore *pRaftStore) { return 0; } + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + memset(storeBuf, 0, sizeof(storeBuf)); -int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0; } + if (!raftStoreFileExist(pRaftStore->path)) { + ret = raftStoreInit(pRaftStore); + assert(ret == 0); + } -static bool raftStoreFileExist(char *path) { return 0; } + pRaftStore->pFile = taosOpenFile(path, TD_FILE_READ | TD_FILE_WRITE); + assert(pRaftStore->pFile != NULL); -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } + int len = taosReadFile(pRaftStore->pFile, storeBuf, RAFT_STORE_BLOCK_SIZE); + assert(len == RAFT_STORE_BLOCK_SIZE); -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } + ret = raftStoreDeserialize(pRaftStore, storeBuf, len); + assert(ret == 0); -void raftStorePrint(SRaftStore *pRaftStore) {} + return pRaftStore; +} + +static int32_t raftStoreInit(SRaftStore *pRaftStore) { + assert(pRaftStore != NULL); + + pRaftStore->pFile = taosOpenFile(pRaftStore->path, TD_FILE_CTEATE | TD_FILE_WRITE); + assert(pRaftStore->pFile != NULL); + + pRaftStore->currentTerm = 0; + pRaftStore->voteFor.addr = 0; + pRaftStore->voteFor.vgId = 0; + + int32_t ret = raftStorePersist(pRaftStore); + assert(ret == 0); + + taosCloseFile(&pRaftStore->pFile); + return 0; +} + +int32_t raftStoreClose(SRaftStore *pRaftStore) { + assert(pRaftStore != NULL); + + taosCloseFile(&pRaftStore->pFile); + free(pRaftStore); + pRaftStore = NULL; + return 0; +} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { + assert(pRaftStore != NULL); + + int32_t ret; + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + assert(ret == 0); + + taosLSeekFile(pRaftStore->pFile, 0, SEEK_SET); + + ret = taosWriteFile(pRaftStore->pFile, storeBuf, sizeof(storeBuf)); + assert(ret == RAFT_STORE_BLOCK_SIZE); + + taosFsyncFile(pRaftStore->pFile); + return 0; +} + +static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { + assert(pRaftStore != NULL); + + cJSON *pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); + cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + + char *serialized = cJSON_Print(pRoot); + int len2 = strlen(serialized); + assert(len2 < len); + memset(buf, 0, len); + snprintf(buf, len, "%s", serialized); + free(serialized); + + cJSON_Delete(pRoot); + return 0; +} + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { + assert(pRaftStore != NULL); + + assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); + cJSON *pRoot = cJSON_Parse(buf); + + cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); + pRaftStore->currentTerm = pCurrentTerm->valueint; + + cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); + pRaftStore->voteFor.addr = pVoteForAddr->valueint; + + cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); + pRaftStore->voteFor.vgId = pVoteForVgid->valueint; + + cJSON_Delete(pRoot); + return 0; +} + +void raftStorePrint(SRaftStore *pRaftStore) { + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + printf("%s\n", storeBuf); +} #if 0 diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index f28bbf46cc..2913d230b2 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(syncIOTickPingTest "") add_executable(syncIOSendMsgTest "") add_executable(syncIOSendMsgClientTest "") add_executable(syncIOSendMsgServerTest "") +add_executable(syncRaftStoreTest "") target_sources(syncTest @@ -45,6 +46,10 @@ target_sources(syncIOSendMsgServerTest PRIVATE "syncIOSendMsgServerTest.cpp" ) +target_sources(syncRaftStoreTest + PRIVATE + "syncRaftStoreTest.cpp" +) target_include_directories(syncTest @@ -92,6 +97,11 @@ target_include_directories(syncIOSendMsgServerTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncRaftStoreTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -130,6 +140,10 @@ target_link_libraries(syncIOSendMsgServerTest sync gtest_main ) +target_link_libraries(syncRaftStoreTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp new file mode 100644 index 0000000000..e533b89a92 --- /dev/null +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -0,0 +1,42 @@ +#include "syncRaftStore.h" +#include +#include "gtest/gtest.h" +#include "syncIO.h" +#include "syncInt.h" + +void *pingFunc(void *param) { + SSyncIO *io = (SSyncIO *)param; + while (1) { + sDebug("io->ping"); + // io->ping(io); + sleep(1); + } + return NULL; +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + sTrace("sync log test: trace"); + sDebug("sync log test: debug"); + sInfo("sync log test: info"); + sWarn("sync log test: warn"); + sError("sync log test: error"); + sFatal("sync log test: fatal"); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + assert(pRaftStore != NULL); + + raftStorePrint(pRaftStore); + + pRaftStore->currentTerm = 100; + pRaftStore->voteFor.addr = 200; + pRaftStore->voteFor.vgId = 300; + + raftStorePrint(pRaftStore); + raftStorePersist(pRaftStore); + + return 0; +} From 7986a372e9160594ecf16cdd059a219395d16355 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 11:29:08 +0800 Subject: [PATCH 52/82] monitor --- include/libs/monitor/monitor.h | 9 +- source/dnode/mgmt/impl/inc/dndEnv.h | 3 + source/dnode/mgmt/impl/src/dndEnv.c | 2 + source/dnode/mgmt/impl/src/dndMgmt.c | 20 +++- source/libs/monitor/src/monitor.c | 157 ++++++++++++++++++++++++++- 5 files changed, 183 insertions(+), 8 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 4b7c6a9bc9..090fea9dab 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -57,7 +57,6 @@ typedef struct { typedef struct { int32_t dnode_id; - int8_t vnode_online; char vnode_role[8]; } SMonVnodeDesc; @@ -69,7 +68,7 @@ typedef struct { typedef struct { char database_name[TSDB_DB_NAME_LEN]; int32_t tables_num; - int8_t status; + char status[10]; SArray *vgroups; // array of SMonVgroupDesc } SMonVgroupInfo; @@ -107,7 +106,7 @@ typedef struct { int32_t errors; int32_t vnodes_num; int32_t masters; - int32_t has_mnode; + int8_t has_mnode; } SMonDnodeInfo; typedef struct { @@ -117,7 +116,9 @@ typedef struct { } SMonDiskDesc; typedef struct { - SArray *disks; // array of SMonDiskDesc + SArray *datadirs; // array of SMonDiskDesc + SMonDiskDesc logdir; + SMonDiskDesc tempdir; } SMonDiskInfo; typedef struct { diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index cbd5eb5827..724ba30155 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -137,6 +137,9 @@ typedef struct SDnode { SStartupReq startup; } SDnode; + +int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 2539443b41..ae08d6387c 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -323,3 +323,5 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } + +int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 4bf8b82043..afad4b87bc 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -474,11 +474,14 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -void dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { +static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->dnode_id = dndGetDnodeId(pDnode); tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); + return 0; } +static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } + static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return; SMonInfo *pMonitor = monCreateMonitorInfo(); @@ -487,8 +490,9 @@ static void dndSendMonitorReport(SDnode *pDnode) { dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); SMonBasicInfo basicInfo = {0}; - dndGetBasicInfo(pDnode, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); + if (dndGetBasicInfo(pDnode, &basicInfo) == 0) { + monSetBasicInfo(pMonitor, &basicInfo); + } SMonClusterInfo clusterInfo = {0}; SMonVgroupInfo vgroupInfo = {0}; @@ -499,6 +503,16 @@ static void dndSendMonitorReport(SDnode *pDnode) { monSetGrantInfo(pMonitor, &grantInfo); } + SMonDnodeInfo dnodeInfo = {0}; + if (dndGetDnodeInfo(pDnode, &dnodeInfo) == 0) { + monSetDnodeInfo(pMonitor, &dnodeInfo); + } + + SMonDiskInfo diskInfo = {0}; + if (dndGetDiskInfo(pDnode, &diskInfo) == 0) { + monSetDiskInfo(pMonitor, &diskInfo); + } + monSendReport(pMonitor); monCleanupMonitorInfo(pMonitor); } diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 811ee40dc8..67605432ac 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -97,21 +97,176 @@ void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { } void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { + SJson *pParentJson = pMonitor->pJson; + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pParentJson, "cluster_info", pJson) != 0) return; + tjsonAddStringToObject(pJson, "first_ep", pInfo->first_ep); + tjsonAddDoubleToObject(pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); + tjsonAddStringToObject(pJson, "version", pInfo->version); + tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); + tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); + tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); + tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); + tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); + tjsonAddDoubleToObject(pJson, "vnodes_alive", pInfo->vnodes_alive); + tjsonAddDoubleToObject(pJson, "connections_total", pInfo->connections_total); + + SJson *pDnodesJson = tjsonAddArrayToObject(pJson, "dnodes"); + if (pDnodesJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SJson *pDnodeJson = tjsonCreateObject(); + if (pDnodeJson == NULL) continue; + + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + if (tjsonAddDoubleToObject(pDnodesJson, "dnode_id", pDnodeDesc->dnode_id) != 0) tjsonDelete(pDnodeJson); + if (tjsonAddStringToObject(pDnodesJson, "dnode_ep", pDnodeDesc->dnode_ep) != 0) tjsonDelete(pDnodeJson); + if (tjsonAddStringToObject(pDnodesJson, "status", pDnodeDesc->status) != 0) tjsonDelete(pDnodeJson); + + if (tjsonAddItemToArray(pDnodesJson, pDnodeJson) != 0) tjsonDelete(pDnodeJson); + } + + SJson *pMnodesJson = tjsonAddArrayToObject(pJson, "mnodes"); + if (pMnodesJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SJson *pMnodeJson = tjsonCreateObject(); + if (pMnodeJson == NULL) continue; + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i); + if (tjsonAddDoubleToObject(pMnodesJson, "mnode_id", pMnodeDesc->mnode_id) != 0) tjsonDelete(pMnodeJson); + if (tjsonAddStringToObject(pMnodesJson, "mnode_ep", pMnodeDesc->mnode_ep) != 0) tjsonDelete(pMnodeJson); + if (tjsonAddStringToObject(pMnodesJson, "role", pMnodeDesc->role) != 0) tjsonDelete(pMnodeJson); + + if (tjsonAddItemToArray(pMnodesJson, pMnodeJson) != 0) tjsonDelete(pMnodeJson); + } } void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { + SJson *pParentJson = pMonitor->pJson; + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pParentJson, "vgroups_info", pJson) != 0) return; + tjsonAddStringToObject(pJson, "database_name", pInfo->database_name); + tjsonAddDoubleToObject(pJson, "tables_num", pInfo->tables_num); + tjsonAddStringToObject(pJson, "status", pInfo->status); + + SJson *pVgroupsJson = tjsonAddArrayToObject(pJson, "vgroups"); + if (pVgroupsJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SJson *pVgroupJson = tjsonCreateObject(); + if (pVgroupJson == NULL) continue; + + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + if (tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id) != 0) tjsonDelete(pVgroupJson); + + SJson *pVnodesJson = tjsonAddArrayToObject(pVgroupJson, "vnodes"); + if (pVnodesJson == NULL) tjsonDelete(pVgroupJson); + + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j]; + if (pVnodeDesc[j].dnode_id <= 0) continue; + + SJson *pVnodeJson = tjsonCreateObject(); + if (pVnodeJson == NULL) continue; + + if (tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id) != 0) tjsonDelete(pVnodeJson); + if (tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role) != 0) tjsonDelete(pVnodeJson); + + if (tjsonAddItemToArray(pVnodesJson, pVnodeJson) != 0) tjsonDelete(pVnodeJson); + } + } } void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { + SJson *pParentJson = pMonitor->pJson; + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pParentJson, "grant_info", pJson) != 0) return; + tjsonAddDoubleToObject(pJson, "expire_time", pInfo->expire_time); + tjsonAddDoubleToObject(pJson, "timeseries_used", pInfo->timeseries_used); + tjsonAddDoubleToObject(pJson, "timeseries_total", pInfo->timeseries_total); } void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { + SJson *pParentJson = pMonitor->pJson; + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pParentJson, "dnode_info", pJson) != 0) return; + tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime); + tjsonAddDoubleToObject(pJson, "cpu_engine", pInfo->cpu_engine); + tjsonAddDoubleToObject(pJson, "cpu_system", pInfo->cpu_system); + tjsonAddDoubleToObject(pJson, "cpu_cores", pInfo->cpu_cores); + tjsonAddDoubleToObject(pJson, "mem_engine", pInfo->mem_engine); + tjsonAddDoubleToObject(pJson, "mem_system", pInfo->mem_system); + tjsonAddDoubleToObject(pJson, "mem_total", pInfo->mem_total); + tjsonAddDoubleToObject(pJson, "disk_engine", pInfo->disk_engine); + tjsonAddDoubleToObject(pJson, "disk_used", pInfo->disk_used); + tjsonAddDoubleToObject(pJson, "disk_total", pInfo->disk_total); + tjsonAddDoubleToObject(pJson, "net_in", pInfo->net_in); + tjsonAddDoubleToObject(pJson, "net_out", pInfo->net_out); + tjsonAddDoubleToObject(pJson, "io_read", pInfo->io_read); + tjsonAddDoubleToObject(pJson, "io_write", pInfo->io_write); + tjsonAddDoubleToObject(pJson, "io_read_disk", pInfo->io_read_disk); + tjsonAddDoubleToObject(pJson, "io_write_disk", pInfo->io_write_disk); + tjsonAddDoubleToObject(pJson, "req_select", pInfo->req_select); + tjsonAddDoubleToObject(pJson, "req_select_rate", pInfo->req_select_rate); + tjsonAddDoubleToObject(pJson, "req_insert", pInfo->req_insert); + tjsonAddDoubleToObject(pJson, "req_insert_success", pInfo->req_insert_success); + tjsonAddDoubleToObject(pJson, "req_insert_rate", pInfo->req_insert_rate); + tjsonAddDoubleToObject(pJson, "req_insert_batch", pInfo->req_insert_batch); + tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pInfo->req_insert_batch_success); + tjsonAddDoubleToObject(pJson, "req_insert_batch_rate", pInfo->req_insert_batch_rate); + tjsonAddDoubleToObject(pJson, "errors", pInfo->errors); + tjsonAddDoubleToObject(pJson, "vnodes_num", pInfo->vnodes_num); + tjsonAddDoubleToObject(pJson, "masters", pInfo->masters); + tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); } void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { - + SJson *pParentJson = pMonitor->pJson; + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pParentJson, "disks_info", pJson) != 0) return; + + SJson *pDatadirsJson = tjsonAddArrayToObject(pJson, "datadirs"); + if (pDatadirsJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { + SJson *pDatadirJson = tjsonCreateObject(); + if (pDatadirJson == NULL) continue; + + SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i); + if (tjsonAddStringToObject(pDatadirJson, "name", pDatadirDesc->name) != 0) tjsonDelete(pDatadirJson); + if (tjsonAddDoubleToObject(pDatadirJson, "level", pDatadirDesc->level) != 0) tjsonDelete(pDatadirJson); + if (tjsonAddDoubleToObject(pDatadirJson, "avail", pDatadirDesc->size.avail) != 0) tjsonDelete(pDatadirJson); + if (tjsonAddDoubleToObject(pDatadirJson, "used", pDatadirDesc->size.used) != 0) tjsonDelete(pDatadirJson); + if (tjsonAddDoubleToObject(pDatadirJson, "total", pDatadirDesc->size.total) != 0) tjsonDelete(pDatadirJson); + + if (tjsonAddItemToArray(pDatadirsJson, pDatadirJson) != 0) tjsonDelete(pDatadirJson); + } + + SJson *pLogdirJson = tjsonCreateObject(); + if (pLogdirJson == NULL) return; + if (tjsonAddItemToObject(pJson, "logdir", pLogdirJson) != 0) return; + tjsonAddStringToObject(pLogdirJson, "name", pInfo->logdir.name); + tjsonAddDoubleToObject(pLogdirJson, "level", pInfo->logdir.level); + tjsonAddDoubleToObject(pLogdirJson, "avail", pInfo->logdir.size.avail); + tjsonAddDoubleToObject(pLogdirJson, "used", pInfo->logdir.size.used); + tjsonAddDoubleToObject(pLogdirJson, "total", pInfo->logdir.size.total); + + SJson *pTempdirJson = tjsonCreateObject(); + if (pTempdirJson == NULL) return; + if (tjsonAddItemToObject(pJson, "tempdir", pTempdirJson) != 0) return; + tjsonAddStringToObject(pTempdirJson, "name", pInfo->tempdir.name); + tjsonAddDoubleToObject(pTempdirJson, "level", pInfo->tempdir.level); + tjsonAddDoubleToObject(pTempdirJson, "avail", pInfo->tempdir.size.avail); + tjsonAddDoubleToObject(pTempdirJson, "used", pInfo->tempdir.size.used); + tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); } From 76c4fce8d03c0201ea71bfb5aa42c774b6ea86f9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 11:37:19 +0800 Subject: [PATCH 53/82] ping test --- source/libs/sync/src/syncIO.c | 14 ++++-------- source/libs/sync/src/syncMain.c | 10 --------- source/libs/sync/test/syncPingTest.cpp | 31 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 8d04b5bb26..3ba145a96b 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -211,23 +211,17 @@ static void *syncIOConsumerFunc(void *param) { if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { SyncPing *pSyncMsg; - - SRpcMsg tmpRpcMsg; - memcpy(&tmpRpcMsg, pRpcMsg, sizeof(SRpcMsg)); - pSyncMsg = syncPingBuild(tmpRpcMsg.contLen); - + pSyncMsg = syncPingBuild(pRpcMsg->contLen); syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); - io->FpOnSyncPing(io->pSyncNode, pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); - syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); - if (io->FpOnSyncPingReply != NULL) { + SyncPingReply *pSyncMsg; + pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); + syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); } } else { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 49fac038da..9cb3a61fff 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -171,16 +171,6 @@ static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, Syn SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - - /* - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "xxxxxxxxxxxxxx"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 1; - */ - syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); { diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 24d9ead5e3..8268128347 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -13,7 +13,9 @@ void logTest() { sFatal("--- sync log test: fatal"); } -SSyncNode* doSync() { +uint16_t ports[3] = {7010, 7110, 7210}; + +SSyncNode* doSync(int myIndex) { SSyncFSM* pFsm; SSyncInfo syncInfo; @@ -24,18 +26,18 @@ SSyncNode* doSync() { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = 0; - pCfg->replicaNum = 2; + pCfg->myIndex = myIndex; + pCfg->replicaNum = 3; - pCfg->nodeInfo[0].nodePort = 7010; + pCfg->nodeInfo[0].nodePort = ports[0]; snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - pCfg->nodeInfo[1].nodePort = 7110; + pCfg->nodeInfo[1].nodePort = ports[1]; snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - pCfg->nodeInfo[2].nodePort = 7210; + pCfg->nodeInfo[2].nodePort = ports[2]; snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); @@ -53,20 +55,25 @@ void timerPingAll(void* param, void* tmrId) { syncNodePingAll(pSyncNode); } -int main() { +int main(int argc, char** argv) { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; logTest(); - int32_t ret = syncIOStart((char*)"127.0.0.1", 7010); + int myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(); + SSyncNode* pSyncNode = doSync(myIndex); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; @@ -74,9 +81,9 @@ int main() { assert(ret == 0); /* - taosMsleep(10000); - ret = syncNodeStopPingTimer(pSyncNode); - assert(ret == 0); + taosMsleep(10000); + ret = syncNodeStopPingTimer(pSyncNode); + assert(ret == 0); */ while (1) { From aec7bef25b05afb69696533bafe00e7b53210c0c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 3 Mar 2022 12:29:54 +0800 Subject: [PATCH 54/82] add UT --- source/libs/index/src/index_tfile.c | 3 +++ source/libs/index/test/fstTest.cc | 23 +++++++++++++---------- source/libs/index/test/jsonUT.cc | 22 ++++++++++++++++++---- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a2089e8eee..0947c796b2 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -275,6 +275,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { __compar_fn_t fn; int8_t colType = tw->header.colType; + colType = INDEX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { fn = tfileStrCompare; } else { @@ -572,6 +573,8 @@ static int tfileWriteHeader(TFileWriter* writer) { static int tfileWriteData(TFileWriter* write, TFileValue* tval) { TFileHeader* header = &write->header; uint8_t colType = header->colType; + + colType = INDEX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); if (fstBuilderInsert(write->fb, key, tval->offset)) { diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 65118a2bce..74dcec4490 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -312,15 +312,18 @@ void validateTFile(char* arg) { tfCleanup(); } -void iterTFileReader(char* path, char* ver) { +void iterTFileReader(char* path, char* uid, char* colName, char* ver) { tfInit(); - int version = atoi(ver); - TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); - Iterate* iter = tfileIteratorCreate(reader); - bool tn = iter ? iter->next(iter) : false; - int count = 0; - int termCount = 0; + uint64_t suid = atoi(uid); + int version = atoi(ver); + + TFileReader* reader = tfileReaderOpen(path, suid, version, colName); + + Iterate* iter = tfileIteratorCreate(reader); + bool tn = iter ? iter->next(iter) : false; + int count = 0; + int termCount = 0; while (tn == true) { count++; IterateValue* cv = iter->getValue(iter); @@ -337,9 +340,9 @@ void iterTFileReader(char* path, char* ver) { int main(int argc, char* argv[]) { // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } - if (argc > 2) { - // opt - iterTFileReader(argv[1], argv[2]); + if (argc > 4) { + // path suid colName ver + iterTFileReader(argv[1], argv[2], argv[3], argv[4]); } // checkFstCheckIterator(); // checkFstLongTerm(); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e184bc49ae..e5c79d137f 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -22,7 +22,7 @@ class JsonEnv : public ::testing::Test { virtual void SetUp() { taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); - + printf("set up\n"); opts = indexOptsCreate(); int ret = tIndexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); @@ -30,6 +30,7 @@ class JsonEnv : public ::testing::Test { virtual void TearDown() { tIndexJsonClose(index); indexOptsDestroy(opts); + printf("destory\n"); } SIndexJsonOpts* opts; SIndexJson* index; @@ -37,7 +38,7 @@ class JsonEnv : public ::testing::Test { TEST_F(JsonEnv, testWrite) { { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -76,7 +77,7 @@ TEST_F(JsonEnv, testWrite) { indexMultiTermDestroy(terms); } { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); @@ -91,6 +92,19 @@ TEST_F(JsonEnv, testWrite) { } } TEST_F(JsonEnv, testWriteMillonData) { + { + std::string colName("test"); + std::string colVal("ab"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); @@ -105,7 +119,7 @@ TEST_F(JsonEnv, testWriteMillonData) { indexMultiTermDestroy(terms); } { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); From db0c47cc1f76e808a0af7881997de9ec8e8e6455 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 3 Mar 2022 12:39:27 +0800 Subject: [PATCH 55/82] Feature/td 11463 3.0 - block-wise SMA (#10510) * Block-wise SMA extraction * refactor the SBlock * add method tsdbLoadBlockOffset * set method tsdbLoadBlockOffset static * refactor * trigger CI * minor change * trigger CI --- include/common/taosdef.h | 4 + source/dnode/vnode/inc/tsdb.h | 10 +- source/dnode/vnode/src/inc/tsdbFS.h | 2 - source/dnode/vnode/src/inc/tsdbFile.h | 47 ++++++- source/dnode/vnode/src/inc/tsdbReadImpl.h | 132 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbCommit.c | 142 ++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbCompact.c | 2 + source/dnode/vnode/src/tsdb/tsdbFS.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 45 ++++--- source/dnode/vnode/src/tsdb/tsdbMain.c | 7 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 101 +++++++++++++-- source/util/src/terror.c | 1 + 13 files changed, 411 insertions(+), 94 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 66cb0bb4ba..69c2618ac8 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -51,6 +51,10 @@ typedef enum { } ECheckItemType; typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; +typedef enum { + TSDB_STATIS_OK = 0, // statis part exist and load successfully + TSDB_STATIS_NONE = 1, // statis part not exist +} ETsdbStatisStatus; extern char *qtypeStr[]; diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 677ca9c336..6bc89ddd66 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -27,12 +27,12 @@ extern "C" { typedef struct SDataStatis { int16_t colId; - int64_t sum; - int64_t max; - int64_t min; int16_t maxIndex; int16_t minIndex; int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; } SDataStatis; typedef struct STable { @@ -53,6 +53,8 @@ typedef struct STsdb STsdb; typedef struct STsdbCfg { int8_t precision; + int8_t update; + int8_t compression; uint64_t lruCacheSize; int32_t daysPerFile; int32_t minRowsPerFileBlock; @@ -60,8 +62,6 @@ typedef struct STsdbCfg { int32_t keep; int32_t keep1; int32_t keep2; - int8_t update; - int8_t compression; } STsdbCfg; // query condition to build multi-table data block iterator diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index af432aa9d9..dab697ce8b 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -18,8 +18,6 @@ #include "tsdbFile.h" -#define TSDB_FS_VERSION 0 - // ================== TSDB global config extern bool tsdbForceKeepFile; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 15c8d512d6..bbeb6c6a0f 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -44,7 +44,37 @@ #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; +typedef enum { + TSDB_FILE_HEAD = 0, // .head + TSDB_FILE_DATA, // .data + TSDB_FILE_LAST, // .last + TSDB_FILE_SMAD, // .smad(Block-wise SMA) + TSDB_FILE_SMAL, // .smal(Block-wise SMA) + TSDB_FILE_MAX, // + TSDB_FILE_META, // meta + TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA + TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA +} TSDB_FILE_T; + +typedef enum { + TSDB_FS_VER_0 = 0, + TSDB_FS_VER_MAX, +} ETsdbFsVer; + +#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile +#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file + +static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile + switch (fType) { + case TSDB_FILE_HEAD: // .head + case TSDB_FILE_DATA: // .data + case TSDB_FILE_LAST: // .last + case TSDB_FILE_SMAD: // .smad(Block-wise SMA) + case TSDB_FILE_SMAL: // .smal(Block-wise SMA) + default: + return TSDB_LATEST_FVER; + } +} #if 0 // =============== SMFile @@ -169,6 +199,7 @@ static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nby // =============== SDFile typedef struct { uint32_t magic; + uint32_t fver; uint32_t len; uint32_t totalBlocks; uint32_t totalSubBlocks; @@ -188,7 +219,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); int tsdbEncodeSDFile(void** buf, SDFile* pDFile); void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); -int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader); +int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType); int tsdbUpdateDFileHeader(SDFile* pDFile); int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); @@ -292,12 +323,18 @@ static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { // =============== SDFileSet typedef struct { - int fid; - int state; - SDFile files[TSDB_FILE_MAX]; + int fid; + int8_t state; // -128~127 + uint8_t ver; // 0~255, DFileSet version + uint16_t reserve; + SDFile files[TSDB_FILE_MAX]; } SDFileSet; +#define TSDB_LATEST_FSET_VER 0 + #define TSDB_FSET_FID(s) ((s)->fid) +#define TSDB_FSET_STATE(s) ((s)->state) +#define TSDB_FSET_VER(s) ((s)->ver) #define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) #define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) #define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 8ecb4e4a6e..16eb55967c 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -36,6 +36,7 @@ typedef struct { TSKEY maxKey; } SBlockIdx; +#ifdef TD_REFACTOR_3 typedef struct { int64_t last : 1; int64_t offset : 63; @@ -49,22 +50,35 @@ typedef struct { TSKEY keyLast; } SBlock; +#else + +typedef enum { + TSDB_SBLK_VER_0 = 0, + TSDB_SBLK_VER_MAX, +} ESBlockVer; + +#define SBlockVerLatest TSDB_SBLK_VER_0 + typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - uint8_t reserve0; + uint8_t last : 1; + uint8_t blkVer : 7; uint8_t numOfSubBlocks; int16_t numOfCols; // not including timestamp column - uint32_t len : 32; // data block length + uint32_t len; // data block length uint32_t keyLen : 24; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - uint32_t reserve1 : 8; - uint64_t blkVer : 8; - uint64_t aggrOffset : 56; + uint32_t reserve : 8; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int64_t offset; + uint64_t aggrStat : 1; + uint64_t aggrOffset : 63; TSKEY keyFirst; TSKEY keyLast; -} SBlock_3; +} SBlockV0; + +#define SBlock SBlockV0 // latest SBlock definition + +#endif typedef struct { int32_t delimiter; // For recovery usage @@ -73,6 +87,7 @@ typedef struct { SBlock blocks[]; } SBlockInfo; +#ifdef TD_REFACTOR_3 typedef struct { int16_t colId; uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM @@ -89,17 +104,50 @@ typedef struct { uint8_t offsetH; char padding[1]; } SBlockCol; +#else +typedef struct { + int16_t colId; + uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint8_t reserve : 7; + uint8_t type; + int32_t len; + uint32_t offset; +} SBlockColV0; + +#define SBlockCol SBlockColV0 // latest SBlockCol definition + +typedef struct { + int16_t colId; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; +} SAggrBlkColV0; + +#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition + +#endif // Code here just for back-ward compatibility static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { +#ifdef TD_REFACTOR_3 pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); pBlockCol->offsetH = (uint8_t)(offset >> 24); +#else + pBlockCol->offset = offset; +#endif } static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { +#ifdef TD_REFACTOR_3 uint32_t offset1 = pBlockCol->offset; uint32_t offset2 = pBlockCol->offsetH; return (offset1 | (offset2 << 24)); +#else + return pBlockCol->offset; +#endif } typedef struct { @@ -109,31 +157,57 @@ typedef struct { SBlockCol cols[]; } SBlockData; +typedef void SAggrBlkData; // SBlockCol cols[]; + struct SReadH { - STsdb * pRepo; - SDFileSet rSet; // FSET to read - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; // table to read - SBlockIdx * pBlkIdx; // current reading table SBlockIdx - int cidx; - SBlockInfo *pBlkInfo; - SBlockData *pBlkData; // Block info - SDataCols * pDCols[2]; - void * pBuf; // buffer - void * pCBuf; // compression buffer + STsdb * pRepo; + SDFileSet rSet; // FSET to read + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; // table to read + SBlockIdx * pBlkIdx; // current reading table SBlockIdx + int cidx; + SBlockInfo * pBlkInfo; + SBlockData * pBlkData; // Block info + SAggrBlkData *pAggrBlkData; // Aggregate Block info + SDataCols * pDCols[2]; + void * pBuf; // buffer + void * pCBuf; // compression buffer + void * pExBuf; // extra buffer }; -#define TSDB_READ_REPO(rh) ((rh)->pRepo) -#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) -#define TSDB_READ_FSET(rh) (&((rh)->rSet)) -#define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_REPO(rh) ((rh)->pRepo) +#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) +#define TSDB_READ_FSET(rh) (&((rh)->rSet)) +#define TSDB_READ_TABLE(rh) ((rh)->pTable) #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) -#define TSDB_READ_BUF(rh) ((rh)->pBuf) -#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) +#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD) +#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL) +#define TSDB_READ_BUF(rh) ((rh)->pBuf) +#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) +#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) -#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ + (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_STATIS_SIZE(nCols, 0); + } +} + +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_AGGR_SIZE(nCols, 0); + } +} int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo); void tsdbDestroyReadH(SReadH *pReadh); @@ -147,7 +221,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); -void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols); +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { void * pBuf = *ppBuf; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f7e4d56fe2..2e7116969e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -50,8 +50,11 @@ typedef struct { #define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) #define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) #define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD) +#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL) #define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) #define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh)) #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) @@ -509,7 +512,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // TSDB_FILE_HEAD SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWHeadf, true, TSDB_FILE_HEAD) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tstrerror(terrno)); @@ -560,7 +563,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); pCommith->isLFileSame = false; - if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWLastf, true, TSDB_FILE_LAST) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); @@ -572,6 +575,74 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid } } } + + // TSDB_FILE_SMAD + SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh)); + SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); + + if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) { + tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); + tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); + + if (tsdbCreateDFile(pRepo, pWSmadF, true, TSDB_FILE_SMAD) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFileEx(pWSmadF, pRSmadF); + if (tsdbOpenDFile(pWSmadF, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + + // TSDB_FILE_SMAL + SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh)); + SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith); + + if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) { + tsdbInitDFileEx(pWSmalF, pRSmalF); + if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmalF)); + tsdbInitDFile(pRepo, pWSmalF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAL); + + if (tsdbCreateDFile(pRepo, pWSmalF, true, TSDB_FILE_SMAL) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } } return 0; @@ -1131,41 +1202,57 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { } } -int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - SBlockData *pBlockData; - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; +int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, + SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData * pBlockData = NULL; + SAggrBlkData *pAggrBlkData = NULL; + int64_t offset = 0, offsetAggr = 0; + int rowsToWrite = pDataCols->numOfRows; ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); // Make buffer space - if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + if (tsdbMakeRoom(ppBuf, tsdbBlockStatisSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); + if (tsdbMakeRoom(ppExBuf, tsdbBlockAggrSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) { + return -1; + } + pAggrBlkData = (SAggrBlkData *)(*ppExBuf); + // Get # of cols not all NULL(not including key column) int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol *pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol * pBlockCol = pBlockData->cols + nColsNotAllNull; + SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } memset(pBlockCol, 0, sizeof(*pBlockCol)); + memset(pAggrBlkCol, 0, sizeof(*pAggrBlkCol)); pBlockCol->colId = pDataCol->colId; pBlockCol->type = pDataCol->type; + pAggrBlkCol->colId = pDataCol->colId; + if (tDataTypes[pDataCol->type].statisFunc) { +#if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->numOfNull)); - if (pBlockCol->numOfNull == 0) { +#endif + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), + &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), + &(pAggrBlkCol->numOfNull)); + + if (pAggrBlkCol->numOfNull == 0) { TD_SET_COL_ROWS_NORM(pBlockCol); } else { TD_SET_COL_ROWS_MISC(pBlockCol); @@ -1181,13 +1268,14 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // Compress the data if neccessary int tcol = 0; // counter of not all NULL and written columns uint32_t toffset = 0; - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest); int32_t lsize = tsize; + uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); int32_t tBitmaps = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish if (ncol != 0 && tcol >= nColsNotAllNull) break; @@ -1248,7 +1336,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); pBlockCol->len = flen; - tcol++; + ++tcol; } else { keyLen = flen; } @@ -1269,6 +1357,18 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * return -1; } + uint32_t aggrStatus = nColsNotAllNull > 0 ? 1 : 0; + if (aggrStatus > 0) { + + taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr); + tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFileAggr, (void *)pAggrBlkData, tsizeAggr, &offsetAggr) < tsizeAggr) { + return -1; + } + } + // Update pBlock membership vairables pBlock->last = isLast; pBlock->offset = offset; @@ -1280,6 +1380,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->numOfCols = nColsNotAllNull; pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); + pBlock->aggrStat = aggrStatus; + pBlock->blkVer = SBlockVerLatest; + pBlock->aggrOffset = (uint64_t)offsetAggr; tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, @@ -1291,9 +1394,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper) { - return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, - isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), - (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); + return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, + isLast ? TSDB_COMMIT_SMAL_FILE(pCommith) : TSDB_COMMIT_SMAD_FILE(pCommith), pDataCols, + pBlock, isLast, isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), + (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith))), (void **)(&(TSDB_COMMIT_EXBUF(pCommith)))); } static int tsdbWriteBlockInfo(SCommitH *pCommih) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index 80b59d8756..e24d5974af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -38,6 +38,8 @@ typedef struct { #define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD) #define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA) #define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST) +#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD) +#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL) #define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) #define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 24c765d3e5..dcc9700d83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -422,7 +422,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { return -1; } - fsheader.version = TSDB_FS_VERSION; + fsheader.version = TSDB_LATEST_SFS_VER; if (taosArrayGetSize(pStatus->df) == 0) { fsheader.len = 0; } else { @@ -697,7 +697,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { ptr = tsdbDecodeFSHeader(ptr, &fsheader); ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta)); - if (fsheader.version != TSDB_FS_VERSION) { + if (fsheader.version != TSDB_LATEST_SFS_VER) { // TODO: handle file version change } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 5a00fa20a0..4d70c429ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -19,8 +19,12 @@ static const char *TSDB_FNAME_SUFFIX[] = { "head", // TSDB_FILE_HEAD "data", // TSDB_FILE_DATA "last", // TSDB_FILE_LAST + "smad", // TSDB_FILE_SMAD + "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META + "tsma", // TSDB_FILE_TSMA + "rsma", // TSDB_FILE_RSMA }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); @@ -304,6 +308,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t memset(&(pDFile->info), 0, sizeof(pDFile->info)); pDFile->info.magic = TSDB_FILE_INIT_MAGIC; + pDFile->info.fver = tsdbGetDFSVersion(ftype); tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname); tfsInitFile(pRepo->pTfs, &(pDFile->f), did, fname); @@ -341,7 +346,7 @@ static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) { } static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { - char *aname; + char *aname = NULL; buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); buf = taosDecodeString(buf, &aname); @@ -352,7 +357,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { return buf; } -int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { +int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); @@ -382,6 +387,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } pDFile->info.size += TSDB_FILE_HEAD_SIZE; + pDFile->info.fver = tsdbGetDFSVersion(fType); if (tsdbUpdateDFileHeader(pDFile) < 0) { tsdbCloseDFile(pDFile); @@ -493,6 +499,7 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { int tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); + tlen += taosEncodeFixedU32(buf, pInfo->fver); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); @@ -505,6 +512,7 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + buf = taosDecodeFixedU32(buf, &(pInfo->fver)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); @@ -562,8 +570,10 @@ static int tsdbRollBackDFile(SDFile *pDFile) { // ============== Operations on SDFileSet void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) { - pSet->fid = fid; - pSet->state = 0; + TSDB_FSET_FID(pSet) = fid; + TSDB_FSET_VER(pSet) = TSDB_LATEST_FSET_VER; + TSDB_FSET_STATE(pSet) = 0; + pSet->reserve = 0; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); @@ -572,7 +582,7 @@ void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint3 } void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { - pSet->fid = pOSet->fid; + TSDB_FSET_FID(pSet) = TSDB_FSET_FID(pOSet); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); } @@ -581,7 +591,10 @@ void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pSet->fid); + tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet)); + // state not included + tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet)); + tlen += taosEncodeFixedU16(buf, pSet->reserve); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -590,11 +603,11 @@ int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { } void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { - int32_t fid; + buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet))); + TSDB_FSET_STATE(pSet) = 0; + buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet))); + buf = taosDecodeFixedU16(buf, &(pSet->reserve)); - buf = taosDecodeFixedI32(buf, &(fid)); - pSet->state = 0; - pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -604,7 +617,9 @@ void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pSet->fid); + tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet)); + tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet)); + tlen += taosEncodeFixedU16(buf, pSet->reserve); for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -613,10 +628,10 @@ int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { } void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) { - int32_t fid; + buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet))); + buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet))); + buf = taosDecodeFixedU16(buf, &(pSet->reserve)); - buf = taosDecodeFixedI32(buf, &(fid)); - pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } @@ -637,7 +652,7 @@ int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { + if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader, ftype) < 0) { tsdbCloseDFileSet(pSet); tsdbRemoveDFileSet(pSet); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 812ec67ec4..2d8c470113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -821,9 +821,10 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // file block with sub-blocks has no statistics data if (pBlock->numOfSubBlocks <= 1) { - tsdbLoadBlockStatis(pReadh, pBlock); - tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); - loadStatisData = true; + if (tsdbLoadBlockStatis(pReadh, pBlock) == TSDB_STATIS_OK) { + tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns, pBlock); + loadStatisData = true; + } } for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c1cf776cb5..4bd0ee959e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3276,8 +3276,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati } int64_t stime = taosGetTimestampUs(); - if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { + int statisStatus = tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock); + if (statisStatus < TSDB_STATIS_OK) { return terrno; + } else if (statisStatus > TSDB_STATIS_OK) { + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; } int16_t* colIds = pHandle->defaultLoadColumn->pData; @@ -3288,7 +3292,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati pHandle->statis[i].colId = colIds[i]; } - tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); + tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols, pBlockInfo->compBlock); // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 7e8e7866d7..f6827eaae1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -19,6 +19,7 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); +static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, @@ -63,10 +64,12 @@ int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) { void tsdbDestroyReadH(SReadH *pReadh) { if (pReadh == NULL) return; + pReadh->pExBuf = taosTZfree(pReadh->pExBuf); pReadh->pCBuf = taosTZfree(pReadh->pCBuf); pReadh->pBuf = taosTZfree(pReadh->pBuf); pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]); pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]); + pReadh->pAggrBlkData = taosTZfree(pReadh->pAggrBlkData); pReadh->pBlkData = taosTZfree(pReadh->pBlkData); pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo); pReadh->cidx = 0; @@ -305,15 +308,58 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); - SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + if (!pBlock->aggrStat) { + return TSDB_STATIS_NONE; + } + SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh); + + if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, + tstrerror(terrno)); + return -1; + } + + size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); + if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1; + + int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); + if (nreadAggr < 0) { + tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno), + (uint64_t)pBlock->aggrOffset, sizeAggr); + return -1; + } + + if (nreadAggr < sizeAggr) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr, + nreadAggr); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr); + return -1; + } + return 0; +} + +static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { + ASSERT(pBlock->numOfSubBlocks <= 1); + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); return -1; } - size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); @@ -337,7 +383,6 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); return -1; } - return 0; } @@ -375,13 +420,14 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { return buf; } -void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock) { +#ifdef TD_REFACTOR_3 SBlockData *pBlockData = pReadh->pBlkData; for (int i = 0, j = 0; i < numOfCols;) { if (j >= pBlockData->numOfCols) { pStatis[i].numOfNull = -1; - i++; + ++i; continue; } @@ -392,15 +438,45 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { pStatis[i].maxIndex = pBlockData->cols[j].maxIndex; pStatis[i].minIndex = pBlockData->cols[j].minIndex; pStatis[i].numOfNull = pBlockData->cols[j].numOfNull; - i++; - j++; + ++i; + ++j; } else if (pStatis[i].colId < pBlockData->cols[j].colId) { pStatis[i].numOfNull = -1; - i++; + ++i; } else { - j++; + ++j; } } +#else + if (pBlock->aggrStat) { + SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData; + + for (int i = 0, j = 0; i < numOfCols;) { + if (j >= pBlock->numOfCols) { + pStatis[i].numOfNull = -1; + ++i; + continue; + } + SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j; + if (pStatis[i].colId == pAggrBlkCol->colId) { + pStatis[i].sum = pAggrBlkCol->sum; + pStatis[i].max = pAggrBlkCol->max; + pStatis[i].min = pAggrBlkCol->min; + pStatis[i].maxIndex = pAggrBlkCol->maxIndex; + pStatis[i].minIndex = pAggrBlkCol->minIndex; + pStatis[i].numOfNull = pAggrBlkCol->numOfNull; + ++i; + ++j; + } else if (pStatis[i].colId < pAggrBlkCol->colId) { + pStatis[i].numOfNull = -1; + ++i; + } else { + ++j; + } + } + } + +#endif } static void tsdbResetReadTable(SReadH *pReadh) { @@ -449,7 +525,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return -1; } - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", @@ -592,7 +668,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); // If only load timestamp column, no need to load SBlockData part - if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1; + if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1; pDataCols->numOfRows = pBlock->numOfRows; @@ -686,7 +762,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; - int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol); + int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + + tsdbGetBlockColOffset(pBlockCol); if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c8cafb93dd..176290ba38 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -345,6 +345,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information t TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") From 1fe0c21fd22ab64be6c6afa75c0c49cf191c9ea8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 13:48:13 +0800 Subject: [PATCH 56/82] monitor --- source/dnode/mgmt/impl/src/dndMgmt.c | 6 +- source/libs/monitor/src/monitor.c | 118 +++++++++++++++++++++------ 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index afad4b87bc..268ed7e362 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -483,12 +483,12 @@ static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } static void dndSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return; + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); + SMonInfo *pMonitor = monCreateMonitorInfo(); if (pMonitor == NULL) return; - dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); - SMonBasicInfo basicInfo = {0}; if (dndGetBasicInfo(pDnode, &basicInfo) == 0) { monSetBasicInfo(pMonitor, &basicInfo); diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 67605432ac..7ba65b87b0 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -54,7 +54,10 @@ void monAddLogItem(SMonLogItem *pItem) { SMonInfo *monCreateMonitorInfo() { SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo)); - if (pMonitor == NULL) return NULL; + if (pMonitor == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } taosWLockLatch(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); @@ -77,30 +80,24 @@ void monCleanupMonitorInfo(SMonInfo *pMonitor) { free(pMonitor); } -void monSendReport(SMonInfo *pMonitor) { - char *pCont = tjsonToString(pMonitor->pJson); - if (pCont != NULL) { - taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont)); - free(pCont); - } -} - void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { - SJson *pJson = pMonitor->pJson; - tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); - tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); - + SJson *pJson = pMonitor->pJson; int64_t ms = taosGetTimestampMs(); char buf[40] = {0}; taosFormatUtcTime(buf, sizeof(buf), ms, TSDB_TIME_PRECISION_MILLI); + tjsonAddStringToObject(pJson, "ts", buf); + tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); + tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); } void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { - SJson *pParentJson = pMonitor->pJson; SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pParentJson, "cluster_info", pJson) != 0) return; + if (tjsonAddItemToObject(pMonitor->pJson, "cluster_info", pJson) != 0) { + tjsonDelete(pJson); + return; + } tjsonAddStringToObject(pJson, "first_ep", pInfo->first_ep); tjsonAddDoubleToObject(pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); @@ -145,10 +142,12 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { } void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { - SJson *pParentJson = pMonitor->pJson; SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pParentJson, "vgroups_info", pJson) != 0) return; + if (tjsonAddItemToObject(pMonitor->pJson, "vgroup_infos", pJson) != 0) { + tjsonDelete(pJson); + return; + } tjsonAddStringToObject(pJson, "database_name", pInfo->database_name); tjsonAddDoubleToObject(pJson, "tables_num", pInfo->tables_num); @@ -183,10 +182,12 @@ void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { } void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { - SJson *pParentJson = pMonitor->pJson; SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pParentJson, "grant_info", pJson) != 0) return; + if (tjsonAddItemToObject(pMonitor->pJson, "grant_info", pJson) != 0) { + tjsonDelete(pJson); + return; + } tjsonAddDoubleToObject(pJson, "expire_time", pInfo->expire_time); tjsonAddDoubleToObject(pJson, "timeseries_used", pInfo->timeseries_used); @@ -194,10 +195,12 @@ void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { } void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { - SJson *pParentJson = pMonitor->pJson; SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pParentJson, "dnode_info", pJson) != 0) return; + if (tjsonAddItemToObject(pMonitor->pJson, "dnode_info", pJson) != 0) { + tjsonDelete(pJson); + return; + } tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime); tjsonAddDoubleToObject(pJson, "cpu_engine", pInfo->cpu_engine); @@ -230,12 +233,14 @@ void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { } void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { - SJson *pParentJson = pMonitor->pJson; SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pParentJson, "disks_info", pJson) != 0) return; + if (tjsonAddItemToObject(pMonitor->pJson, "disks_infos", pJson) != 0) { + tjsonDelete(pJson); + return; + } - SJson *pDatadirsJson = tjsonAddArrayToObject(pJson, "datadirs"); + SJson *pDatadirsJson = tjsonAddArrayToObject(pJson, "datadir"); if (pDatadirsJson == NULL) return; for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { @@ -270,3 +275,68 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { tjsonAddDoubleToObject(pTempdirJson, "used", pInfo->tempdir.size.used); tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); } + +static void monSetLogInfo(SMonInfo *pMonitor) { + SJson *pJson = tjsonCreateObject(); + if (pJson == NULL) return; + if (tjsonAddItemToObject(pMonitor->pJson, "log_infos", pJson) != 0) { + tjsonDelete(pJson); + return; + } + + SJson *pLogsJson = tjsonAddArrayToObject(pJson, "logs"); + if (pLogsJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pMonitor->logs); ++i) { + SJson *pLogJson = tjsonCreateObject(); + if (pLogJson == NULL) continue; + + SMonLogItem *pLogItem = taosArrayGet(pMonitor->logs, i); + + char buf[40] = {0}; + taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); + + if (tjsonAddStringToObject(pLogItem, "ts", buf) != 0) tjsonDelete(pLogJson); + if (tjsonAddDoubleToObject(pLogItem, "level", pLogItem->level) != 0) tjsonDelete(pLogJson); + if (tjsonAddStringToObject(pLogItem, "content", pLogItem->content) != 0) tjsonDelete(pLogJson); + + if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); + } + + SJson *pSummaryJson = tjsonAddArrayToObject(pJson, "summary"); + if (pSummaryJson == NULL) return; + + SJson *pLogError = tjsonCreateObject(); + if (pLogError == NULL) return; + tjsonAddStringToObject(pLogError, "level", "error"); + tjsonAddDoubleToObject(pLogError, "total", 1); + if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError); + + SJson *pLogInfo = tjsonCreateObject(); + if (pLogInfo == NULL) return; + tjsonAddStringToObject(pLogInfo, "level", "info"); + tjsonAddDoubleToObject(pLogInfo, "total", 1); + if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo); + + SJson *pLogDebug = tjsonCreateObject(); + if (pLogDebug == NULL) return; + tjsonAddStringToObject(pLogDebug, "level", "debug"); + tjsonAddDoubleToObject(pLogDebug, "total", 1); + if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug); + + SJson *pLogTrace = tjsonCreateObject(); + if (pLogTrace == NULL) return; + tjsonAddStringToObject(pLogTrace, "level", "trace"); + tjsonAddDoubleToObject(pLogTrace, "total", 1); + if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); +} + +void monSendReport(SMonInfo *pMonitor) { + monSetLogInfo(pMonitor); + + char *pCont = tjsonToString(pMonitor->pJson); + if (pCont != NULL) { + taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont)); + free(pCont); + } +} From f263a623dd7186525c28bb79668426158404a71e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 14:52:30 +0800 Subject: [PATCH 57/82] sync encode/decode --- source/libs/sync/inc/syncMessage.h | 162 ++++++++------- source/libs/sync/src/syncMessage.c | 245 ++++++++++++++++------- source/libs/sync/test/syncEncodeTest.cpp | 129 +++++++++--- 3 files changed, 368 insertions(+), 168 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index be53559e8a..a51567d1dd 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -28,30 +28,25 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" -// encode as uint64 +// encode as uint32 typedef enum ESyncMessageType { SYNC_PING = 101, SYNC_PING_REPLY = 103, - SYNC_CLIENT_REQUEST, - SYNC_CLIENT_REQUEST_REPLY, - SYNC_REQUEST_VOTE, - SYNC_REQUEST_VOTE_REPLY, - SYNC_APPEND_ENTRIES, - SYNC_APPEND_ENTRIES_REPLY, + SYNC_CLIENT_REQUEST = 105, + SYNC_CLIENT_REQUEST_REPLY = 107, + SYNC_REQUEST_VOTE = 109, + SYNC_REQUEST_VOTE_REPLY = 111, + SYNC_APPEND_ENTRIES = 113, + SYNC_APPEND_ENTRIES_REPLY = 115, } ESyncMessageType; -/* -typedef struct SRaftId { - SyncNodeId addr; // typedef uint64_t SyncNodeId; - SyncGroupId vgId; // typedef int32_t SyncGroupId; -} SRaftId; -*/ - +// --------------------------------------------- typedef struct SyncPing { uint32_t bytes; uint32_t msgType; SRaftId srcId; SRaftId destId; + // private data uint32_t dataLen; char data[]; } SyncPing; @@ -59,28 +54,22 @@ typedef struct SyncPing { #define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) SyncPing* syncPingBuild(uint32_t dataLen); - -void syncPingDestroy(SyncPing* pMsg); - -void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); - -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); - -void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); - -void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); - -cJSON* syncPing2Json(const SyncPing* pMsg); - +void syncPingDestroy(SyncPing* pMsg); +void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); +cJSON* syncPing2Json(const SyncPing* pMsg); SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); - SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId); +// --------------------------------------------- typedef struct SyncPingReply { uint32_t bytes; uint32_t msgType; SRaftId srcId; SRaftId destId; + // private data uint32_t dataLen; char data[]; } SyncPingReply; @@ -89,72 +78,95 @@ typedef struct SyncPingReply { (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) SyncPingReply* syncPingReplyBuild(uint32_t dataLen); - -void syncPingReplyDestroy(SyncPingReply* pMsg); - -void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); - -void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); - -void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); - -void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); - -cJSON* syncPingReply2Json(const SyncPingReply* pMsg); - +void syncPingReplyDestroy(SyncPingReply* pMsg); +void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); +void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); +void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); +void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); +cJSON* syncPingReply2Json(const SyncPingReply* pMsg); SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); - SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId); +// --------------------------------------------- typedef struct SyncClientRequest { - ESyncMessageType msgType; - char* data; - uint32_t dataLen; - int64_t seqNum; - bool isWeak; + uint32_t bytes; + uint32_t msgType; + int64_t seqNum; + bool isWeak; + uint32_t dataLen; + char data[]; } SyncClientRequest; +// --------------------------------------------- typedef struct SyncClientRequestReply { - ESyncMessageType msgType; - int32_t errCode; - SSyncBuffer* pErrMsg; - SSyncBuffer* pLeaderHint; + uint32_t bytes; + uint32_t msgType; + int32_t errCode; + SRaftId leaderHint; } SyncClientRequestReply; +// --------------------------------------------- typedef struct SyncRequestVote { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncGroupId vgId; - SyncIndex lastLogIndex; - SyncTerm lastLogTerm; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm currentTerm; + SyncIndex lastLogIndex; + SyncTerm lastLogTerm; } SyncRequestVote; +SyncRequestVote* syncRequestVoteBuild(); +void syncRequestVoteDestroy(SyncRequestVote* pMsg); +void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg); +void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg); +cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg); + +// --------------------------------------------- typedef struct SyncRequestVoteReply { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncGroupId vgId; - bool voteGranted; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + bool voteGranted; } SyncRequestVoteReply; +SyncRequestVoteReply* SyncRequestVoteReplyBuild(); +void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg); +void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg); +void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg); +cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg); + +// --------------------------------------------- typedef struct SyncAppendEntries { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncIndex prevLogIndex; - SyncTerm prevLogTerm; - int32_t entryCount; - SSyncRaftEntry* logEntries; - SyncIndex commitIndex; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + SyncIndex commitIndex; + uint32_t dataLen; + char data[]; } SyncAppendEntries; +// --------------------------------------------- typedef struct SyncAppendEntriesReply { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - bool success; - SyncIndex matchIndex; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + bool success; + SyncIndex matchIndex; } SyncAppendEntriesReply; #ifdef __cplusplus diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index a26c8401b2..ac3f550e3e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -60,12 +60,15 @@ void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg) { } cJSON* syncPing2Json(const SyncPing* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; cJSON* pTmp = pSrcId; @@ -79,7 +82,8 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; cJSON* pTmp = pDestId; @@ -154,12 +158,15 @@ void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg) { } cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; cJSON* pTmp = pSrcId; @@ -173,7 +180,8 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; cJSON* pTmp = pDestId; @@ -208,72 +216,169 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) return pMsg; } -#if 0 -void syncPingSerialize(const SyncPing* pMsg, char** ppBuf, uint32_t* bufLen) { - *bufLen = sizeof(SyncPing) + pMsg->dataLen; - *ppBuf = (char*)malloc(*bufLen); - void* pStart = *ppBuf; - uint32_t allBytes = *bufLen; - - int len = 0; - len = taosEncodeFixedU32(&pStart, pMsg->msgType); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU64(&pStart, pMsg->srcId.addr); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedI32(&pStart, pMsg->srcId.vgId); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU64(&pStart, pMsg->destId.addr); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedI32(&pStart, pMsg->destId.vgId); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU32(&pStart, pMsg->dataLen); - allBytes -= len; - assert(len > 0); - pStart += len; - - memcpy(pStart, pMsg->data, pMsg->dataLen); - allBytes -= pMsg->dataLen; - assert(allBytes == 0); +// ---- message process SyncRequestVote---- +SyncRequestVote* syncRequestVoteBuild() { + uint32_t bytes = sizeof(SyncRequestVote); + SyncRequestVote* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_REQUEST_VOTE; } - -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { - void* pStart = (void*)buf; - uint64_t u64; - int32_t i32; - uint32_t u32; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->msgType = u64; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->srcId.addr = u64; - - pStart = taosDecodeFixedI32(pStart, &i32); - pMsg->srcId.vgId = i32; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->destId.addr = u64; - - pStart = taosDecodeFixedI32(pStart, &i32); - pMsg->destId.vgId = i32; - - pStart = taosDecodeFixedU32(pStart, &u32); - pMsg->dataLen = u32; +void syncRequestVoteDestroy(SyncRequestVote* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } } -#endif \ No newline at end of file + +void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncRequestVoteSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg) { + syncRequestVoteDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm); + cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); + cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncRequestVote", pRoot); + return pJson; +} + +// ---- message process SyncRequestVoteReply---- +SyncRequestVoteReply* SyncRequestVoteReplyBuild() { + uint32_t bytes = sizeof(SyncRequestVoteReply); + SyncRequestVoteReply* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; +} + +void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncRequestVoteReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg) { + syncRequestVoteReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot); + return pJson; +} \ No newline at end of file diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 4deceb603e..e0179b2c8b 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -3,6 +3,7 @@ #include "syncIO.h" #include "syncInt.h" #include "syncMessage.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -21,16 +22,16 @@ void test1() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); - pMsg->srcId.addr = 1; - pMsg->srcId.vgId = 2; - pMsg->destId.addr = 3; - pMsg->destId.vgId = 4; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPing: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -45,7 +46,7 @@ void test1() { { cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPing2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -61,16 +62,16 @@ void test2() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "hello raft"); SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); - pMsg->srcId.addr = 100; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 3333); pMsg->srcId.vgId = 200; - pMsg->destId.addr = 300; - pMsg->destId.vgId = 400; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 4444); + pMsg->destId.vgId = 200; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPing: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -84,7 +85,7 @@ void test2() { { cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPing2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -99,16 +100,16 @@ void test3() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); - pMsg->srcId.addr = 19; - pMsg->srcId.vgId = 29; - pMsg->destId.addr = 39; - pMsg->destId.vgId = 49; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 5555); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 6666); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPingReply2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -123,7 +124,7 @@ void test3() { { cJSON* pJson = syncPingReply2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -139,16 +140,16 @@ void test4() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "hello raft"); SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); - pMsg->srcId.addr = 66; - pMsg->srcId.vgId = 77; - pMsg->destId.addr = 88; - pMsg->destId.vgId = 99; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 7777); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 8888); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPingReply2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -162,7 +163,7 @@ void test4() { { cJSON* pJson = syncPingReply2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -170,6 +171,86 @@ void test4() { syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } + +void test5() { + sTrace("test5: ---- syncRequestVoteSerialize, syncRequestVoteDeserialize"); + + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->currentTerm = 20; + pMsg->lastLogIndex = 21; + pMsg->lastLogTerm = 22; + + { + cJSON* pJson = syncRequestVote2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncRequestVoteSerialize(pMsg, buf, bufLen); + + SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes); + syncRequestVoteDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncRequestVote2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteDestroy(pMsg); + syncRequestVoteDestroy(pMsg2); + free(buf); +} + +void test6() { + sTrace("test6: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); + + SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 20; + pMsg->voteGranted = 1; + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncRequestVoteReplySerialize(pMsg, buf, bufLen); + + SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes); + syncRequestVoteReplyDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteReplyDestroy(pMsg); + syncRequestVoteReplyDestroy(pMsg2); + free(buf); +} + int main() { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; @@ -179,6 +260,8 @@ int main() { test2(); test3(); test4(); + test5(); + test6(); return 0; } From c572b9cad254265f4cf4905f809905323c41bd2c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Mar 2022 15:07:33 +0800 Subject: [PATCH 58/82] feature/qnode --- include/libs/scheduler/scheduler.h | 10 +- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 12 +- source/libs/scheduler/inc/schedulerInt.h | 15 +- source/libs/scheduler/src/scheduler.c | 283 ++++++++++-------- source/libs/scheduler/test/schedulerTests.cpp | 120 +++++--- 6 files changed, 244 insertions(+), 198 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index e856adaf31..b2080cb655 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -23,8 +23,6 @@ extern "C" { #include "catalog.h" #include "planner.h" -struct SSchJob; - typedef struct SSchedulerCfg { uint32_t maxJobNum; } SSchedulerCfg; @@ -72,7 +70,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +78,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob); /** * Fetch query result from the remote query executor @@ -88,7 +86,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDa * @param data * @return */ -int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); +int32_t schedulerFetchRows(int64_t job, void **data); /** @@ -102,7 +100,7 @@ int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); * Free the query job * @param pJob */ -void schedulerFreeJob(void *pJob); +void schedulerFreeJob(int64_t job); void schedulerDestroy(void); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 79909a5696..c93ea1aabe 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,7 +171,7 @@ typedef struct SRequestSendRecvBody { void* fp; SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + int64_t queryJob; // query job, created according to sql query DAG. struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3c6a49c3db..a0ba668f8a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -227,10 +227,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); } pRequest->code = code; @@ -240,8 +240,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); } } @@ -494,7 +494,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void**)&pResInfo->pData); + int32_t code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 42270cd645..50c274ad48 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -36,6 +36,11 @@ enum { SCH_WRITE, }; +typedef struct SSchTrans { + void *transInst; + void *transHandle; +} SSchTrans; + typedef struct SSchApiStat { } SSchApiStat; @@ -59,12 +64,13 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; - SHashObj *jobs; // key: queryId, value: SQueryJob* + int32_t jobRef; SSchedulerStat stat; } SSchedulerMgmt; typedef struct SSchCallbackParam { uint64_t queryId; + int64_t refId; uint64_t taskId; } SSchCallbackParam; @@ -75,7 +81,8 @@ typedef struct SSchLevel { int32_t taskFailed; int32_t taskSucceed; int32_t taskNum; - SArray *subTasks; // Element is SQueryTask + int32_t taskLaunchIdx; // launch startup index + SArray *subTasks; // Element is SQueryTask } SSchLevel; typedef struct SSchTask { @@ -105,6 +112,7 @@ typedef struct SSchJobAttr { } SSchJobAttr; typedef struct SSchJob { + int64_t refId; uint64_t queryId; SSchJobAttr attr; int32_t levelNum; @@ -119,7 +127,6 @@ typedef struct SSchJob { SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* - int32_t ref; int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; @@ -168,6 +175,8 @@ typedef struct SSchJob { static int32_t schLaunchTask(SSchJob *job, SSchTask *task); static int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); +SSchJob *schAcquireJob(int64_t refId); +int32_t schReleaseJob(int64_t refId); #ifdef __cplusplus } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f14b95873a..f1ed0cef7d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -17,12 +17,17 @@ #include "tmsg.h" #include "query.h" #include "catalog.h" +#include "tref.h" -typedef struct SSchTrans { - void *transInst; - void *transHandle; -}SSchTrans; -static SSchedulerMgmt schMgmt = {0}; +SSchedulerMgmt schMgmt = {0}; + +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { + return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); +} + +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { + return taosReleaseRef(schMgmt.jobRef, refId); +} uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); @@ -886,7 +891,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } case TDMT_VND_DROP_TASK_RSP: { // SHOULD NEVER REACH HERE - SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref)); + SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } @@ -908,28 +913,23 @@ _return: int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchJob *pJob = NULL; SSchTask *pTask = NULL; - SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); - if (NULL == job || NULL == (*job)) { - qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, pParam->refId); + if (NULL == pJob) { + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } - pJob = *job; - - atomic_add_fetch_32(&pJob->ref, 1); - int32_t s = taosHashGetSize(pJob->execTasks); if (s <= 0) { - qError("QID:%"PRIx64",TID:%"PRId64" no task in execTask list", pParam->queryId, pParam->taskId); + SCH_JOB_ELOG("empty execTask list, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { - qError("QID:%"PRIx64",TID:%"PRId64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); + SCH_JOB_ELOG("task not found in execTask list, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -942,7 +942,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in _return: if (pJob) { - atomic_sub_fetch_32(&pJob->ref, 1); + taosReleaseRef(schMgmt.jobRef, pParam->refId); } tfree(param); @@ -1003,28 +1003,29 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { } -int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); if (NULL == param) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - param->queryId = qId; - param->taskId = tId; + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = pTask->taskId; pMsgSendInfo->param = param; @@ -1040,7 +1041,7 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t SCH_ERR_JRET(code); } - qDebug("QID:0x%"PRIx64 ",TID:0x%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); + SCH_TASK_DLOG("req msg sent, refId:%" PRIx64 ", type:%d, %s", pJob->refId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: @@ -1160,7 +1161,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, atomic_store_32(&pTask->lastMsgType, msgType); SSchTrans trans = {.transInst = pJob->transport, .transHandle = pTask->handle}; - SCH_ERR_JRET(schAsyncSendMsg(&trans, &epSet, pJob->queryId, pTask->taskId, msgType, msg, msgSize)); + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize)); if (isCandidateAddr) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); @@ -1283,7 +1284,60 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, const char* sql, bool syncSchedule) { + +int32_t schCancelJob(SSchJob *pJob) { + //TODO + + //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + +} + +void schFreeJobImpl(void *job) { + if (NULL == job) { + return; + } + + SSchJob *pJob = job; + uint64_t queryId = pJob->queryId; + int64_t refId = pJob->refId; + + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { + schCancelJob(pJob); + } + + schDropJobAllTasks(pJob); + + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + + int32_t numOfLevels = taosArrayGetSize(pJob->levels); + for(int32_t i = 0; i < numOfLevels; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); + for(int32_t j = 0; j < numOfTasks; ++j) { + SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); + schFreeTask(pTask); + } + + taosArrayDestroy(pLevel->subTasks); + } + + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); + + taosArrayDestroy(pJob->levels); + taosArrayDestroy(pJob->nodeList); + + tfree(pJob->res); + + tfree(pJob); + + qDebug("QID:0x%"PRIx64" job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); +} + + +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, int64_t *job, const char* sql, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { @@ -1327,21 +1381,20 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDa tsem_init(&pJob->rspSem, 0, 0); - code = taosHashPut(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId), &pJob, POINTER_BYTES); - if (0 != code) { - if (HASH_NODE_EXIST(code)) { - SCH_JOB_ELOG("job already exist, isQueryJob:%d", pJob->attr.queryJob); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } else { - SCH_JOB_ELOG("taosHashPut job failed, errno:%d", errno); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + pJob->refId = taosAddRef(schMgmt.jobRef, pJob); + if (pJob->refId < 0) { + SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); } + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); - *(SSchJob **)job = pJob; + taosAcquireRef(schMgmt.jobRef, pJob->refId); + + *job = pJob->refId; if (syncSchedule) { SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); @@ -1349,25 +1402,20 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDa } SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); + + taosReleaseRef(schMgmt.jobRef, pJob->refId); + return TSDB_CODE_SUCCESS; _return: - *(SSchJob **)job = NULL; - schedulerFreeJob(pJob); + schFreeJobImpl(pJob); SCH_RET(code); } -int32_t schCancelJob(SSchJob *pJob) { - //TODO - - //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST - -} - int32_t schedulerInit(SSchedulerCfg *cfg) { - if (schMgmt.jobs) { + if (schMgmt.jobRef) { qError("scheduler already initialized"); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -1381,9 +1429,9 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } else { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } - - schMgmt.jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == schMgmt.jobs) { + + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); + if (schMgmt.jobRef < 0) { qError("init schduler jobs failed, num:%u", schMgmt.cfg.maxJobNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1398,24 +1446,28 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); - pRes->code = atomic_load_32(&(*pJob)->errCode); - pRes->numOfRows = (*pJob)->resNumOfRows; + + SSchJob *job = taosAcquireRef(schMgmt.jobRef, *pJob); + pRes->code = atomic_load_32(&job->errCode); + pRes->numOfRows = job->resNumOfRows; + taosReleaseRef(schMgmt.jobRef, *pJob); return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); + return TSDB_CODE_SUCCESS; } @@ -1541,28 +1593,35 @@ _return: } -int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { - if (NULL == pJob || NULL == pData) { +int32_t schedulerFetchRows(int64_t job, void** pData) { + if (NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - atomic_add_fetch_32(&pJob->ref, 1); + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } int8_t status = SCH_GET_JOB_STATUS(pJob); if (status == JOB_TASK_STATUS_DROPPING) { SCH_JOB_ELOG("job is dropping, status:%d", status); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { @@ -1588,7 +1647,6 @@ int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } -_return: while (true) { *pData = atomic_load_ptr(&pJob->res); @@ -1609,96 +1667,47 @@ _return: SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, totalRows:%d, code:%s", pJob->resNumOfRows, tstrerror(code)); - atomic_sub_fetch_32(&pJob->ref, 1); + +_return: + + atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); + + taosReleaseRef(schMgmt.jobRef, job); SCH_RET(code); } -int32_t scheduleCancelJob(void *job) { - SSchJob *pJob = (SSchJob *)job; - - atomic_add_fetch_32(&pJob->ref, 1); +int32_t scheduleCancelJob(int64_t job) { + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } int32_t code = schCancelJob(pJob); - atomic_sub_fetch_32(&pJob->ref, 1); + taosReleaseRef(schMgmt.jobRef, job); SCH_RET(code); } -void schedulerFreeJob(void *job) { - if (NULL == job) { +void schedulerFreeJob(int64_t job) { + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); return; } - SSchJob *pJob = job; - uint64_t queryId = pJob->queryId; - bool setJobFree = false; - - if (SCH_GET_JOB_STATUS(pJob) > 0) { - if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { - SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); - return; - } - - SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); - - while (true) { - int32_t ref = atomic_load_32(&pJob->ref); - if (0 == ref) { - break; - } else if (ref > 0) { - if (1 == ref && atomic_load_8(&pJob->userFetch) > 0 && !setJobFree) { - schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); - setJobFree = true; - } - - usleep(1); - } else { - SCH_JOB_ELOG("invalid job ref number, ref:%d", ref); - break; - } - } - - SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); - - if (pJob->status == JOB_TASK_STATUS_EXECUTING) { - schCancelJob(pJob); - } - - schDropJobAllTasks(pJob); + if (atomic_load_8(&pJob->userFetch) > 0) { + schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); } - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - - int32_t numOfLevels = taosArrayGetSize(pJob->levels); - for(int32_t i = 0; i < numOfLevels; ++i) { - SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + SCH_JOB_DLOG("start to remove job from jobRef list, refId:%" PRIx64, job); - int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); - schFreeTask(pTask); - } - - taosArrayDestroy(pLevel->subTasks); + if (taosRemoveRef(schMgmt.jobRef, job)) { + SCH_JOB_ELOG("remove job from job list failed, refId:%" PRIx64, job); } - - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); - taosArrayDestroy(pJob->nodeList); - - tfree(pJob->res); - - tfree(pJob); - - qDebug("QID:0x%"PRIx64" job freed", queryId); } void schedulerFreeTaskList(SArray *taskList) { @@ -1716,9 +1725,17 @@ void schedulerFreeTaskList(SArray *taskList) { } void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); //TODO - schMgmt.jobs = NULL; + if (schMgmt.jobRef) { + SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); + + while (pJob) { + taosRemoveRef(schMgmt.jobRef, pJob->refId); + + pJob = taosIterateRef(schMgmt.jobRef, pJob->refId); + } + + taosCloseRef(schMgmt.jobRef); + schMgmt.jobRef = 0; } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 89d365a7e7..11ed3335e6 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,15 +38,15 @@ #include "schedulerInt.h" #include "stub.h" #include "addr_any.h" - +#include "tref.h" namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode); -struct SSchJob *pInsertJob = NULL; -struct SSchJob *pQueryJob = NULL; +int64_t insertJobRefId = 0; +int64_t queryJobRefId = 0; uint64_t schtMergeTemplateId = 0x4; uint64_t schtFetchTaskId = 0; @@ -65,6 +65,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -255,34 +256,40 @@ void schtSetAsyncSendMsgToServer() { void *schtSendRsp(void *param) { - SSchJob *job = NULL; + SSchJob *pJob = NULL; + int64_t job = 0; int32_t code = 0; while (true) { - job = *(SSchJob **)param; + job = *(int64_t *)param; if (job) { break; } usleep(1000); } + + pJob = schAcquireJob(job); - void *pIter = taosHashIterate(job->execTasks, NULL); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SSubmitRsp rsp = {0}; rsp.affectedRows = 10; - schHandleResponseMsg(job, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); + schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } + schReleaseJob(job); + return NULL; } void *schtCreateFetchRspThread(void *param) { - struct SSchJob* job = (struct SSchJob*)param; + int64_t job = *(int64_t *)param; + SSchJob* pJob = schAcquireJob(job); sleep(1); @@ -291,8 +298,10 @@ void *schtCreateFetchRspThread(void *param) { rsp->completed = 1; rsp->numOfRows = 10; - code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); - + code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); + + schReleaseJob(job); + assert(code == 0); } @@ -329,9 +338,9 @@ void *schtFetchRspThread(void *aa) { void schtFreeQueryJob(int32_t freeThread) { static uint32_t freeNum = 0; - SSchJob *job = atomic_load_ptr(&pQueryJob); + int64_t job = queryJobRefId; - if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { + if (job && atomic_val_compare_exchange_64(&queryJobRefId, job, 0)) { schedulerFreeJob(job); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { @@ -360,7 +369,7 @@ void* schtRunJobThread(void *aa) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - SSchJob *job = NULL; + SSchJob *pJob = NULL; SSchCallbackParam *param = NULL; SHashObj *execTasks = NULL; SDataBuf dataBuf = {0}; @@ -376,24 +385,29 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &queryJobRefId); assert(code == 0); + pJob = schAcquireJob(queryJobRefId); + if (NULL == pJob) { + taosArrayDestroy(qnodeList); + schtFreeQueryDag(&dag); + continue; + } + execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - void *pIter = taosHashIterate(job->execTasks, NULL); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; - - pQueryJob = job; - + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -412,8 +426,9 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; - + param->refId = queryJobRefId; + param->queryId = pJob->queryId; + pIter = taosHashIterate(execTasks, NULL); while (pIter) { SSchTask *task = (SSchTask *)pIter; @@ -431,7 +446,8 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -450,7 +466,8 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -470,7 +487,7 @@ void* schtRunJobThread(void *aa) { atomic_store_32(&schtStartFetch, 1); void *data = NULL; - code = schedulerFetchRows(pQueryJob, &data); + code = schedulerFetchRows(queryJobRefId, &data); assert(code == 0 || code); if (0 == code) { @@ -480,12 +497,13 @@ void* schtRunJobThread(void *aa) { } data = NULL; - code = schedulerFetchRows(pQueryJob, &data); + code = schedulerFetchRows(queryJobRefId, &data); assert(code == 0 || code); schtFreeQueryJob(0); taosHashCleanup(execTasks); + taosArrayDestroy(qnodeList); schtFreeQueryDag(&dag); @@ -516,7 +534,7 @@ TEST(queryTest, normalCase) { char *dbname = "1.db1"; char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SSchJob *pJob = NULL; + int64_t job = 0; SQueryDag dag = {0}; schtInitLogFile(); @@ -537,59 +555,61 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); ASSERT_EQ(code, 0); - SSchJob *job = (SSchJob *)pJob; - void *pIter = taosHashIterate(job->execTasks, NULL); + + SSchJob *pJob = schAcquireJob(job); + + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); printf("code:%d", code); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); + pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -603,9 +623,11 @@ TEST(queryTest, normalCase) { data = NULL; code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); - ASSERT_TRUE(data); + ASSERT_TRUE(data == NULL); - schedulerFreeJob(pJob); + schReleaseJob(job); + + schedulerFreeJob(job); schtFreeQueryDag(&dag); @@ -644,14 +666,14 @@ TEST(insertTest, normalCase) { pthread_attr_init(&thattr); pthread_t thread1; - pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); + pthread_create(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, "insert into tb values(now,1)", &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); - schedulerFreeJob(pInsertJob); + schedulerFreeJob(insertJobRefId); schedulerDestroy(); } @@ -684,4 +706,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop From aeb94af5e0b382e2ca29db907de5c6c78c649bf3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 16:15:18 +0800 Subject: [PATCH 59/82] sync encode/decode --- source/libs/sync/inc/syncMessage.h | 20 ++ source/libs/sync/src/syncMessage.c | 176 ++++++++++++++++ source/libs/sync/test/syncEncodeTest.cpp | 244 ++++++++++++++++++++++- 3 files changed, 439 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index a51567d1dd..3057e23bc2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -158,6 +158,18 @@ typedef struct SyncAppendEntries { char data[]; } SyncAppendEntries; +#define SYNC_APPEND_ENTRIES_FIX_LEN \ + (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(SyncIndex) + sizeof(SyncTerm) + \ + sizeof(SyncIndex) + sizeof(uint32_t)) + +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen); +void syncAppendEntriesDestroy(SyncAppendEntries* pMsg); +void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg); +void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg); +cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg); + // --------------------------------------------- typedef struct SyncAppendEntriesReply { uint32_t bytes; @@ -169,6 +181,14 @@ typedef struct SyncAppendEntriesReply { SyncIndex matchIndex; } SyncAppendEntriesReply; +SyncAppendEntriesReply* syncAppendEntriesReplyBuild(); +void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg); +cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index ac3f550e3e..4c44b4691c 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -381,4 +381,180 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot); return pJson; +} + +// ---- message process SyncAppendEntries---- +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { + uint32_t bytes = SYNC_APPEND_ENTRIES_FIX_LEN + dataLen; + SyncAppendEntries* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_APPEND_ENTRIES; + pMsg->dataLen = dataLen; +} + +void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == SYNC_APPEND_ENTRIES_FIX_LEN + pMsg->dataLen); +} + +void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncAppendEntriesSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg) { + syncAppendEntriesDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", pMsg->data); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntries", pRoot); + return pJson; +} + +// ---- message process SyncAppendEntriesReply---- +SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { + uint32_t bytes = sizeof(SyncAppendEntriesReply); + SyncAppendEntriesReply* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; +} + +void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncAppendEntriesReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg) { + syncAppendEntriesReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + cJSON_AddNumberToObject(pRoot, "success", pMsg->success); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + cJSON_AddStringToObject(pRoot, "match_index", u64buf); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot); + return pJson; } \ No newline at end of file diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index e0179b2c8b..6197621051 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -15,6 +15,7 @@ void logTest() { } #define PING_MSG_LEN 20 +#define APPEND_ENTRIES_VALUE_LEN 32 void test1() { sTrace("test1: ---- syncPingSerialize, syncPingDeserialize"); @@ -213,7 +214,45 @@ void test5() { } void test6() { - sTrace("test6: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); + sTrace("test6: ---- syncRequestVote2RpcMsg, syncRequestVoteFromRpcMsg"); + + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->currentTerm = 20; + pMsg->lastLogIndex = 21; + pMsg->lastLogTerm = 22; + + { + cJSON* pJson = syncRequestVote2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncRequestVote2RpcMsg(pMsg, &rpcMsg); + SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes); + syncRequestVoteFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncRequestVote2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteDestroy(pMsg); + syncRequestVoteDestroy(pMsg2); +} + +void test7() { + sTrace("test7: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); @@ -251,6 +290,203 @@ void test6() { free(buf); } +void test8() { + sTrace("test8: ---- syncRequestVoteReply2RpcMsg, syncRequestVoteReplyFromRpcMsg"); + + SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 20; + pMsg->voteGranted = 1; + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); + SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes); + syncRequestVoteReplyFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteReplyDestroy(pMsg); + syncRequestVoteReplyDestroy(pMsg2); +} + +void test9() { + sTrace("test9: ---- syncAppendEntriesSerialize, syncAppendEntriesDeserialize"); + + char msg[APPEND_ENTRIES_VALUE_LEN]; + snprintf(msg, sizeof(msg), "%s", "test value"); + SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->prevLogIndex = 55; + pMsg->prevLogTerm = 66; + pMsg->commitIndex = 77; + memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncAppendEntriesSerialize(pMsg, buf, bufLen); + + SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes); + syncAppendEntriesDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesDestroy(pMsg); + syncAppendEntriesDestroy(pMsg2); + free(buf); +} + +void test10() { + sTrace("test10: ---- syncAppendEntries2RpcMsg, syncAppendEntriesFromRpcMsg"); + + char msg[APPEND_ENTRIES_VALUE_LEN]; + snprintf(msg, sizeof(msg), "%s", "test value"); + SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->prevLogIndex = 55; + pMsg->prevLogTerm = 66; + pMsg->commitIndex = 77; + memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncAppendEntries2RpcMsg(pMsg, &rpcMsg); + SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes); + syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesDestroy(pMsg); + syncAppendEntriesDestroy(pMsg2); +} + +void test11() { + sTrace("test11: ---- syncAppendEntriesReplySerialize, syncAppendEntriesReplyDeserialize"); + + SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->success = 1; + pMsg->matchIndex = 23; + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncAppendEntriesReplySerialize(pMsg, buf, bufLen); + + SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes); + syncAppendEntriesReplyDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesReplyDestroy(pMsg); + syncAppendEntriesReplyDestroy(pMsg2); + free(buf); +} + +void test12() { + sTrace("test12: ---- syncAppendEntriesReply2RpcMsg, syncAppendEntriesReplyFromRpcMsg"); + + SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->success = 1; + pMsg->matchIndex = 23; + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); + SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes); + syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesReplyDestroy(pMsg); + syncAppendEntriesReplyDestroy(pMsg2); +} + int main() { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; @@ -262,6 +498,12 @@ int main() { test4(); test5(); test6(); + test7(); + test8(); + test9(); + test10(); + test11(); + test12(); return 0; } From cc82966e2e2281a89e3436c47b6e6bfc92f3f044 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 16:37:37 +0800 Subject: [PATCH 60/82] monitor --- include/libs/monitor/monitor.h | 21 ++- source/libs/monitor/src/monitor.c | 56 +++--- source/libs/monitor/test/monTest.cpp | 255 ++++++++++++++++++++++++++- 3 files changed, 286 insertions(+), 46 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 090fea9dab..b3b6091b95 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -23,6 +23,11 @@ extern "C" { #endif +#define MON_STATUS_LEN 8 +#define MON_ROLE_LEN 9 +#define MON_VER_LEN 12 +#define MON_LOG_LEN 1024 + typedef struct { int32_t dnode_id; char dnode_ep[TSDB_EP_LEN]; @@ -31,19 +36,19 @@ typedef struct { typedef struct { int32_t dnode_id; char dnode_ep[TSDB_EP_LEN]; - char status[8]; + char status[MON_STATUS_LEN]; } SMonDnodeDesc; typedef struct { int32_t mnode_id; char mnode_ep[TSDB_EP_LEN]; - char role[8]; + char role[MON_ROLE_LEN]; } SMonMnodeDesc; typedef struct { char first_ep[TSDB_EP_LEN]; int32_t first_ep_dnode_id; - char version[12]; + char version[MON_VER_LEN]; float master_uptime; // day int32_t monitor_interval; // sec int32_t vgroups_total; @@ -57,18 +62,18 @@ typedef struct { typedef struct { int32_t dnode_id; - char vnode_role[8]; + char vnode_role[MON_ROLE_LEN]; } SMonVnodeDesc; typedef struct { int32_t vgroup_id; + char database_name[TSDB_DB_NAME_LEN]; + int32_t tables_num; + char status[MON_STATUS_LEN]; SMonVnodeDesc vnodes[TSDB_MAX_REPLICA]; } SMonVgroupDesc; typedef struct { - char database_name[TSDB_DB_NAME_LEN]; - int32_t tables_num; - char status[10]; SArray *vgroups; // array of SMonVgroupDesc } SMonVgroupInfo; @@ -124,7 +129,7 @@ typedef struct { typedef struct { int64_t ts; int8_t level; - char content[1024]; + char content[MON_LOG_LEN]; } SMonLogItem; typedef struct SMonInfo SMonInfo; diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 7ba65b87b0..ecf9da218b 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -23,7 +23,7 @@ static SMonitor tsMonitor = {0}; int32_t monInit(const SMonCfg *pCfg) { - tsMonitor.logs = taosArrayInit(16, sizeof(SMonInfo)); + tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem)); if (tsMonitor.logs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -44,7 +44,7 @@ void monCleanup() { void monAddLogItem(SMonLogItem *pItem) { taosWLockLatch(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); - if (size > tsMonitor.maxLogs) { + if (size >= tsMonitor.maxLogs) { uInfo("too many logs for monitor"); } else { taosArrayPush(tsMonitor.logs, pItem); @@ -118,9 +118,9 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { if (pDnodeJson == NULL) continue; SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); - if (tjsonAddDoubleToObject(pDnodesJson, "dnode_id", pDnodeDesc->dnode_id) != 0) tjsonDelete(pDnodeJson); - if (tjsonAddStringToObject(pDnodesJson, "dnode_ep", pDnodeDesc->dnode_ep) != 0) tjsonDelete(pDnodeJson); - if (tjsonAddStringToObject(pDnodesJson, "status", pDnodeDesc->status) != 0) tjsonDelete(pDnodeJson); + tjsonAddDoubleToObject(pDnodeJson, "dnode_id", pDnodeDesc->dnode_id); + tjsonAddStringToObject(pDnodeJson, "dnode_ep", pDnodeDesc->dnode_ep); + tjsonAddStringToObject(pDnodeJson, "status", pDnodeDesc->status); if (tjsonAddItemToArray(pDnodesJson, pDnodeJson) != 0) tjsonDelete(pDnodeJson); } @@ -133,48 +133,44 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { if (pMnodeJson == NULL) continue; SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i); - if (tjsonAddDoubleToObject(pMnodesJson, "mnode_id", pMnodeDesc->mnode_id) != 0) tjsonDelete(pMnodeJson); - if (tjsonAddStringToObject(pMnodesJson, "mnode_ep", pMnodeDesc->mnode_ep) != 0) tjsonDelete(pMnodeJson); - if (tjsonAddStringToObject(pMnodesJson, "role", pMnodeDesc->role) != 0) tjsonDelete(pMnodeJson); + tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id); + tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep); + tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role); if (tjsonAddItemToArray(pMnodesJson, pMnodeJson) != 0) tjsonDelete(pMnodeJson); } } void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { - SJson *pJson = tjsonCreateObject(); + SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "vgroup_infos"); if (pJson == NULL) return; - if (tjsonAddItemToObject(pMonitor->pJson, "vgroup_infos", pJson) != 0) { - tjsonDelete(pJson); - return; - } - - tjsonAddStringToObject(pJson, "database_name", pInfo->database_name); - tjsonAddDoubleToObject(pJson, "tables_num", pInfo->tables_num); - tjsonAddStringToObject(pJson, "status", pInfo->status); - - SJson *pVgroupsJson = tjsonAddArrayToObject(pJson, "vgroups"); - if (pVgroupsJson == NULL) return; for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { SJson *pVgroupJson = tjsonCreateObject(); if (pVgroupJson == NULL) continue; + if (tjsonAddItemToArray(pJson, pVgroupJson) != 0) { + tjsonDelete(pVgroupJson); + continue; + } SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); - if (tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id) != 0) tjsonDelete(pVgroupJson); + tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id); + tjsonAddStringToObject(pVgroupJson, "database_name", pVgroupDesc->database_name); + tjsonAddDoubleToObject(pVgroupJson, "tables_num", pVgroupDesc->tables_num); + tjsonAddStringToObject(pVgroupJson, "status", pVgroupDesc->status); SJson *pVnodesJson = tjsonAddArrayToObject(pVgroupJson, "vnodes"); - if (pVnodesJson == NULL) tjsonDelete(pVgroupJson); + if (pVnodesJson == NULL) continue; for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j]; - if (pVnodeDesc[j].dnode_id <= 0) continue; + if (pVnodeDesc->dnode_id <= 0) continue; SJson *pVnodeJson = tjsonCreateObject(); if (pVnodeJson == NULL) continue; - if (tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id) != 0) tjsonDelete(pVnodeJson); - if (tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role) != 0) tjsonDelete(pVnodeJson); + tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id); + tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role); if (tjsonAddItemToArray(pVnodesJson, pVnodeJson) != 0) tjsonDelete(pVnodeJson); } @@ -235,7 +231,7 @@ void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; - if (tjsonAddItemToObject(pMonitor->pJson, "disks_infos", pJson) != 0) { + if (tjsonAddItemToObject(pMonitor->pJson, "disk_infos", pJson) != 0) { tjsonDelete(pJson); return; } @@ -261,7 +257,6 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { if (pLogdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "logdir", pLogdirJson) != 0) return; tjsonAddStringToObject(pLogdirJson, "name", pInfo->logdir.name); - tjsonAddDoubleToObject(pLogdirJson, "level", pInfo->logdir.level); tjsonAddDoubleToObject(pLogdirJson, "avail", pInfo->logdir.size.avail); tjsonAddDoubleToObject(pLogdirJson, "used", pInfo->logdir.size.used); tjsonAddDoubleToObject(pLogdirJson, "total", pInfo->logdir.size.total); @@ -270,7 +265,6 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { if (pTempdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "tempdir", pTempdirJson) != 0) return; tjsonAddStringToObject(pTempdirJson, "name", pInfo->tempdir.name); - tjsonAddDoubleToObject(pTempdirJson, "level", pInfo->tempdir.level); tjsonAddDoubleToObject(pTempdirJson, "avail", pInfo->tempdir.size.avail); tjsonAddDoubleToObject(pTempdirJson, "used", pInfo->tempdir.size.used); tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); @@ -296,9 +290,9 @@ static void monSetLogInfo(SMonInfo *pMonitor) { char buf[40] = {0}; taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); - if (tjsonAddStringToObject(pLogItem, "ts", buf) != 0) tjsonDelete(pLogJson); - if (tjsonAddDoubleToObject(pLogItem, "level", pLogItem->level) != 0) tjsonDelete(pLogJson); - if (tjsonAddStringToObject(pLogItem, "content", pLogItem->content) != 0) tjsonDelete(pLogJson); + tjsonAddStringToObject(pLogJson, "ts", buf); + tjsonAddDoubleToObject(pLogJson, "level", pLogItem->level); + tjsonAddStringToObject(pLogJson, "content", pLogItem->content); if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); } diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index a1805d0c9c..ad48ed5407 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -13,21 +13,262 @@ #include "os.h" #include "monitor.h" +#include "tglobal.h" class MonitorTest : public ::testing::Test { protected: - static void SetUpTestSuite() { root = "/tmp/monTest"; } - static void TearDownTestSuite() {} + static void SetUpTestSuite() { + SMonCfg cfg; + cfg.maxLogs = 2; + cfg.port = 80; + cfg.server = "localhost"; + monInit(&cfg); + } + + static void TearDownTestSuite() { monCleanup(); } public: void SetUp() override {} void TearDown() override {} - static const char *root; + void GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); + void GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); + void GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); + void GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo); + void GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); + void GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); + void AddLogInfo1(); + void AddLogInfo2(); }; -const char *MonitorTest::root; - -TEST_F(MonitorTest, 01_Open_Close) { - +void MonitorTest::GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { + pInfo->dnode_id = 1; + strcpy(pInfo->dnode_ep, "localhost"); +} + +void MonitorTest::GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { + strcpy(pInfo->first_ep, "localhost:6030"); + pInfo->first_ep_dnode_id = 1; + strcpy(pInfo->version, "3.0.0.0"); + pInfo->master_uptime = 1; + pInfo->monitor_interval = 2; + pInfo->vgroups_total = 3; + pInfo->vgroups_alive = 43; + pInfo->vnodes_total = 5; + pInfo->vnodes_alive = 6; + pInfo->connections_total = 7; + + pInfo->dnodes = taosArrayInit(4, sizeof(SMonDnodeDesc)); + SMonDnodeDesc d1 = {0}; + d1.dnode_id = 1; + strcpy(d1.dnode_ep, "localhost:6030"); + strcpy(d1.status, "ready"); + taosArrayPush(pInfo->dnodes, &d1); + SMonDnodeDesc d2 = {0}; + d2.dnode_id = 2; + strcpy(d2.dnode_ep, "localhost:7030"); + strcpy(d2.status, "offline"); + taosArrayPush(pInfo->dnodes, &d2); + + pInfo->mnodes = taosArrayInit(4, sizeof(SMonMnodeDesc)); + SMonMnodeDesc m1 = {0}; + m1.mnode_id = 1; + strcpy(m1.mnode_ep, "localhost:6030"); + strcpy(m1.role, "master"); + taosArrayPush(pInfo->mnodes, &m1); + SMonMnodeDesc m2 = {0}; + m2.mnode_id = 2; + strcpy(m2.mnode_ep, "localhost:7030"); + strcpy(m2.role, "unsynced"); + taosArrayPush(pInfo->mnodes, &m2); +} + +void MonitorTest::GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { + pInfo->vgroups = taosArrayInit(4, sizeof(SMonVgroupDesc)); + + SMonVgroupDesc vg1 = {0}; + vg1.vgroup_id = 1; + strcpy(vg1.database_name, "d1"); + vg1.tables_num = 4; + strcpy(vg1.status, "ready"); + vg1.vnodes[0].dnode_id = 1; + strcpy(vg1.vnodes[0].vnode_role, "master"); + vg1.vnodes[1].dnode_id = 2; + strcpy(vg1.vnodes[1].vnode_role, "slave"); + taosArrayPush(pInfo->vgroups, &vg1); + + SMonVgroupDesc vg2 = {0}; + vg2.vgroup_id = 2; + strcpy(vg2.database_name, "d2"); + vg2.tables_num = 5; + strcpy(vg2.status, "offline"); + vg2.vnodes[0].dnode_id = 1; + strcpy(vg2.vnodes[0].vnode_role, "master"); + vg2.vnodes[1].dnode_id = 2; + strcpy(vg2.vnodes[1].vnode_role, "unsynced"); + taosArrayPush(pInfo->vgroups, &vg2); + + SMonVgroupDesc vg3 = {0}; + vg3.vgroup_id = 3; + strcpy(vg3.database_name, "d3"); + vg3.tables_num = 6; + strcpy(vg3.status, "ready"); + vg3.vnodes[0].dnode_id = 1; + strcpy(vg3.vnodes[0].vnode_role, "master"); + taosArrayPush(pInfo->vgroups, &vg3); +} + +void MonitorTest::GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { + pInfo->expire_time = 1234567; + pInfo->timeseries_total = 234567; + pInfo->timeseries_used = 34567; +} + +void MonitorTest::GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { + pInfo->uptime = 1.2; + pInfo->cpu_engine = 2.1; + pInfo->cpu_system = 2.1; + pInfo->cpu_cores = 2; + pInfo->mem_engine = 3.1; + pInfo->mem_system = 3.2; + pInfo->mem_total = 3.3; + pInfo->disk_engine = 4.1; + pInfo->disk_used = 4.2; + pInfo->disk_total = 4.3; + pInfo->net_in = 5.1; + pInfo->net_out = 5.2; + pInfo->io_read = 6.1; + pInfo->io_write = 6.2; + pInfo->io_read_disk = 7.1; + pInfo->io_write_disk = 7.2; + pInfo->req_select = 8; + pInfo->req_select_rate = 8.1; + pInfo->req_insert = 9; + pInfo->req_insert_success = 10; + pInfo->req_insert_rate = 10.1; + pInfo->req_insert_batch = 11; + pInfo->req_insert_batch_success = 12; + pInfo->req_insert_batch_rate = 12.3; + pInfo->errors = 4; + pInfo->vnodes_num = 5; + pInfo->masters = 6; + pInfo->has_mnode = 1; +} + +void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { + pInfo->datadirs = taosArrayInit(2, sizeof(SMonDiskDesc)); + SMonDiskDesc d1 = {0}; + strcpy(d1.name, "/t1/d1/d"); + d1.level = 0; + d1.size.avail = 11; + d1.size.total = 12; + d1.size.used = 13; + taosArrayPush(pInfo->datadirs, &d1); + + SMonDiskDesc d2 = {0}; + strcpy(d2.name, "/t2d2/d"); + d2.level = 2; + d2.size.avail = 21; + d2.size.total = 22; + d2.size.used = 23; + taosArrayPush(pInfo->datadirs, &d2); + + SMonDiskDesc d3 = {0}; + strcpy(d3.name, "/t3/d3/d"); + d3.level = 3; + d3.size.avail = 31; + d3.size.total = 32; + d3.size.used = 33; + taosArrayPush(pInfo->datadirs, &d3); + + strcpy(pInfo->logdir.name, "/log/dir/d"); + pInfo->logdir.size.avail = 41; + pInfo->logdir.size.total = 42; + pInfo->logdir.size.used = 43; + + strcpy(pInfo->tempdir.name, "/data/dir/d"); + pInfo->tempdir.size.avail = 51; + pInfo->tempdir.size.total = 52; + pInfo->tempdir.size.used = 53; +} + +void MonitorTest::AddLogInfo1() { + SMonLogItem log1 = {0}; + log1.ts = taosGetTimestampMs(); + log1.level = 1; + strcpy(log1.content, "1 -------------------------- a"); + monAddLogItem(&log1); + + SMonLogItem log2 = {0}; + log2.ts = taosGetTimestampMs(); + log2.level = 1; + strcpy(log2.content, "1 ------------------------ b"); + monAddLogItem(&log2); + + SMonLogItem log3 = {0}; + log3.ts = taosGetTimestampMs(); + log3.level = 1; + strcpy(log3.content, "1 ------- c"); + monAddLogItem(&log3); +} + +void MonitorTest::AddLogInfo2() { + SMonLogItem log1; + log1.ts = taosGetTimestampMs(); + log1.level = 01; + strcpy(log1.content, "2 ------- a"); + monAddLogItem(&log1); + + SMonLogItem log2; + log2.ts = taosGetTimestampMs(); + log2.level = 0; + strcpy(log2.content, "2 ------- b"); + monAddLogItem(&log2); +} + +TEST_F(MonitorTest, 01_Full) { + AddLogInfo1(); + + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + SMonBasicInfo basicInfo = {0}; + GetBasicInfo(pMonitor, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); + + SMonClusterInfo clusterInfo = {0}; + SMonVgroupInfo vgroupInfo = {0}; + SMonGrantInfo grantInfo = {0}; + GetClusterInfo(pMonitor, &clusterInfo); + GetVgroupInfo(pMonitor, &vgroupInfo); + GetGrantInfo(pMonitor, &grantInfo); + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + + SMonDnodeInfo dnodeInfo = {0}; + GetDnodeInfo(pMonitor, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); + + SMonDiskInfo diskInfo = {0}; + GetDiskInfo(pMonitor, &diskInfo); + monSetDiskInfo(pMonitor, &diskInfo); + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); + + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); +} + +TEST_F(MonitorTest, 02_Log) { + AddLogInfo2(); + + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); } From 6cfff92b21d0d38e7e9e155402f1b40e62a7f522 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 17:11:24 +0800 Subject: [PATCH 61/82] get tfs monitor info --- include/libs/tfs/tfs.h | 9 +++++++++ source/dnode/mgmt/impl/inc/dndEnv.h | 5 ++--- source/dnode/mgmt/impl/src/dndEnv.c | 9 ++++++++- source/dnode/mgmt/impl/src/dndMgmt.c | 15 ++++++++++----- source/libs/tfs/CMakeLists.txt | 2 +- source/libs/tfs/src/tfs.c | 21 +++++++++++++++++++++ 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index b6d10c81bf..1b41da33bb 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -17,6 +17,7 @@ #define _TD_TFS_H_ #include "tdef.h" +#include "monitor.h" #ifdef __cplusplus extern "C" { @@ -237,6 +238,14 @@ const STfsFile *tfsReaddir(STfsDir *pDir); */ void tfsClosedir(STfsDir *pDir); +/** + * @brief Get disk info of tfs. + * + * @param pTfs The fs object. + * @param pInfo The info object. + */ +int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 724ba30155..efe54ee1e5 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -31,7 +31,7 @@ typedef struct { SDnode *pDnode; STaosQueue *queue; union { - SQWorkerPool pool; + SQWorkerPool pool; SWWorkerPool mpool; }; } SDnodeWorker; @@ -137,8 +137,7 @@ typedef struct SDnode { SStartupReq startup; } SDnode; - -int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); +int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index ae08d6387c..1cbdf0c1dc 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -324,4 +324,11 @@ void dndCleanup() { dInfo("dnode env is cleaned up"); } -int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; } \ No newline at end of file +int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; + + return tfsGetMonitorInfo(pDnode->pTfs, pInfo); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 268ed7e362..70f5060ead 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -474,13 +474,13 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { +static int32_t dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->dnode_id = dndGetDnodeId(pDnode); tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); return 0; } -static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } +static int32_t dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; @@ -490,7 +490,7 @@ static void dndSendMonitorReport(SDnode *pDnode) { if (pMonitor == NULL) return; SMonBasicInfo basicInfo = {0}; - if (dndGetBasicInfo(pDnode, &basicInfo) == 0) { + if (dndGetMonitorBasicInfo(pDnode, &basicInfo) == 0) { monSetBasicInfo(pMonitor, &basicInfo); } @@ -504,15 +504,20 @@ static void dndSendMonitorReport(SDnode *pDnode) { } SMonDnodeInfo dnodeInfo = {0}; - if (dndGetDnodeInfo(pDnode, &dnodeInfo) == 0) { + if (dndGetMonitorDnodeInfo(pDnode, &dnodeInfo) == 0) { monSetDnodeInfo(pMonitor, &dnodeInfo); } SMonDiskInfo diskInfo = {0}; - if (dndGetDiskInfo(pDnode, &diskInfo) == 0) { + if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { monSetDiskInfo(pMonitor, &diskInfo); } + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); + monSendReport(pMonitor); monCleanupMonitorInfo(pMonitor); } diff --git a/source/libs/tfs/CMakeLists.txt b/source/libs/tfs/CMakeLists.txt index 607ccd4c48..97e02fc8a9 100644 --- a/source/libs/tfs/CMakeLists.txt +++ b/source/libs/tfs/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(tfs os util common) +target_link_libraries(tfs os util common monitor) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index f686703643..944e67c863 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -544,3 +544,24 @@ static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { return pDisk; } + +int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { + pInfo->datadirs = taosArrayInit(32, sizeof(SMonDiskDesc)); + if (pInfo->datadirs == NULL) return -1; + + tfsUpdateSize(pTfs); + + tfsLock(pTfs); + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = &pTfs->tiers[level]; + for (int32_t disk = 0; disk < pTier->ndisk; ++disk) { + STfsDisk *pDisk = pTier->disks[disk]; + SMonDiskDesc dinfo = {0}; + dinfo.size = pDisk->size; + dinfo.level = pDisk->level; + tstrncpy(dinfo.name, pDisk->path, sizeof(dinfo.name)); + taosArrayPush(pInfo->datadirs, &dinfo); + } + } + tfsUnLock(pTfs); +} \ No newline at end of file From 64d224a0d2b5450e221b0136b8a80b8d57184025 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 17:28:00 +0800 Subject: [PATCH 62/82] syncInt --- include/libs/sync/sync.h | 4 +- source/libs/sync/inc/syncInt.h | 72 +++++++++++++++++++----------- source/libs/sync/inc/syncVoteMgr.h | 6 +++ source/libs/sync/src/syncMain.c | 2 +- 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index cd04783dbc..53fad4607a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -34,9 +34,7 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 0, TAOS_SYNC_STATE_CANDIDATE = 1, TAOS_SYNC_STATE_LEADER = 2, -} ESyncRole; - -typedef ESyncRole ESyncState; +} ESyncState; typedef struct SSyncBuffer { void* data; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0901330488..aedb9662b1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -25,6 +25,7 @@ extern "C" { #include #include "sync.h" #include "taosdef.h" +#include "tglobal.h" #include "tlog.h" #include "ttimer.h" @@ -91,31 +92,61 @@ typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; struct SSyncEnv; typedef struct SSyncEnv SSyncEnv; +struct SRaftStore; +typedef struct SRaftStore SRaftStore; + +struct SVotesGranted; +typedef struct SVotesGranted SVotesGranted; + +struct SVotesResponded; +typedef struct SVotesResponded SVotesResponded; + typedef struct SRaftId { SyncNodeId addr; // typedef uint64_t SyncNodeId; SyncGroupId vgId; // typedef int32_t SyncGroupId; } SRaftId; typedef struct SSyncNode { + // init by SSyncInfo SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SSyncFSM* pFsm; - - // passed from outside - void* rpcClient; + void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); + // init internal + SNodeInfo me; + int32_t peersNum; + SNodeInfo peers[TSDB_MAX_REPLICA]; + + // raft algorithm + SSyncFSM* pFsm; + SRaftId raftId; + SRaftId peersId[TSDB_MAX_REPLICA]; + int32_t replicaNum; + int32_t quorum; + + // life cycle int32_t refCount; int64_t rid; - SNodeInfo me; - SNodeInfo peers[TSDB_MAX_REPLICA]; - int32_t peersNum; + // tla+ server vars + ESyncState state; + SRaftStore* pRaftStore; - ESyncRole role; - SRaftId raftId; + // tla+ candidate vars + SVotesGranted* pVotesGranted; + SVotesResponded* pVotesResponded; + // tla+ leader vars + SHashObj* pNextIndex; + SHashObj* pMatchIndex; + + // tla+ log vars + SSyncLogStore* pLogStore; + SyncIndex commitIndex; + + // timer tmr_h pPingTimer; int32_t pingTimerMS; uint8_t pingTimerStart; @@ -136,32 +167,21 @@ typedef struct SSyncNode { // callback int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); - int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); - int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); } SSyncNode; SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); - -void syncNodeClose(SSyncNode* pSyncNode); - -void syncNodePingAll(SSyncNode* pSyncNode); - -void syncNodePingPeers(SSyncNode* pSyncNode); - -void syncNodePingSelf(SSyncNode* pSyncNode); - -int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); - -int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); +void syncNodeClose(SSyncNode* pSyncNode); +void syncNodePingAll(SSyncNode* pSyncNode); +void syncNodePingPeers(SSyncNode* pSyncNode); +void syncNodePingSelf(SSyncNode* pSyncNode); +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index cfcf58bee2..b841f2e316 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -26,6 +26,12 @@ extern "C" { #include "syncInt.h" #include "taosdef.h" +typedef struct SVotesGranted { +} SVotesGranted; + +typedef struct SVotesResponded { +} SVotesResponded; + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9cb3a61fff..7e01e7e81c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -88,7 +88,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { } } - pSyncNode->role = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId); pSyncNode->pPingTimer = NULL; From c018e91859645c280a8f12d7784a0ba87abd8885 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 17:30:33 +0800 Subject: [PATCH 63/82] minor changes --- include/os/osSysinfo.h | 6 +- source/dnode/mnode/impl/src/mndTelem.c | 4 +- source/os/src/osSysinfo.c | 110 ++++++++++++------------- 3 files changed, 58 insertions(+), 62 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index bf9d7662e6..7ad9c57a3c 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -34,9 +34,9 @@ typedef struct { } SDiskSpace; void taosGetSystemInfo(); -bool taosGetEmail(char *email, int32_t maxLen); -bool taosGetOsReleaseName(char *releaseName, int32_t maxLen); -bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); +int32_t taosGetEmail(char *email, int32_t maxLen); +int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); int32_t taosGetCpuCores(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetTotalSysMemoryKB(uint64_t *kb); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 1cd7593846..da2c52ec8f 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -52,12 +52,12 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { tjsonAddStringToObject(pJson, "instanceId", clusterName); tjsonAddDoubleToObject(pJson, "reportVersion", 1); - if (taosGetOsReleaseName(tmp, sizeof(tmp))) { + if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } int32_t numOfCores = 0; - if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) { + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { tjsonAddStringToObject(pJson, "cpuModel", tmp); tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); } else { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 3e761c6d91..d3f56e5901 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -80,7 +80,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { return true; } - int32_t taosGetCpuCores() { SYSTEM_INFO info; GetSystemInfo(&info); @@ -106,7 +105,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); return 0; } else { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -173,12 +172,12 @@ void taosGetSystemInfo() { } void taosKillSystem() { - //printf("function taosKillSystem, exit!"); + // printf("function taosKillSystem, exit!"); exit(0); } int taosSystem(const char *cmd) { - //printf("taosSystem not support"); + // printf("taosSystem not support"); return -1; } @@ -241,9 +240,8 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include - void taosKillSystem() { - //printf("function taosKillSystem, exit!"); + // printf("function taosKillSystem, exit!"); exit(0); } @@ -300,7 +298,7 @@ bool taosGetSysMemory(float *memoryUsedMB) { } int taosSystem(const char *cmd) { - //printf("un support funtion"); + // printf("un support funtion"); return -1; } @@ -309,7 +307,7 @@ void taosSetCoreDump() {} int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { struct statvfs info; if (statvfs(dataDir, &info)) { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { @@ -404,12 +402,12 @@ bool taosGetProcMemory(float *memoryUsedMB) { // FILE *fp = fopen(tsProcMemFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsProcMemFile); + // printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - char * line = NULL; + char *line = NULL; while (!taosEOFFile(pFile)) { _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { @@ -421,7 +419,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { } if (line == NULL) { - //printf("read file:%s failed", tsProcMemFile); + // printf("read file:%s failed", tsProcMemFile); taosCloseFile(&pFile); return false; } @@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); return true; } @@ -440,14 +438,14 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { // FILE *fp = fopen(tsSysCpuFile, "r"); TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsSysCpuFile); + // printf("open file:%s failed", tsSysCpuFile); return false; } - char * line = NULL; + char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - //printf("read file:%s failed", tsSysCpuFile); + // printf("read file:%s failed", tsSysCpuFile); taosCloseFile(&pFile); return false; } @@ -456,7 +454,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); return true; } @@ -465,14 +463,14 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { // FILE *fp = fopen(tsProcCpuFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsProcCpuFile); + // printf("open file:%s failed", tsProcCpuFile); return false; } - char * line = NULL; + char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - //printf("read file:%s failed", tsProcCpuFile); + // printf("read file:%s failed", tsProcCpuFile); taosCloseFile(&pFile); return false; } @@ -486,12 +484,11 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); return true; } - int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { @@ -550,12 +547,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { // FILE *fp = fopen(tsSysNetFile, "r"); TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsSysNetFile); + // printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - char * line = NULL; + char *line = NULL; while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; @@ -590,7 +587,7 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); return true; @@ -636,12 +633,12 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { // FILE *fp = fopen(tsProcIOFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsProcIOFile); + // printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - char * line = NULL; + char *line = NULL; char tmp[10]; int readIndex = 0; @@ -662,11 +659,11 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); if (readIndex < 2) { - //printf("read file:%s failed", tsProcIOFile); + // printf("read file:%s failed", tsProcIOFile); return false; } @@ -713,12 +710,11 @@ void taosGetSystemInfo() { taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - } void taosKillSystem() { // SIGINT - //printf("taosd will shut down soon"); + // printf("taosd will shut down soon"); kill(tsProcId, 2); } @@ -727,22 +723,22 @@ int taosSystem(const char *cmd) { int res; char buf[1024]; if (cmd == NULL) { - //printf("taosSystem cmd is NULL!"); + // printf("taosSystem cmd is NULL!"); return -1; } if ((fp = popen(cmd, "r")) == NULL) { - //printf("popen cmd:%s error: %s", cmd, strerror(errno)); + // printf("popen cmd:%s error: %s", cmd, strerror(errno)); return -1; } else { while (fgets(buf, sizeof(buf), fp)) { - //printf("popen result:%s", buf); + // printf("popen result:%s", buf); } if ((res = pclose(fp)) == -1) { - //printf("close popen file pointer fp error!"); + // printf("close popen file pointer fp error!"); } else { - //printf("popen res is :%d", res); + // printf("popen res is :%d", res); } return res; @@ -757,14 +753,14 @@ void taosSetCoreDump(bool enable) { struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { #ifndef _ALPINE - //printf("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + // printf("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); #else - //printf("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); + // printf("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); #endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { - //printf("set unlimited fail, error: %s", strerror(errno)); + // printf("set unlimited fail, error: %s", strerror(errno)); rlim_new.rlim_cur = rlim.rlim_max; rlim_new.rlim_max = rlim.rlim_max; (void)setrlimit(RLIMIT_CORE, &rlim_new); @@ -773,9 +769,9 @@ void taosSetCoreDump(bool enable) { if (getrlimit(RLIMIT_CORE, &rlim) == 0) { #ifndef _ALPINE - //printf("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + // printf("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); #else - //printf("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); + // printf("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); #endif } @@ -801,10 +797,10 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - //printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - //printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + // printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); old_usespid = 0; old_len = 0; @@ -817,10 +813,10 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - //printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - //printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + // printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif } @@ -885,7 +881,7 @@ SysNameInfo taosGetSysNameInfo() { return info; } -bool taosGetEmail(char *email, int32_t maxLen) { +int32_t taosGetEmail(char *email, int32_t maxLen) { const char *filepath = "/usr/local/taos/email"; TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); @@ -893,17 +889,17 @@ bool taosGetEmail(char *email, int32_t maxLen) { if (taosReadFile(pFile, (void *)email, maxLen) < 0) { taosCloseFile(&pFile); - return false; + return -1; } taosCloseFile(&pFile); - return true; + return 0; } -bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { - char *line = NULL; - size_t size = 0; - bool ret = false; +int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { + char *line = NULL; + size_t size = 0; + int32_t code = -1; TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return false; @@ -917,21 +913,21 @@ bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { line[size - 2] = 0; } tstrncpy(releaseName, p, maxLen); - ret = true; + code = 0; break; } } if (line != NULL) free(line); taosCloseFile(&pFile); - return ret; + return code; } -bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { char *line = NULL; size_t size = 0; int32_t done = 0; - bool ret = false; + int32_t code = -1; TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return false; @@ -941,7 +937,7 @@ bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char *v = strchr(line, ':') + 2; tstrncpy(cpuModel, v, maxLen); - ret = true; + code = 0; done |= 1; } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { const char *v = strchr(line, ':') + 2; @@ -953,7 +949,7 @@ bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { if (line != NULL) free(line); taosCloseFile(&pFile); - return ret; + return code; } bool taosGetTotalSysMemoryKB(uint64_t *kb) { From c890b0b5c1987ffbec0186c5c72ba4a0db27d100 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 18:10:56 +0800 Subject: [PATCH 64/82] sysinfo in monitor --- include/libs/monitor/monitor.h | 6 +- include/os/osEnv.h | 6 +- include/os/osSysinfo.h | 12 +- source/common/src/tglobal.c | 10 +- source/dnode/mgmt/impl/src/dndMgmt.c | 43 ++++-- source/dnode/mnode/impl/src/mndTelem.c | 11 +- source/os/src/osEnv.c | 11 +- source/os/src/osSysinfo.c | 183 +++++++++++-------------- 8 files changed, 140 insertions(+), 142 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index b3b6091b95..487b2bfe88 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -88,9 +88,9 @@ typedef struct { float cpu_engine; float cpu_system; float cpu_cores; - float mem_engine; // MB - float mem_system; // MB - float mem_total; // MB + int64_t mem_engine; // KB + int64_t mem_system; // KB + int64_t mem_total; // KB float disk_engine; // GB float disk_used; // GB float disk_total; // GB diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 4ac073f6c2..1426ba87f6 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -28,11 +28,11 @@ extern char tsCharset[]; extern char tsLocale[]; extern int8_t tsDaylight; extern bool tsEnableCoreFile; -extern int64_t tsPageSize; +extern int64_t tsPageSizeKB; extern int64_t tsOpenMax; extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; +extern float tsNumOfCores; +extern int64_t tsTotalMemoryKB; extern char configDir[]; extern char tsDataDir[]; diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 7ad9c57a3c..df5ed28a99 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -36,12 +36,12 @@ typedef struct { void taosGetSystemInfo(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); -int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); -int32_t taosGetCpuCores(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetTotalSysMemoryKB(uint64_t *kb); -bool taosGetProcMemory(float *memoryUsedMB); // -bool taosGetSysMemory(float *memoryUsedMB); // +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); +int32_t taosGetCpuCores(float *numOfCores); +int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine); +int32_t taosGetTotalMemory(int64_t *totalKB); +int32_t taosGetProcMemory(int64_t *usedKB); +int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); bool taosReadProcIO(int64_t *rchars, int64_t *wchars); bool taosGetProcIO(float *readKB, float *writeKB); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5998b7c9ce..93b66e3369 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -279,11 +279,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; @@ -404,10 +404,6 @@ static void taosSetSystemCfg(SConfig *pCfg) { const char *charset = cfgGetItem(pCfg, "charset")->str; taosSetSystemLocale(locale, charset); - if (tsNumOfCores <= 1) { - tsNumOfCores = 2; - } - bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetConsoleEcho(enableCore); diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 70f5060ead..704847894e 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -474,13 +474,40 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -static int32_t dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { +static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->dnode_id = dndGetDnodeId(pDnode); tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); - return 0; } -static int32_t dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } +static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f); + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + pInfo->cpu_cores = tsNumOfCores; + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 4.1; + pInfo->disk_used = 4.2; + pInfo->disk_total = 4.3; + pInfo->net_in = 5.1; + pInfo->net_out = 5.2; + pInfo->io_read = 6.1; + pInfo->io_write = 6.2; + pInfo->io_read_disk = 7.1; + pInfo->io_write_disk = 7.2; + pInfo->req_select = 8; + pInfo->req_select_rate = 8.1; + pInfo->req_insert = 9; + pInfo->req_insert_success = 10; + pInfo->req_insert_rate = 10.1; + pInfo->req_insert_batch = 11; + pInfo->req_insert_batch_success = 12; + pInfo->req_insert_batch_rate = 12.3; + pInfo->errors = 4; + pInfo->vnodes_num = 5; + pInfo->masters = 6; + pInfo->has_mnode = 1; +} static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; @@ -490,9 +517,8 @@ static void dndSendMonitorReport(SDnode *pDnode) { if (pMonitor == NULL) return; SMonBasicInfo basicInfo = {0}; - if (dndGetMonitorBasicInfo(pDnode, &basicInfo) == 0) { - monSetBasicInfo(pMonitor, &basicInfo); - } + dndGetMonitorBasicInfo(pDnode, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); SMonClusterInfo clusterInfo = {0}; SMonVgroupInfo vgroupInfo = {0}; @@ -504,9 +530,8 @@ static void dndSendMonitorReport(SDnode *pDnode) { } SMonDnodeInfo dnodeInfo = {0}; - if (dndGetMonitorDnodeInfo(pDnode, &dnodeInfo) == 0) { - monSetDnodeInfo(pMonitor, &dnodeInfo); - } + dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); SMonDiskInfo diskInfo = {0}; if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index da2c52ec8f..968319e7b4 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -56,19 +56,16 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { tjsonAddStringToObject(pJson, "os", tmp); } - int32_t numOfCores = 0; + float numOfCores = 0; if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { tjsonAddStringToObject(pJson, "cpuModel", tmp); tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); } else { - tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores()); + tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); } - uint64_t memoryKB = 0; - if (taosGetTotalSysMemoryKB(&memoryKB)) { - snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB); - tjsonAddStringToObject(pJson, "memory", tmp); - } + snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); tjsonAddStringToObject(pJson, "version", version); tjsonAddStringToObject(pJson, "buildInfo", buildinfo); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 4c368fe895..0b2fe904b3 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -31,11 +31,11 @@ char tsLocale[TD_LOCALE_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0}; int8_t tsDaylight = 0; bool tsEnableCoreFile = 0; -int64_t tsPageSize = 0; +int64_t tsPageSizeKB = 0; int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; -int32_t tsNumOfCores = 0; -int32_t tsTotalMemoryMB = 0; +float tsNumOfCores = 0; +int64_t tsTotalMemoryKB = 0; void osInit() { srand(taosSafeRand()); @@ -44,6 +44,11 @@ void osInit() { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); taosGetSystemInfo(); + // deadlock in query + if (tsNumOfCores < 2) { + tsNumOfCores = 2; + } + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index d3f56e5901..bf036354e9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -39,32 +39,32 @@ #include #pragma warning(pop) -static int32_t taosGetTotalMemory() { +int32_t taosGetTotalMemory(int64_t *totalKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return 0; + return -1; } - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - return (int32_t)nMemTotal; + *totalKB = memsStat.ullTotalPhys / 1024; + return 0; } -bool taosGetSysMemory(float *memoryUsedMB) { +int32_t taosGetSysMemory(int64_t *usedKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return false; + return -1; } - float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + int64_t nMemFree = memsStat.ullAvailPhys / 1024; + int64_t nMemTotal = memsStat.ullTotalPhys / 1024.0; - *memoryUsedMB = nMemTotal - nMemFree; - return true; + *usedKB = nMemTotal - nMemFree; + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { +int32_t taosGetProcMemory(int64_t *usedKB) { unsigned bytes_used = 0; #if defined(_WIN64) && defined(_MSC_VER) @@ -76,20 +76,21 @@ bool taosGetProcMemory(float *memoryUsedMB) { } #endif - *memoryUsedMB = (float)bytes_used / 1024 / 1024; - return true; + *usedKB = bytes_used / 1024; + return 0; } -int32_t taosGetCpuCores() { +int32_t taosGetCpuCores(float *numOfCores) { SYSTEM_INFO info; GetSystemInfo(&info); - return (int32_t)info.dwNumberOfProcessors; + *numOfCores = info.dwNumberOfProcessors; + return 0; } -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { *sysCpuUsage = 0; *procCpuUsage = 0; - return true; + return 0; } int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { @@ -162,8 +163,8 @@ bool taosGetProcIO(float *readKB, float *writeKB) { } void taosGetSystemInfo() { - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); float tmp1, tmp2; taosGetBandSpeed(&tmp1); @@ -245,16 +246,17 @@ void taosKillSystem() { exit(0); } -int32_t taosGetCpuCores() { return sysconf(_SC_NPROCESSORS_ONLN); } +int32_t taosGetCpuCores(float *numOfCores) { + *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + return 0; +} void taosGetSystemInfo() { - // taosGetProcInfos(); - - tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); long physical_pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); - tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); - tsPageSize = page_size; + tsTotalMemoryKB = physical_pages * page_size / 1024; + tsPageSizeKB = page_size / 1024; + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -281,20 +283,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { return true; } -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { *sysCpuUsage = 0; *procCpuUsage = 0; - return true; + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; +int32_t taosGetProcMemory(int64_t *usedKB) { + *usedKB = 0; + return 0; } -bool taosGetSysMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; +int32_t taosGetSysMemory(int64_t *usedKB) { + *usedKB = 0; + return 0; } int taosSystem(const char *cmd) { @@ -375,35 +377,34 @@ static char tsSysCpuFile[] = "/proc/stat"; static char tsProcCpuFile[25] = {0}; static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; -static float tsPageSizeKB = 0; static void taosGetProcInfos() { - tsPageSize = sysconf(_SC_PAGESIZE); + tsPageSizeKB = sysconf(_SC_PAGESIZE) / 1024; tsOpenMax = sysconf(_SC_OPEN_MAX); tsStreamMax = sysconf(_SC_STREAM_MAX); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + snprintf(tsProcMemFile, sizeof(tsProcMemFile), "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } -static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } - -bool taosGetSysMemory(float *memoryUsedMB) { - float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; - *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; - return true; +int32_t taosGetTotalMemory(int64_t *totalKB) { + *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { +int32_t taosGetSysMemory(int64_t *usedKB) { + *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; + return 0; +} + +int32_t taosGetProcMemory(int64_t *usedKB) { // FILE *fp = fopen(tsProcMemFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // printf("open file:%s failed", tsProcMemFile); - return false; + return -1; } ssize_t _bytes = 0; @@ -421,25 +422,23 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { // printf("read file:%s failed", tsProcMemFile); taosCloseFile(&pFile); - return false; + return -1; } - int64_t memKB = 0; - char tmp[10]; - sscanf(line, "%s %" PRId64, tmp, &memKB); - *memoryUsedMB = (float)((double)memKB / 1024); + char tmp[10]; + sscanf(line, "%s %" PRId64, tmp, usedKB); if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { +static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { // FILE *fp = fopen(tsSysCpuFile, "r"); TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // printf("open file:%s failed", tsSysCpuFile); - return false; + return -1; } char *line = NULL; @@ -447,7 +446,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { if ((_bytes < 0) || (line == NULL)) { // printf("read file:%s failed", tsSysCpuFile); taosCloseFile(&pFile); - return false; + return -1; } char cpu[10] = {0}; @@ -456,15 +455,15 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { +static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { // FILE *fp = fopen(tsProcCpuFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // printf("open file:%s failed", tsProcCpuFile); - return false; + return -1; } char *line = NULL; @@ -472,7 +471,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { if ((_bytes < 0) || (line == NULL)) { // printf("read file:%s failed", tsProcCpuFile); taosCloseFile(&pFile); - return false; + return -1; } for (int i = 0, blank = 0; line[i] != 0; ++i) { @@ -486,23 +485,26 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } +int32_t taosGetCpuCores(float *numOfCores) { + *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + return 0; +} -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; static uint64_t lastProcTotal = 0; SysCpuInfo sysCpu; ProcCpuInfo procCpu; - if (!taosGetSysCpuInfo(&sysCpu)) { - return false; + if (taosGetSysCpuInfo(&sysCpu) != 0) { + return -1; } - if (!taosGetProcCpuInfo(&procCpu)) { - return false; + if (taosGetProcCpuInfo(&procCpu) != 0) { + return -1; } uint64_t curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; @@ -513,21 +515,21 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { lastSysUsed = curSysUsed > 1 ? curSysUsed : 1; lastSysTotal = curSysTotal > 1 ? curSysTotal : 1; lastProcTotal = curProcTotal > 1 ? curProcTotal : 1; - return false; + return -1; } if (curSysTotal == lastSysTotal) { - return false; + return -1; } - *sysCpuUsage = (float)((double)(curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100); - *procCpuUsage = (float)((double)(curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100); + *cpu_engine = (float)((double)(curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100); + *cpu_system = (float)((double)(curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100); lastSysUsed = curSysUsed; lastSysTotal = curSysTotal; lastProcTotal = curProcTotal; - return true; + return 0; } int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { @@ -700,13 +702,10 @@ bool taosGetProcIO(float *readKB, float *writeKB) { void taosGetSystemInfo() { taosGetProcInfos(); - - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); float tmp1, tmp2; - taosGetSysMemory(&tmp1); - taosGetProcMemory(&tmp2); taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); @@ -923,7 +922,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { return code; } -int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { char *line = NULL; size_t size = 0; int32_t done = 0; @@ -941,7 +940,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { done |= 1; } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { const char *v = strchr(line, ':') + 2; - *numOfCores = atoi(v); + *numOfCores = atof(v); done |= 2; } } @@ -952,28 +951,4 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { return code; } -bool taosGetTotalSysMemoryKB(uint64_t *kb) { - char *line = NULL; - size_t size = 0; - bool ret = false; - - TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "MemTotal", 8) == 0) { - const char *p = strchr(line, ':') + 1; - while (*p == ' ') p++; - ret = true; - *kb = atoll(p); - break; - } - } - - if (line != NULL) free(line); - taosCloseFile(&pFile); - return ret; -} - #endif From 0ca24f3ea18db333adb829c0ecd2a528bb315d01 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Mar 2022 18:49:39 +0800 Subject: [PATCH 65/82] [td-13039] refactor cache. --- include/util/tcache.h | 19 ++- source/dnode/mnode/impl/src/mndProfile.c | 65 ++++---- source/util/src/tcache.c | 193 ++++++++++++++++++----- source/util/test/cacheTest.cpp | 24 ++- 4 files changed, 230 insertions(+), 71 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index 7c29ab4f58..b5c1578380 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -40,9 +40,9 @@ typedef struct SCacheStatis { int64_t refreshCount; } SCacheStatis; -typedef struct SCacheObj SCacheObj; - -struct STrashElem; +typedef struct SCacheObj SCacheObj; +typedef struct SCacheIter SCacheIter; +typedef struct STrashElem STrashElem; /** * initialize the cache object @@ -106,6 +106,13 @@ void *taosCacheTransferData(SCacheObj *pCacheObj, void **data); */ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); +/** + * + * @param pCacheObj + * @return + */ +size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj); + /** * move all data node into trash, clear node in trash can if it is not referenced by any clients * @param handle @@ -138,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); */ void taosStopCacheRefreshWorker(); +SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj); +bool taosCacheIterNext(SCacheIter* pIter); +void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen); +void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen); +void taosCacheDestroyIter(SCacheIter* pIter); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e313c4d676..54c2b2fbcd 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); -static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); +static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); static int32_t mndProcessConnectReq(SMnodeMsg *pReq); @@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { taosCacheRelease(pMgmt->cache, (void **)&pConn, false); } -static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - *pConn = NULL; - - pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); - if (pIter == NULL) return NULL; - - SCacheDataNode **pNode = pIter; - if (pNode == NULL || *pNode == NULL) { - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); - return NULL; +void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) { + SConnObj* pConn = NULL; + bool hasNext = taosCacheIterNext(pIter); + if (hasNext) { + size_t dataLen = 0; + pConn = taosCacheIterGetData(pIter, &dataLen); + } else { + taosCacheDestroyIter(pIter); } - *pConn = (SConnObj *)((*pNode)->data); - return pIter; + return pConn; } static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + if (pIter != NULL) { + taosCacheDestroyIter(pIter); + } } static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { @@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { int32_t rspLen = 0; mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv); + SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); } break; } @@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { int32_t rspLen = 0; mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv); + SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); } break; } @@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); + pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; strcpy(pMeta->tbName, mndShowStr(pShow->type)); @@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in char *pWrite; char ipStr[TSDB_IPv4ADDR_LEN + 6]; + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->cache); + } + while (numOfRows < rows) { - pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + pConn = mndGetNextConn(pMnode, pShow->pIter); if (pConn == NULL) break; cols = 0; @@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, void *pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->cache); + } + while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + pConn = mndGetNextConn(pMnode, pShow->pIter); if (pConn == NULL) { - pShow->pIter = pIter; + pShow->pIter = NULL; break; } if (numOfRows + pConn->numOfQueries >= rows) { - mndCancelGetNextConn(pMnode, pIter); + taosCacheDestroyIter(pShow->pIter); + pShow->pIter = NULL; break; } - pShow->pIter = pIter; for (int32_t i = 0; i < pConn->numOfQueries; ++i) { SQueryDesc *pDesc = pConn->pQueries + i; cols = 0; @@ -913,6 +919,7 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, } static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + if (pIter != NULL) { + taosCacheDestroyIter(pIter); + } } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index aec2530f5d..3920558276 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -19,6 +19,9 @@ #include "tlog.h" #include "tutil.h" +#define CACHE_MAX_CAPACITY 1024*1024*16 +#define CACHE_DEFAULT_CAPACITY 1024*4 + static pthread_t cacheRefreshWorker = {0}; static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; @@ -55,6 +58,14 @@ typedef struct STrashElem { SCacheNode *pData; } STrashElem; +typedef struct SCacheIter { + SCacheObj *pCacheObj; + SCacheNode **pCurrent; + int32_t entryIndex; + int32_t index; + int32_t numOfObj; +} SCacheIter; + /* * to accommodate the old data which has the same key value of new one in hashList * when an new node is put into cache, if an existed one with the same key: @@ -264,6 +275,7 @@ static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* prev->pNext = pNode->pNext; } + pNode->pNext = NULL; pe->num -= 1; } @@ -287,6 +299,57 @@ doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode* return pNode; } +static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; + SCacheObj *pCacheObj = ps->pCacheObj; + + if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + + // this node should be remove from hash table + return false; + } + + if (ps->fp) { + (ps->fp)(pNode->data, ps->param1); + } + + // do not remove element in hash table + return true; +} + +static bool doRemoveNodeFn(void *param, SCacheNode *pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; + SCacheObj *pCacheObj = ps->pCacheObj; + + if (T_REF_VAL_GET(pNode) == 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { // do add to trashcan + taosAddToTrashcan(pCacheObj, pNode); + } + + // this node should be remove from hash table + return false; +} + +static FORCE_INLINE int32_t getCacheCapacity(int32_t length) { + int32_t len = 0; + if (length < CACHE_DEFAULT_CAPACITY) { + len = CACHE_DEFAULT_CAPACITY; + return len; + } else if (length > CACHE_MAX_CAPACITY) { + len = CACHE_MAX_CAPACITY; + return len; + } + + len = CACHE_DEFAULT_CAPACITY; + while (len < length && len < CACHE_MAX_CAPACITY) { + len = (len << 1u); + } + + return len > CACHE_MAX_CAPACITY? CACHE_MAX_CAPACITY:len; +} + SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName) { const int32_t SLEEP_DURATION = 500; // 500 ms @@ -301,7 +364,9 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pEntryList = calloc(4096, sizeof(SCacheEntry)); + // TODO add the auto extend procedure + pCacheObj->capacity = 4096; + pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry)); if (pCacheObj->pEntryList == NULL) { free(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -309,7 +374,6 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext } // set free cache node callback function - pCacheObj->capacity = 4096; // todo refactor pCacheObj->hashFp = taosGetDefaultHashFunction(keyType); pCacheObj->freeFp = fn; pCacheObj->refreshTime = refreshTimeInSeconds * 1000; @@ -582,20 +646,6 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } } -static bool doRemoveNodeFn(void *param, SCacheNode *pNode) { - SCacheObjTravSup *ps = (SCacheObjTravSup *)param; - SCacheObj *pCacheObj = ps->pCacheObj; - - if (T_REF_VAL_GET(pNode) == 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { // do add to trashcan - taosAddToTrashcan(pCacheObj, pNode); - } - - // this node should be remove from hash table - return false; -} - void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) { int32_t numOfEntries = (int32_t)pCacheObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { @@ -757,25 +807,6 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { free(pCacheObj); } -bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { - SCacheObjTravSup *ps = (SCacheObjTravSup *)param; - SCacheObj *pCacheObj = ps->pCacheObj; - - if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - - // this node should be remove from hash table - return false; - } - - if (ps->fp) { - (ps->fp)(pNode->data, ps->param1); - } - - // do not remove element in hash table - return true; -} - static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { assert(pCacheObj != NULL); @@ -877,4 +908,94 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) doCacheRefresh(pCacheObj, now, fp, param1); } -void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; } \ No newline at end of file +void taosStopCacheRefreshWorker(void) { + stopRefreshWorker = true; +} + +size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { + return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash; +} + +SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { + ASSERT(pCacheObj != NULL); + SCacheIter* pIter = calloc(1, sizeof(SCacheIter)); + pIter->pCacheObj = (SCacheObj*) pCacheObj; + pIter->entryIndex = -1; + pIter->index = -1; + return pIter; +} + +bool taosCacheIterNext(SCacheIter* pIter) { + SCacheObj* pCacheObj = pIter->pCacheObj; + + if (pIter->index + 1 >= pIter->numOfObj) { + if (pIter->entryIndex + 1 >= pCacheObj->capacity) { + return false; + } + + // release the reference for all objects in the snapshot + for(int32_t i = 0; i < pIter->numOfObj; ++i) { + char* p= pIter->pCurrent[i]->data; + taosCacheRelease(pCacheObj, (void**) &p, false); + pIter->pCurrent[i] = NULL; + } + + while(1) { + SCacheEntry *pEntry = &pCacheObj->pEntryList[++pIter->entryIndex]; + taosRLockLatch(&pEntry->latch); + + if (pEntry->num == 0) { + taosRUnLockLatch(&pEntry->latch); + continue; + } + + if (pIter->numOfObj < pEntry->num) { + char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pEntry->latch); + return false; + } + + pIter->pCurrent = (SCacheNode **)tmp; + } + + SCacheNode* pNode = pEntry->next; + for (int32_t i = 0; i < pEntry->num; ++i) { + ASSERT(pNode != NULL); + + pIter->pCurrent[i] = pNode; + int32_t ref = T_REF_INC(pIter->pCurrent[i]); + ASSERT(ref >= 1); + + pNode = pNode->pNext; + } + + pIter->numOfObj = pEntry->num; + taosRUnLockLatch(&pEntry->latch); + + pIter->index = -1; + break; + } + } + + pIter->index += 1; + return true; +} + +void* taosCacheIterGetData(const SCacheIter* pIter, size_t* len) { + SCacheNode* pNode = pIter->pCurrent[pIter->index]; + *len = pNode->dataLen; + return pNode->data; +} + +void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { + SCacheNode* pNode = pIter->pCurrent[pIter->index]; + *len = pNode->keyLen; + return pNode->key; +} + +void taosCacheDestroyIter(SCacheIter* pIter) { + tfree(pIter->pCurrent); + tfree(pIter); +} \ No newline at end of file diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index 970f1c23a9..2fca340599 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -6,7 +6,7 @@ #include "tcache.h" // test cache -TEST(testCase, client_cache_test) { +TEST(cacheTest, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); @@ -92,7 +92,7 @@ TEST(testCase, client_cache_test) { taosCacheCleanup(tscMetaCache); } -TEST(testCase, cache_resize_test) { +TEST(cacheTest, cache_iter_test) { const int32_t REFRESH_TIME_IN_SEC = 2; auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL, "test"); @@ -107,6 +107,7 @@ TEST(testCase, cache_resize_test) { int32_t len = sprintf(key, "abc_%7d", i); taosCachePut(pCache, key, strlen(key), data, len, 3600); } + uint64_t endTime = taosGetTimestampUs(); printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); @@ -120,5 +121,22 @@ TEST(testCase, cache_resize_test) { endTime = taosGetTimestampUs(); printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); + int32_t count = 0; + SCacheIter* pIter = taosCacheCreateIter(pCache); + while(taosCacheIterNext(pIter)) { + size_t keyLen = 0; + size_t dataLen = 0; + + char* key1 = static_cast(taosCacheIterGetKey(pIter, &keyLen)); + char* data1 = static_cast(taosCacheIterGetData(pIter, &dataLen)); + +// char d[256] = {0}; +// memcpy(d, data1, dataLen); +// char k[256] = {0}; +// memcpy(k, key1, keyLen); + } + + ASSERT_EQ(count, num); + taosCacheCleanup(pCache); -} \ No newline at end of file +} From 3fbb78a951fd383b6a4882f915393ba867544565 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 18:55:16 +0800 Subject: [PATCH 66/82] io --- include/os/osSysinfo.h | 4 +-- source/dnode/mgmt/impl/src/dndMgmt.c | 7 ++--- source/os/src/osSysinfo.c | 42 ++++++++++++++-------------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index df5ed28a99..3c68615c64 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -43,8 +43,8 @@ int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -bool taosReadProcIO(int64_t *rchars, int64_t *wchars); -bool taosGetProcIO(float *readKB, float *writeKB); +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars); +int32_t taosGetProcIO(float *readKB, float *writeKB); bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); bool taosGetBandSpeed(float *bandSpeedKb); diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 704847894e..6a743fcf48 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -491,10 +491,9 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->disk_total = 4.3; pInfo->net_in = 5.1; pInfo->net_out = 5.2; - pInfo->io_read = 6.1; - pInfo->io_write = 6.2; - pInfo->io_read_disk = 7.1; - pInfo->io_write_disk = 7.2; + taosGetProcIO(&pInfo->io_read, &pInfo->io_write); + pInfo->io_read_disk = 0; + pInfo->io_write_disk = 0; pInfo->req_select = 8; pInfo->req_select_rate = 8.1; pInfo->req_insert = 9; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index bf036354e9..3683943c61 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -124,31 +124,31 @@ bool taosGetBandSpeed(float *bandSpeedKb) { return true; } -bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { +int32_t taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { IO_COUNTERS io_counter; if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { if (readbyte) *readbyte = io_counter.ReadTransferCount; if (writebyte) *writebyte = io_counter.WriteTransferCount; - return true; + return 0; } - return false; + return -1; } -bool taosGetProcIO(float *readKB, float *writeKB) { +int32_t taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; int64_t curReadbyte = 0; int64_t curWritebyte = 0; - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; + if (taosReadProcIO(&curReadbyte, &curWritebyte) != 0) { + return -1; } if (lastReadbyte == -1 || lastWritebyte == -1) { lastReadbyte = curReadbyte; lastWritebyte = curWritebyte; - return false; + return -1; } *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); @@ -159,7 +159,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { lastReadbyte = curReadbyte; lastWritebyte = curWritebyte; - return true; + return 0; } void taosGetSystemInfo() { @@ -259,16 +259,16 @@ void taosGetSystemInfo() { tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); } -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (rchars) *rchars = 0; if (wchars) *wchars = 0; - return true; + return 0; } -bool taosGetProcIO(float *readKB, float *writeKB) { +int32_t taosGetProcIO(float *readKB, float *writeKB) { *readKB = 0; *writeKB = 0; - return true; + return 0; } bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { @@ -631,12 +631,12 @@ bool taosGetBandSpeed(float *bandSpeedKb) { return true; } -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars) { // FILE *fp = fopen(tsProcIOFile, "r"); TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // printf("open file:%s failed", tsProcIOFile); - return false; + return -1; } ssize_t _bytes = 0; @@ -666,27 +666,27 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex < 2) { // printf("read file:%s failed", tsProcIOFile); - return false; + return -1; } - return true; + return 0; } -bool taosGetProcIO(float *readKB, float *writeKB) { +int32_t taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; int64_t curReadbyte = 0; int64_t curWritebyte = 0; - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; + if (taosReadProcIO(&curReadbyte, &curWritebyte) != 0) { + return -1; } if (lastReadbyte == -1 || lastWritebyte == -1) { lastReadbyte = curReadbyte; lastWritebyte = curWritebyte; - return false; + return -1; } *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); @@ -697,7 +697,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { lastReadbyte = curReadbyte; lastWritebyte = curWritebyte; - return true; + return 0; } void taosGetSystemInfo() { From 2bf46ddc54a1282620b4ad725f2a94a68c95008c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 19:12:08 +0800 Subject: [PATCH 67/82] monitor --- include/libs/monitor/monitor.h | 12 ++++----- include/os/osSysinfo.h | 4 +-- source/dnode/mgmt/impl/inc/dndMnode.h | 1 + source/dnode/mgmt/impl/src/dndMgmt.c | 33 +++++++++++------------ source/dnode/mgmt/impl/src/dndMnode.c | 7 +++++ source/os/src/osSysinfo.c | 38 +++++++++++++-------------- 6 files changed, 51 insertions(+), 44 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 487b2bfe88..00cb7c3dbc 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -94,12 +94,12 @@ typedef struct { float disk_engine; // GB float disk_used; // GB float disk_total; // GB - float net_in; // Kb/s - float net_out; // Kb/s - float io_read; // Mb/s - float io_write; // Mb/s - float io_read_disk; // Mb/s - float io_write_disk; // Mb/s + int64_t net_in; + int64_t net_out; + float io_read; + float io_write; + float io_read_disk; + float io_write_disk; int32_t req_select; float req_select_rate; int32_t req_insert; diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 3c68615c64..e14dba8269 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -45,8 +45,8 @@ int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars); int32_t taosGetProcIO(float *readKB, float *writeKB); -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); -bool taosGetBandSpeed(float *bandSpeedKb); +int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); +int32_t taosGetBandSpeed(float *bandSpeedKb); int32_t taosSystem(const char *cmd); void taosKillSystem(); diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h index 0f03bb3832..0aee3b4b43 100644 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -34,6 +34,7 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo); +int8_t dndIsMnode(SDnode *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 6a743fcf48..60cfdc299c 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -486,26 +486,25 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { taosGetProcMemory(&pInfo->mem_engine); taosGetSysMemory(&pInfo->mem_system); pInfo->mem_total = tsTotalMemoryKB; - pInfo->disk_engine = 4.1; - pInfo->disk_used = 4.2; - pInfo->disk_total = 4.3; - pInfo->net_in = 5.1; - pInfo->net_out = 5.2; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used / (1024 * 1024 * 1024.0); + pInfo->disk_total = tsDataSpace.size.avail / (1024 * 1024 * 1024.0); + taosGetCardInfo(NULL, &pInfo->net_in, &pInfo->net_out); taosGetProcIO(&pInfo->io_read, &pInfo->io_write); pInfo->io_read_disk = 0; pInfo->io_write_disk = 0; - pInfo->req_select = 8; - pInfo->req_select_rate = 8.1; - pInfo->req_insert = 9; - pInfo->req_insert_success = 10; - pInfo->req_insert_rate = 10.1; - pInfo->req_insert_batch = 11; - pInfo->req_insert_batch_success = 12; - pInfo->req_insert_batch_rate = 12.3; - pInfo->errors = 4; - pInfo->vnodes_num = 5; - pInfo->masters = 6; - pInfo->has_mnode = 1; + pInfo->req_select = 0; + pInfo->req_select_rate = 0; + pInfo->req_insert = 0; + pInfo->req_insert_success = 0; + pInfo->req_insert_rate = 0; + pInfo->req_insert_batch = 0; + pInfo->req_insert_batch_success = 0; + pInfo->req_insert_batch_rate = 0; + pInfo->errors = 0; + pInfo->vnodes_num = 0; + pInfo->masters = 0; + pInfo->has_mnode = dndIsMnode(pDnode); } static void dndSendMonitorReport(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 6cb117867f..47e74b5c57 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -639,4 +639,11 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); dndReleaseMnode(pDnode, pMnode); return code; +} + +int8_t dndIsMnode(SDnode *pDnode) { + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) return 0; + dndReleaseMnode(pDnode, pMnode); + return 1; } \ No newline at end of file diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 3683943c61..c674aeaca2 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -112,16 +112,16 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { +int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { if (bytes) *bytes = 0; if (rbytes) *rbytes = 0; if (tbytes) *tbytes = 0; - return true; + return 0; } -bool taosGetBandSpeed(float *bandSpeedKb) { +int32_t taosGetBandSpeed(float *bandSpeedKb) { *bandSpeedKb = 0; - return true; + return 0; } int32_t taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { @@ -271,16 +271,16 @@ int32_t taosGetProcIO(float *readKB, float *writeKB) { return 0; } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { +int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { if (bytes) *bytes = 0; if (rbytes) *rbytes = 0; if (tbytes) *tbytes = 0; - return true; + return 0; } -bool taosGetBandSpeed(float *bandSpeedKb) { +int32_t taosGetBandSpeed(float *bandSpeedKb) { *bandSpeedKb = 0; - return true; + return 0; } int32_t taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { @@ -544,13 +544,13 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - *bytes = 0; +int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { + if (bytes) *bytes = 0; // FILE *fp = fopen(tsSysNetFile, "r"); TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // printf("open file:%s failed", tsSysNetFile); - return false; + return -1; } ssize_t _bytes = 0; @@ -586,37 +586,37 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets); if (rbytes) *rbytes = o_rbytes; if (tbytes) *tbytes = o_tbytes; - *bytes += (o_rbytes + o_tbytes); + if (bytes) *bytes += (o_rbytes + o_tbytes); } if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -bool taosGetBandSpeed(float *bandSpeedKb) { +int32_t taosGetBandSpeed(float *bandSpeedKb) { static int64_t lastBytes = 0; static time_t lastTime = 0; int64_t curBytes = 0; time_t curTime = time(NULL); - if (!taosGetCardInfo(&curBytes, NULL, NULL)) { - return false; + if (taosGetCardInfo(&curBytes, NULL, NULL) != 0) { + return -1; } if (lastTime == 0 || lastBytes == 0) { lastTime = curTime; lastBytes = curBytes; *bandSpeedKb = 0; - return true; + return 0; } if (lastTime >= curTime || lastBytes > curBytes) { lastTime = curTime; lastBytes = curBytes; *bandSpeedKb = 0; - return true; + return 0; } double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb @@ -628,7 +628,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) { lastTime = curTime; lastBytes = curBytes; - return true; + return 0; } int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars) { From 4bfece6ee1724a41bd85c2f71c7d56137c8bcaf7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Mar 2022 19:23:11 +0800 Subject: [PATCH 68/82] [td-13039] update test. --- source/libs/executor/test/executorTests.cpp | 666 +++++++++++++++++--- 1 file changed, 582 insertions(+), 84 deletions(-) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 54cfa57595..b5d8bb8019 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -224,90 +224,588 @@ int main(int argc, char** argv) { TEST(testCase, build_executor_tree_Test) { const char* msg = "{\n" - "\t\"Id\":\t{\n" - "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" - "\t\t\"TemplateId\":\t0,\n" - "\t\t\"SubplanId\":\t0\n" - "\t},\n" - "\t\"Node\":\t{\n" - "\t\t\"Name\":\t\"TableScan\",\n" - "\t\t\"Targets\":\t[{\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}],\n" - "\t\t\"InputSchema\":\t[{\n" - "\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t}],\n" - "\t\t\"TableScan\":\t{\n" - "\t\t\t\"TableId\":\t1,\n" - "\t\t\t\"TableType\":\t2,\n" - "\t\t\t\"Flag\":\t0,\n" - "\t\t\t\"Window\":\t{\n" - "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" - "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" - "\t\t\t}\n" - "\t\t}\n" - "\t},\n" - "\t\"DataSink\":\t{\n" - "\t\t\"Name\":\t\"Dispatch\",\n" - "\t\t\"Dispatch\":\t{\n" - "\t\t}\n" - "\t}\n" + " \"Type\": \"33\",\n" + " \"Name\": \"PhysiProject\",\n" + " \"PhysiProject\": {\n" + " \"OutputDataBlockDesc\": {\n" + " \"Type\": \"19\",\n" + " \"Name\": \"TupleDesc\",\n" + " \"TupleDesc\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"Slots\": [\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"0\",\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"1\",\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"2\",\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"3\",\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"4\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"5\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " },\n" + " \"Children\": [\n" + " {\n" + " \"Type\": \"30\",\n" + " \"Name\": \"PhysiTableScan\",\n" + " \"PhysiTableScan\": {\n" + " \"OutputDataBlockDesc\": {\n" + " \"Type\": \"19\",\n" + " \"Name\": \"TupleDesc\",\n" + " \"TupleDesc\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"Slots\": [\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"0\",\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"1\",\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"2\",\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"3\",\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"4\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"5\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " },\n" + " \"ScanCols\": [\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"ts\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"1\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"ts\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"1\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"AliasName\": \"c1\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"2\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c1\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"2\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"AliasName\": \"c2\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"3\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c2\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"3\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c3\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"4\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c3\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"4\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c4\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"5\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c4\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"5\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c5\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"6\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c5\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + " ],\n" + " \"TableId\": \"1\",\n" + " \"TableType\": \"3\",\n" + " \"ScanOrder\": \"1\",\n" + " \"ScanCount\": \"1\",\n" + " \"ReverseScanCount\": \"0\",\n" + " \"ScanFlag\": \"0\",\n" + " \"StartKey\": \"-9223372036854775808\",\n" + " \"EndKey\": \"9223372036854775807\"\n" + " }\n" + " }\n" + " ],\n" + " \"Projections\": [\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"0\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"ts\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"1\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"ts\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"1\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"AliasName\": \"c1\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"2\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c1\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"1\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"2\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"AliasName\": \"c2\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"3\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c2\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"2\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"3\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c3\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"4\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c3\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"3\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"4\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c4\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"5\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c4\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"4\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"5\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c5\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"6\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c5\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"5\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + " ]\n" + " }\n" "}"; SExecTaskInfo* pTaskInfo = nullptr; From 54644652f1141d2503091c153ac9ddacd71cd9b6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Mar 2022 20:23:53 +0800 Subject: [PATCH 69/82] mnode monitor --- source/dnode/mnode/impl/inc/mndInt.h | 8 +- source/dnode/mnode/impl/inc/mndProfile.h | 1 + source/dnode/mnode/impl/src/mndMnode.c | 4 + source/dnode/mnode/impl/src/mndProfile.c | 2 + source/dnode/mnode/impl/src/mndTelem.c | 4 +- source/dnode/mnode/impl/src/mnode.c | 147 ++++++++++++++++++++--- source/libs/monitor/src/monitor.c | 4 +- 7 files changed, 145 insertions(+), 25 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 929d47184b..8e9f47d560 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -85,6 +85,11 @@ typedef struct { ESyncState state; } SSyncMgmt; +typedef struct { + int64_t expireTimeMS; + int64_t timeseriesAllowed; +} SGrantInfo; + typedef struct SMnode { int32_t dnodeId; int64_t clusterId; @@ -105,6 +110,7 @@ typedef struct SMnode { STelemMgmt telemMgmt; SSyncMgmt syncMgmt; SHashObj *infosMeta; + SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; @@ -120,7 +126,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len); -int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndProfile.h b/source/dnode/mnode/impl/inc/mndProfile.h index df857be73e..d5954e1ac6 100644 --- a/source/dnode/mnode/impl/inc/mndProfile.h +++ b/source/dnode/mnode/impl/inc/mndProfile.h @@ -24,6 +24,7 @@ extern "C" { int32_t mndInitProfile(SMnode *pMnode); void mndCleanupProfile(SMnode *pMnode); +int32_t mndGetNumOfConnections(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index cbb31bc504..5866aa6346 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); if (pIter == NULL) break; + ESyncRole lastRole = pObj->role; if (pObj->id == 1) { pObj->role = TAOS_SYNC_STATE_LEADER; } else { pObj->role = TAOS_SYNC_STATE_CANDIDATE; } + if (pObj->role != lastRole) { + pObj->roleTime = taosGetTimestampMs(); + } sdbRelease(pSdb, pObj); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e313c4d676..88a176bb0e 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -916,3 +916,5 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } + +int32_t mndGetNumOfConnections(SMnode *pMnode) { return taosHashGetSize(pMnode->profileMgmt.cache->pHashTable); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 968319e7b4..0e2141a4d7 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -18,15 +18,15 @@ #include "mndCluster.h" #include "mndSync.h" #include "tbuffer.h" -#include "tjson.h" #include "thttp.h" +#include "tjson.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { SMnodeLoad load = {0}; - if (mndGetLoad(pMnode, &load) != 0) return; + mndGetLoad(pMnode, &load); tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index c95857a2d4..55a61b75bc 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + mndUpdateMnodeRole(pMnode); mDebug("mnode open successfully "); return pMnode; } @@ -385,26 +386,6 @@ void mndDestroy(const char *path) { mDebug("mnode is destroyed"); } -int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { - pLoad->numOfDnode = 0; - pLoad->numOfMnode = 0; - pLoad->numOfVgroup = 0; - pLoad->numOfDatabase = 0; - pLoad->numOfSuperTable = 0; - pLoad->numOfChildTable = 0; - pLoad->numOfColumn = 0; - pLoad->totalPoints = 0; - pLoad->totalStorage = 0; - pLoad->compStorage = 0; - - return 0; -} - -int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - return 0; - } - SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); if (pMsg == NULL) { @@ -523,3 +504,129 @@ uint64_t mndGenerateUid(char *name, int32_t len) { } } while (true); } + +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { + memset(pLoad, 0, sizeof(SMnodeLoad)); + + SSdb *pSdb = pMnode->pSdb; + pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE); + pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE); + pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP); + pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB); + pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB); + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + pLoad->numOfChildTable += pVgroup->numOfTables; + pLoad->numOfColumn += pVgroup->numOfTimeSeries; + pLoad->totalPoints += pVgroup->pointsWritten; + pLoad->totalStorage += pVgroup->totalStorage; + pLoad->compStorage += pVgroup->compStorage; + + sdbRelease(pSdb, pVgroup); + } +} + +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + SSdb *pSdb = pMnode->pSdb; + int64_t ms = taosGetTimestampMs(); + + pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); + pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc)); + pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc)); + if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) { + return -1; + } + + // cluster info + tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); + pClusterInfo->monitor_interval = tsMonitorInterval; + pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); + + void *pIter = NULL; + while (1) { + SDnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SMonDnodeDesc desc = {0}; + desc.dnode_id = pObj->id; + tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep)); + if (mndIsDnodeOnline(pMnode, pObj, ms)) { + tstrncpy(desc.status, "ready", sizeof(desc.status)); + } else { + tstrncpy(desc.status, "offline", sizeof(desc.status)); + } + taosArrayPush(pClusterInfo->dnodes, &desc); + sdbRelease(pSdb, pObj); + } + + pIter = NULL; + while (1) { + SMnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SMonMnodeDesc desc = {0}; + desc.mnode_id = pObj->id; + tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep)); + tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role)); + taosArrayPush(pClusterInfo->mnodes, &desc); + sdbRelease(pSdb, pObj); + + if (pObj->role == TAOS_SYNC_STATE_LEADER) { + pClusterInfo->first_ep_dnode_id = pObj->id; + tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); + pClusterInfo->master_uptime = (ms - pObj->roleTime) / (86400000.0f); + } + } + + // vgroup info + pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + pClusterInfo->vgroups_total++; + + SMonVgroupDesc desc = {0}; + desc.vgroup_id = pVgroup->vgId; + strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name)); + desc.tables_num = pVgroup->numOfTables; + pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries; + tstrncpy(desc.status, "unsynced", sizeof(desc.status)); + for (int32_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; + pVnDesc->dnode_id = pVgid->dnodeId; + tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role)); + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + tstrncpy(desc.status, "ready", sizeof(desc.status)); + pClusterInfo->vgroups_alive++; + } + if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { + pClusterInfo->vnodes_alive++; + } + pClusterInfo->vnodes_total++; + } + + taosArrayPush(pVgroupInfo->vgroups, &desc); + sdbRelease(pSdb, pVgroup); + } + + // grant info + pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; + pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; + if (pMnode->grant.expireTimeMS == 0) { + pGrantInfo->expire_time = INT32_MAX; + pGrantInfo->timeseries_total = INT32_MAX; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index ecf9da218b..14375a4ef6 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -128,11 +128,11 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { SJson *pMnodesJson = tjsonAddArrayToObject(pJson, "mnodes"); if (pMnodesJson == NULL) return; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { SJson *pMnodeJson = tjsonCreateObject(); if (pMnodeJson == NULL) continue; - SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i); + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id); tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep); tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role); From 35b3edaea562f5c01d7f63d17634bd5479d534d0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 4 Mar 2022 00:04:57 +0800 Subject: [PATCH 70/82] [TD-13756]: file system stat access func. --- include/os/osDir.h | 2 +- include/os/osFile.h | 24 +++-- source/common/test/commonTests.cpp | 4 +- source/common/test/tmsgTest.cpp | 2 +- source/dnode/vnode/src/tq/tqMetaStore.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 15 ++- source/libs/CMakeLists.txt | 1 - source/libs/catalog/test/catalogTests.cpp | 6 +- source/libs/executor/src/executorimpl.c | 8 +- source/libs/executor/test/executorTests.cpp | 6 +- source/libs/executor/test/sortTests.cpp | 2 +- .../index/src/index_fst_counting_writer.c | 18 ++-- source/libs/parser/test/mockCatalog.cpp | 6 +- source/libs/parser/test/parserTests.cpp | 4 +- source/libs/parser/test/plannerTest.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 4 +- .../libs/scalar/test/filter/filterTests.cpp | 4 +- .../libs/scalar/test/scalar/scalarTests.cpp | 4 +- source/libs/scheduler/test/schedulerTests.cpp | 24 ++--- .../sync/test/syncIOSendMsgClientTest.cpp | 2 +- .../sync/test/syncIOSendMsgServerTest.cpp | 2 +- source/libs/sync/test/syncIOSendMsgTest.cpp | 2 +- source/libs/sync/test/syncIOTickPingTest.cpp | 2 +- source/libs/sync/test/syncIOTickQTest.cpp | 2 +- source/libs/sync/test/syncRaftStoreTest.cpp | 4 +- source/libs/sync/test/syncTest.cpp | 2 +- source/libs/tdb/src/db/tdb.c | 2 +- source/libs/tdb/src/db/tdbUtil.c | 43 ++++---- source/libs/tdb/src/inc/tdbUtil.h | 8 +- source/libs/tdb/test/tdbTest.cpp | 2 +- source/libs/tfs/src/tfs.c | 11 +-- source/libs/tfs/test/tfsTest.cpp | 98 +++++++++---------- source/libs/wal/src/walMeta.c | 22 ++--- source/os/src/osDir.c | 2 +- source/os/src/osFile.c | 39 ++++++++ source/util/test/cacheTest.cpp | 2 +- source/util/test/encodeTest.cpp | 2 +- source/util/test/freelistTest.cpp | 2 +- source/util/test/hashTest.cpp | 4 +- tools/shell/src/backup/shellImport.c | 8 +- 42 files changed, 221 insertions(+), 184 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index eda83ea0ea..223c603352 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -21,7 +21,7 @@ extern "C" { #endif void taosRemoveDir(const char *dirname); -int32_t taosDirExist(char *dirname); +bool taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); diff --git a/include/os/osFile.h b/include/os/osFile.h index cbd5d693c5..7e3a5277c8 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -25,8 +25,12 @@ extern "C" { #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID - // #define close CLOSE_FUNC_TAOS_FORBID - // #define fclose FCLOSE_FUNC_TAOS_FORBID + #define access ACCESS_FUNC_TAOS_FORBID + #define stat STAT_FUNC_TAOS_FORBID + #define lstat LSTAT_FUNC_TAOS_FORBID + #define fstat FSTAT_FUNC_TAOS_FORBID + #define close CLOSE_FUNC_TAOS_FORBID + #define fclose FCLOSE_FUNC_TAOS_FORBID #endif #ifndef PATH_MAX @@ -44,6 +48,12 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +#define TD_FILE_ACCESS_EXIST_OK 0x1 +#define TD_FILE_ACCESS_READ_OK 0x2 +#define TD_FILE_ACCESS_WRITE_OK 0x4 +bool taosCheckAccessFile(const char *pathname, int mode); int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); @@ -51,9 +61,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); - -TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); +bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); @@ -62,7 +72,7 @@ int32_t taosFsyncFile(TdFilePtr pFile); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); -void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); @@ -71,7 +81,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile); int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); -void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); @@ -79,7 +89,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); -int taosGetErrorFile(TdFilePtr pFile); +int32_t taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 4821d60875..563efe4d54 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,6 +1,4 @@ -#include "tcommon.h" #include -#include #include #pragma GCC diagnostic push @@ -10,6 +8,8 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tep.h" +#include "tcommon.h" #include "taos.h" #include "tvariant.h" #include "tdef.h" diff --git a/source/common/test/tmsgTest.cpp b/source/common/test/tmsgTest.cpp index ca33d24a8c..e8c3284ea5 100644 --- a/source/common/test/tmsgTest.cpp +++ b/source/common/test/tmsgTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tmsg.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 6d5085ee74..ce00c98ff9 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -90,7 +90,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F char name[pathLen + 10]; strcpy(name, path); - if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { + if (!taosDirExist(name) && taosMkDir(name) != 0) { terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; tqError("failed to create dir:%s since %s ", name, terrstr()); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2e7116969e..7a95820115 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -580,7 +580,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh)); SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); - if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) { tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); @@ -614,7 +614,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh)); SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith); - if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) { + if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) { tsdbInitDFileEx(pWSmalF, pRSmalF); if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index dcc9700d83..411a166caa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -314,7 +314,7 @@ int tsdbOpenFS(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); tsdbGetRtnSnap(pRepo, &pRepo->rtn); - if (access(current, F_OK) == 0) { + if (taosCheckExistFile(current)) { if (tsdbOpenFSFromCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4d70c429ff..74fb8c1c1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -443,25 +443,24 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { - struct stat dfstat; SDFile df; tsdbInitDFileEx(&df, pDFile); - if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) { tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); return 0; } - - if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) { + int64_t file_size = 0; + if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (pDFile->info.size < dfstat.st_size) { + if (pDFile->info.size < file_size) { // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; @@ -480,10 +479,10 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { tsdbCloseDFile(&df); tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), - dfstat.st_size, pDFile->info.size); - } else if (pDFile->info.size > dfstat.st_size) { + file_size, pDFile->info.size); + } else if (pDFile->info.size > file_size) { tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 78625d1eed..d766f3cbf1 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,4 +1,3 @@ -add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b417a645be..b05fc7812a 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -14,9 +14,7 @@ */ #include -#include #include -#include "os.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -24,8 +22,10 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wformat" +#include -#include "addr_any.h" +#include "os.h" +#include "tglobal.h" #include "catalog.h" #include "stub.h" #include "taos.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c400ded1c1..d449d7fac4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2215,10 +2215,10 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); if (pColInfoData) { - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - if (f) { - fclose(f); - *(FILE **)pColInfoData->pData = NULL; + TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData; // TODO refactor + if (pFile != NULL) { + taosCloseFile(&pFile); + *(TdFilePtr *)pColInfoData->pData = NULL; } } } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 262232ceb1..7d3665a8cc 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include #include -#include #include #pragma GCC diagnostic push @@ -26,6 +23,9 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tglobal.h" +#include "executorimpl.h" +#include "function.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index fc366e4cc8..612b0705ea 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -26,6 +25,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "executorimpl.h" #include "executor.h" #include "stub.h" #include "taos.h" diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 16d9893778..8cb2ff9246 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -63,9 +63,9 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxGetSize(WriterCtx* ctx) { if (ctx->type == TFile) { - struct stat fstat; - stat(ctx->file.buf, &fstat); - return fstat.st_size; + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + return (int)file_size; } return 0; } @@ -99,9 +99,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + int64_t file_size = 0; + taosFStatFile(ctx->file.pFile, &file_size, NULL); + ctx->file.size = (int)file_size; #ifdef USE_MMAP ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif @@ -142,8 +142,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { #endif } if (ctx->file.readOnly == false) { - struct stat fstat; - stat(ctx->file.buf, &fstat); + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + // struct stat fstat; + // stat(ctx->file.buf, &fstat); // indexError("write file size: %d", (int)(fstat.st_size)); } if (remove) { unlink(ctx->file.buf); } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 457d586ea4..e626fc68ae 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -13,19 +13,17 @@ * along with this program. If not, see . */ -#include "mockCatalog.h" - #include - #include "stub.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat" -#include "addr_any.h" +#include #pragma GCC diagnostic pop +#include "mockCatalog.h" namespace { void generateTestT1(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b971760132..b80efe09ed 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include #include #include -#include "tglobal.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -25,6 +23,8 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" +#include "tglobal.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 43daff9fec..095e2f4f97 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -25,6 +24,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8ad5a76388..94d4260696 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,9 +25,11 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" @@ -37,7 +38,6 @@ #include "planner.h" #include "qworker.h" #include "stub.h" -#include "addr_any.h" #include "executor.h" #include "dataSinkMgt.h" diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e..13829618a2 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,15 +25,16 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" #include "tep.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" #include "nodes.h" #include "tlog.h" diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088..8eef1836a5 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,15 +25,16 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" #include "tep.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" #include "nodes.h" #include "tlog.h" diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 11ed3335e6..1a34c20ba0 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -14,19 +14,8 @@ */ #include -#include #include -#include "os.h" - -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "catalog.h" -#include "scheduler.h" -#include "tep.h" -#include "trpc.h" - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" @@ -34,10 +23,21 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wreturn-type" #pragma GCC diagnostic ignored "-Wformat" +#include + +#include "os.h" + +#include "tglobal.h" +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" #include "schedulerInt.h" #include "stub.h" -#include "addr_any.h" #include "tref.h" namespace { diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 9e8c7c8b65..0d06c4f811 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOSendMsgServerTest.cpp index 8af9344ed6..1582e097d3 100644 --- a/source/libs/sync/test/syncIOSendMsgServerTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgServerTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index a297981ee5..9b2cfcf1d8 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 1559b57585..42b9a73432 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 90304079e3..3d31c596bf 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index e533b89a92..9cfbe5a4ea 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,6 +1,6 @@ -#include "syncRaftStore.h" #include -#include "gtest/gtest.h" +#include +#include "syncRaftStore.h" #include "syncIO.h" #include "syncInt.h" diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 0b397ef921..c1c5658aba 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c index cc3b7fa6b9..65d4cf80cc 100644 --- a/source/libs/tdb/src/db/tdb.c +++ b/source/libs/tdb/src/db/tdb.c @@ -90,7 +90,7 @@ int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { // get page file from the env, if not opened yet, open it pPgFile = NULL; snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); - fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0); + fileExist = taosCheckExistFile(fname); if (fileExist) { tdbGnrtFileID(dbfname, fileid, false); pPgFile = tdbEnvGetPageFile(pEnv, fileid); diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index fa9a3297da..fe0f3befd6 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -16,16 +16,16 @@ #include "tdbInt.h" int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { - struct stat statbuf; + int64_t stDev = 0, stIno = 0; - if (stat(fname, &statbuf) < 0) { + if (taosDevInoFile(fname, &stDev, &stIno) < 0) { return -1; } memset(fileid, 0, TDB_FILE_ID_LEN); - ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; - ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[0] = stDev; + ((uint64_t *)fileid)[1] = stIno; if (unique) { ((uint64_t *)fileid)[2] = rand(); } @@ -33,35 +33,34 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { return 0; } -int tdbCheckFileAccess(const char *pathname, int mode) { - int flags = 0; +// int tdbCheckFileAccess(const char *pathname, int mode) { +// int flags = 0; - if (mode & TDB_F_OK) { - flags |= F_OK; - } +// if (mode & TDB_F_OK) { +// flags |= F_OK; +// } - if (mode & TDB_R_OK) { - flags |= R_OK; - } +// if (mode & TDB_R_OK) { +// flags |= R_OK; +// } - if (mode & TDB_W_OK) { - flags |= W_OK; - } +// if (mode & TDB_W_OK) { +// flags |= W_OK; +// } - return access(pathname, flags); -} +// return access(pathname, flags); +// } int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { - struct stat st; int ret; - - ret = stat(fname, &st); + int64_t file_size = 0; + ret = taosStatFile(fname, &file_size, NULL); if (ret != 0) { return -1; } - ASSERT(st.st_size % pgSize == 0); + ASSERT(file_size % pgSize == 0); - *pSize = st.st_size / pgSize; + *pSize = file_size / pgSize; return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 8108e5aba6..ca05790f77 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -30,10 +30,10 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -#define TDB_F_OK 0x1 -#define TDB_R_OK 0x2 -#define TDB_W_OK 0x4 -int tdbCheckFileAccess(const char *pathname, int mode); +// #define TDB_F_OK 0x1 +// #define TDB_R_OK 0x2 +// #define TDB_W_OK 0x4 +// int tdbCheckFileAccess(const char *pathname, int mode); int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5ab0b4c0f1..ad550c7804 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tdb.h" diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 944e67c863..26fa90bdef 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -389,7 +389,6 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; - struct stat pstat; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); @@ -422,19 +421,13 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { return -1; } - if (access(dirName, W_OK | R_OK | F_OK) != 0) { + if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; } - if (stat(dirName, &pstat) < 0) { - fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!S_ISDIR(pstat.st_mode)) { + if (!taosIsDir(dirName)) { fError("failed to mount %s to FS since not a directory", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index af66304f84..9cb914a670 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -119,29 +119,29 @@ TEST_F(TfsTest, 03_Dir) { char p1[] = "p1"; char ap1[128] = {0}; snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap1), 0); + EXPECT_NE(taosDirExist(ap1), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap1), 0); + EXPECT_EQ(taosDirExist(ap1), 1); char p2[] = "p2"; char ap2[128] = {0}; snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2); SDiskID did = {0}; - EXPECT_NE(taosDirExist(ap2), 0); + EXPECT_NE(taosDirExist(ap2), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(ap2), 0); + EXPECT_EQ(taosDirExist(ap2), 1); char p3[] = "p3/p2/p1/p0"; char ap3[128] = {0}; snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(ap3), 0); + EXPECT_EQ(taosDirExist(ap3), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -149,12 +149,12 @@ TEST_F(TfsTest, 03_Dir) { char ap4[128] = {0}; snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(ap4), 0); + EXPECT_EQ(taosDirExist(ap4), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); tfsClose(pTfs); } @@ -251,9 +251,9 @@ TEST_F(TfsTest, 04_File) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); { STfsDir *pDir = tfsOpendir(pTfs, "t3"); @@ -529,35 +529,35 @@ TEST_F(TfsTest, 05_MultiDisk) { snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1); char ap23[128] = {0}; snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap00), 0); - EXPECT_EQ(taosDirExist(ap01), 0); - EXPECT_EQ(taosDirExist(ap10), 0); - EXPECT_EQ(taosDirExist(ap11), 0); - EXPECT_EQ(taosDirExist(ap12), 0); - EXPECT_EQ(taosDirExist(ap20), 0); - EXPECT_EQ(taosDirExist(ap21), 0); - EXPECT_EQ(taosDirExist(ap22), 0); - EXPECT_EQ(taosDirExist(ap23), 0); + EXPECT_EQ(taosDirExist(ap00), 1); + EXPECT_EQ(taosDirExist(ap01), 1); + EXPECT_EQ(taosDirExist(ap10), 1); + EXPECT_EQ(taosDirExist(ap11), 1); + EXPECT_EQ(taosDirExist(ap12), 1); + EXPECT_EQ(taosDirExist(ap20), 1); + EXPECT_EQ(taosDirExist(ap21), 1); + EXPECT_EQ(taosDirExist(ap22), 1); + EXPECT_EQ(taosDirExist(ap23), 1); EXPECT_EQ(tfsRmdir(pTfs, p1), 0); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); char p2[] = "p2"; char _ap21[128] = {0}; @@ -565,22 +565,22 @@ TEST_F(TfsTest, 05_MultiDisk) { SDiskID did = {0}; did.level = 2; did.id = 1; - EXPECT_NE(taosDirExist(_ap21), 0); + EXPECT_NE(taosDirExist(_ap21), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(_ap21), 0); + EXPECT_EQ(taosDirExist(_ap21), 1); char p3[] = "p3/p2/p1/p0"; char _ap12[128] = {0}; snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3); did.level = 1; did.id = 2; - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(_ap12), 0); + EXPECT_EQ(taosDirExist(_ap12), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -590,12 +590,12 @@ TEST_F(TfsTest, 05_MultiDisk) { did.level = 2; did.id = 2; - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(_ap22), 0); + EXPECT_EQ(taosDirExist(_ap22), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); } //------------- File -----------------// @@ -660,7 +660,7 @@ TEST_F(TfsTest, 05_MultiDisk) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); { @@ -678,7 +678,7 @@ TEST_F(TfsTest, 05_MultiDisk) { tfsClosedir(pDir); } - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); EXPECT_GT(tfsCopyFile(&f1, &f2), 0); { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ae0b0bd849..e64260c541 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -64,10 +64,10 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); - int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); - pLastFileInfo->fileSize = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int readSize = TMIN(WAL_MAX_SIZE + 2, file_size); + pLastFileInfo->fileSize = file_size; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pFile == NULL) { @@ -177,11 +177,11 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); - if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) { - pLastFileInfo->fileSize = statbuf.st_size; + if (oldSz != newSz || pLastFileInfo->fileSize != file_size) { + pLastFileInfo->fileSize = file_size; pWal->vers.lastVer = walScanLogGetLastVer(pWal); ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; ASSERT(pWal->vers.lastVer != -1); @@ -395,9 +395,9 @@ int walLoadMeta(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); // read metafile - struct stat statbuf; - stat(fnameStr, &statbuf); - int size = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int size = (int)file_size; char* buf = malloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 91ef97e66b..e0f5f80dab 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) { //printf("dir:%s is removed\n", dirname); } -int32_t taosDirExist(char *dirname) { return access(dirname, F_OK); } +bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2e9b52f88..37004e9d70 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -186,6 +186,27 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; #endif } +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (stDev != NULL) { + *stDev = fileStat.st_dev; + } + + if (stIno != NULL) { + *stIno = fileStat.st_ino; + } + + return 0; +#endif +} void autoDelFileListAdd(const char *path) { return; } @@ -733,3 +754,21 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } +bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { + int flags = 0; + + if (tdFileAccessOptions & TD_FILE_ACCESS_EXIST_OK) { + flags |= F_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_READ_OK) { + flags |= R_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { + flags |= W_OK; + } + + return access(pathname, flags) == 0; +} +bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; \ No newline at end of file diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index 970f1c23a9..4dde374bdd 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -1,7 +1,7 @@ -#include "os.h" #include #include +#include "os.h" #include "taos.h" #include "tcache.h" diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 5505a6207f..7b993ebf35 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tencode.h" diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index cf11d6b5bf..7fd3d693fb 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tfreelist.h" diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index ac1bae2434..4c6ee8d10c 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -1,9 +1,9 @@ -#include "os.h" #include #include -#include #include +#include "os.h" +#include "taosdef.h" #include "thash.h" #include "taos.h" diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index c0ab7ef461..36489a448b 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -93,8 +93,7 @@ static void shellCheckTablesSQLFile(const char *directoryName) { sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - struct stat fstat; - if (stat(shellTablesSQLFile, &fstat) < 0) { + if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) { shellTablesSQLFile[0] = 0; } } @@ -109,13 +108,12 @@ static void shellMallocSQLFiles() static void shellGetDirectoryFileList(char *inputDir) { - struct stat fileStat; - if (stat(inputDir, &fileStat) < 0) { + if (!taosDirExist(inputDir)) { fprintf(stderr, "ERROR: %s not exist\n", inputDir); exit(0); } - if (fileStat.st_mode & S_IFDIR) { + if (taosIsDir(inputDir)) { shellCheckTablesSQLFile(inputDir); shellSQLFileNum = shellGetFilesNum(inputDir, "sql"); int totalSQLFileNum = shellSQLFileNum; From bc25fed811f2c184a6542301a6d5b814da612a2b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 10:15:48 +0800 Subject: [PATCH 71/82] minor changes --- source/dnode/mnode/impl/src/mnode.c | 2 ++ source/os/src/osSysinfo.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 55a61b75bc..213b4ac9d7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -533,6 +533,8 @@ void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { + if (!mndIsMaster(pMnode)) return; + SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index c674aeaca2..12952141a4 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -539,7 +539,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } else { diskSize->total = info.f_blocks * info.f_frsize; diskSize->avail = info.f_bavail * info.f_frsize; - diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + diskSize->used = diskSize->total - diskSize->avail; return 0; } } From a6b168c79d21a09c3e882c3b86266a4db3af0a4f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 10:20:34 +0800 Subject: [PATCH 72/82] fix compile error --- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mnode.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 5866aa6346..9a6297a0f4 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -99,7 +99,7 @@ void mndUpdateMnodeRole(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); if (pIter == NULL) break; - ESyncRole lastRole = pObj->role; + ESyncState lastRole = pObj->role; if (pObj->id == 1) { pObj->role = TAOS_SYNC_STATE_LEADER; } else { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 213b4ac9d7..64c7a66bf9 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -533,7 +533,7 @@ void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { - if (!mndIsMaster(pMnode)) return; + if (!mndIsMaster(pMnode)) return -1; SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); From 92da01a8ba0872f64f460b756cbaff41ea0487d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Mar 2022 13:25:39 +0800 Subject: [PATCH 73/82] [td-13039] refactor. --- include/common/tcommon.h | 18 ------- include/common/{tep.h => tdatablock.h} | 0 include/libs/scalar/filter.h | 5 +- include/libs/sync/sync.h | 2 +- include/util/thash.h | 15 ++---- source/client/inc/clientInt.h | 4 +- source/client/src/clientImpl.c | 2 +- source/client/src/tmq.c | 2 +- source/common/src/{tep.c => tdatablock.c} | 2 +- source/common/src/tglobal.c | 2 +- source/common/test/commonTests.cpp | 4 +- source/dnode/mgmt/impl/inc/dndInt.h | 6 +-- .../dnode/mgmt/impl/test/sut/src/client.cpp | 2 +- source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/executor/src/executorimpl.c | 10 ++-- source/libs/executor/src/tsort.c | 4 +- source/libs/executor/test/executorTests.cpp | 12 ++--- source/libs/executor/test/sortTests.cpp | 2 +- source/libs/function/src/taggfunction.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 17 +++--- source/libs/scalar/inc/filterInt.h | 10 ++-- source/libs/scalar/src/filter.c | 6 +-- source/libs/scalar/src/scalar.c | 10 ++-- source/libs/scalar/src/sclvector.c | 12 ++--- .../libs/scalar/test/filter/filterTests.cpp | 16 +++--- .../libs/scalar/test/scalar/scalarTests.cpp | 12 ++--- source/libs/scheduler/test/schedulerTests.cpp | 8 +-- source/libs/sync/src/syncIO.c | 2 +- source/libs/transport/test/pushClient.c | 2 +- source/libs/transport/test/rclient.c | 2 +- source/libs/transport/test/syncClient.c | 2 +- source/libs/transport/test/transUT.cc | 4 +- source/util/src/thash.c | 52 ++++++++++--------- 35 files changed, 116 insertions(+), 141 deletions(-) rename include/common/{tep.h => tdatablock.h} (100%) rename source/common/src/{tep.c => tdatablock.c} (99%) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a04e2afc94..18c1bd5e9a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,24 +25,6 @@ extern "C" { #endif -// typedef struct STimeWindow { -// TSKEY skey; -// TSKEY ekey; -// } STimeWindow; - -// typedef struct { -// int32_t dataLen; -// char name[TSDB_TABLE_FNAME_LEN]; -// char *data; -// } STagData; - -// typedef struct SSchema { -// uint8_t type; -// char name[TSDB_COL_NAME_LEN]; -// int16_t colId; -// int16_t bytes; -// } SSchema; - enum { TMQ_CONF__RESET_OFFSET__LATEST = -1, TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, diff --git a/include/common/tep.h b/include/common/tdatablock.h similarity index 100% rename from include/common/tep.h rename to include/common/tdatablock.h diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index fedb487931..a93180800e 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -19,10 +19,12 @@ extern "C" { #endif +#include "tcommon.h" +#include "nodes.h" + typedef struct SFilterInfo SFilterInfo; typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); - enum { FLT_OPTION_NO_REWRITE = 1, FLT_OPTION_TIMESTAMP = 2, @@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{ SArray* pDataBlock; } SFilterColumnParam; - extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 53fad4607a..23e3ef7525 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -21,7 +21,7 @@ extern "C" { #endif #include -#include +#include #include "taosdef.h" #include "trpc.h" diff --git a/include/util/thash.h b/include/util/thash.h index 017cc8696f..57b20c65ee 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -28,7 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_NODE_EXIST(code) (code == -2) +#define HASH_KEY_ALREADY_EXISTS (-2) +#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS) /** * murmur hash algorithm @@ -49,24 +50,14 @@ uint32_t taosIntHash_32(const char *key, uint32_t len); uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); - _equal_fn_t taosGetDefaultEqualFunction(int32_t type); -typedef struct SHashNode { - struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t refCount; // reference count - int8_t removed; // flag to indicate removed - char data[]; -} SHashNode; - typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, } SHashLockTypeE; +typedef struct SHashNode SHashNode; typedef struct SHashObj SHashObj; /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c93ea1aabe..158bccf8cc 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,12 +20,12 @@ extern "C" { #endif -#include "tcommon.h" #include "parser.h" #include "query.h" #include "taos.h" +#include "tcommon.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "thash.h" #include "tlist.h" #include "tmsg.h" diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a0ba668f8a..259d0e5799 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -4,8 +4,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d9ab23b9fa..490c352e8e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -20,8 +20,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" diff --git a/source/common/src/tep.c b/source/common/src/tdatablock.c similarity index 99% rename from source/common/src/tep.c rename to source/common/src/tdatablock.c index e2880441be..5b7557b749 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tdatablock.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "tep.h" +#include "tdatablock.h" #include "tcompare.h" #include "tglobal.h" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5998b7c9ce..91fab939cf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,7 +17,7 @@ #include "tglobal.h" #include "tcompare.h" #include "tconfig.h" -#include "tep.h" +#include "tdatablock.h" #include "tlog.h" SConfig *tsCfg = NULL; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 4821d60875..7a7dd3f24b 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,7 +1,7 @@ -#include "tcommon.h" #include -#include +#include #include +#include "tcommon.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 9fabe40186..db90baff78 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -23,9 +23,11 @@ extern "C" { #include "os.h" #include "cJSON.h" +#include "monitor.h" #include "tcache.h" #include "tcrc32c.h" -#include "tep.h" +#include "tdatablock.h" +#include "tglobal.h" #include "thash.h" #include "tlockfree.h" #include "tlog.h" @@ -35,8 +37,6 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tglobal.h" -#include "monitor.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index b89cb02834..f22bc9d276 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -14,7 +14,7 @@ */ #include "sut.h" -#include "tep.h" +#include "tdatablock.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 929d47184b..28bf85e2c0 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -20,7 +20,7 @@ #include "sdb.h" #include "tcache.h" -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" #include "tqueue.h" #include "ttime.h" diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b417a645be..ef8da0f897 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -27,13 +27,13 @@ #include "addr_any.h" #include "catalog.h" +#include "catalogInt.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" -#include "catalogInt.h" namespace { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 209bd643e5..e037211f1e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -15,12 +15,12 @@ #include "os.h" -#include "tep.h" -#include "tsort.h" -#include "texception.h" #include "parser.h" +#include "tdatablock.h" +#include "texception.h" #include "tglobal.h" #include "tmsg.h" +#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -8730,10 +8730,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); - size_t s2 = sizeof(SHashNode); - // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)((s1 + s2) * 1.5 * numOfTables); + return (int64_t)(s1* 1.5 * numOfTables); } int32_t checkForQueryBuf(size_t numOfTables) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d042dc0eff..34dd248ba7 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -16,11 +16,11 @@ #include "tcommon.h" #include "query.h" -#include "tsort.h" -#include "tep.h" +#include "tdatablock.h" #include "tdef.h" #include "tlosertree.h" #include "tpagedbuf.h" +#include "tsort.h" #include "tutil.h" typedef struct STupleHandle { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index b5d8bb8019..3e7cbc83ea 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -26,15 +26,15 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "trpc.h" -#include "stub.h" #include "executor.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "tmsg.h" #include "tname.h" +#include "trpc.h" +#include "tvariant.h" namespace { diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index fc366e4cc8..32d148be1d 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -29,8 +29,8 @@ #include "executor.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index c36fc9c659..7fb63f5910 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -28,8 +28,8 @@ #include "tbuffer.h" #include "tcompression.h" //#include "queryLog.h" +#include "tdatablock.h" #include "tudf.h" -#include "tep.h" #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) #define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 00d64bd12a..cdb547ee1b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -15,10 +15,10 @@ #include "mockCatalogService.h" -#include "tep.h" #include #include #include +#include "tdatablock.h" #include "tname.h" #include "ttypes.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8ad5a76388..e599bbede1 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -29,18 +29,17 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "trpc.h" +#include "addr_any.h" +#include "dataSinkMgt.h" +#include "executor.h" #include "planner.h" #include "qworker.h" #include "stub.h" -#include "addr_any.h" -#include "executor.h" -#include "dataSinkMgt.h" - +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "trpc.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index b4fe9a67ca..e1ffb2efd1 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -20,13 +20,13 @@ extern "C" { #endif +#include "query.h" +#include "querynodes.h" +#include "scalar.h" +#include "tcommon.h" +#include "tdatablock.h" #include "thash.h" #include "tname.h" -#include "tcommon.h" -#include "scalar.h" -#include "querynodes.h" -#include "query.h" -#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e5be22c0a8..a7aea5a7e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -16,11 +16,11 @@ #include #include "thash.h" //#include "queryLog.h" -#include "tcompare.h" +#include "filter.h" #include "filterInt.h" #include "sclInt.h" -#include "filter.h" -#include "tep.h" +#include "tcompare.h" +#include "tdatablock.h" OptrStr gOptrStr[] = { {0, "invalid"}, diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0245226dfc..b8cdda9ed1 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,11 +1,11 @@ -#include "nodes.h" -#include "tcommon.h" -#include "querynodes.h" #include "function.h" #include "functionMgt.h" -#include "sclvector.h" +#include "nodes.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" +#include "sclvector.h" +#include "tcommon.h" +#include "tdatablock.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 3c431ff33f..b066dd2e77 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -15,15 +15,15 @@ #include "os.h" -#include "ttypes.h" -#include "sclvector.h" -#include "tcompare.h" -#include "querynodes.h" +#include "filter.h" #include "filterInt.h" #include "query.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" -#include "filter.h" +#include "sclvector.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "ttypes.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e..cf04f9b334 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -29,16 +29,16 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" #include "addr_any.h" -#include "scalar.h" -#include "nodes.h" -#include "tlog.h" #include "filter.h" +#include "nodes.h" +#include "scalar.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "tlog.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088..2e8188e8a9 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -29,15 +29,15 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" #include "addr_any.h" -#include "scalar.h" #include "nodes.h" +#include "scalar.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "tlog.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 11ed3335e6..27384332e5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -19,13 +19,13 @@ #include "os.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" #include "catalog.h" #include "scheduler.h" -#include "tep.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "trpc.h" +#include "tvariant.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 3ba145a96b..757718282a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -14,7 +14,7 @@ */ #include "syncIO.h" -#include +#include #include "syncOnMessage.h" #include "tglobal.h" #include "ttimer.h" diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index 4842a0c800..f4babc9980 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index bcdf32bf6a..6fc935cb61 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b7ef296b9d..f43fa7aae6 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6db709da51..d5e0b65d41 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -15,10 +15,10 @@ #include #include #include -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" -#include "trpc.h" #include "tlog.h" +#include "trpc.h" using namespace std; const char *label = "APP"; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index efbd9adddf..8b437b9797 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -36,25 +36,35 @@ tfree(_n); \ } while (0); +struct SHashNode { + SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t refCount; // reference count + int8_t removed; // flag to indicate removed + char data[]; +}; + typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; } SHashEntry; -typedef struct SHashObj { - SHashEntry **hashList; - size_t capacity; // number of slots - size_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - _hash_free_fn_t freeFp; // hash node free callback function - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry - _hash_before_fn_t callbackFp; // function invoked before return the value to caller -} SHashObj; +struct SHashObj { + SHashEntry ** hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // hash node free callback function + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray * pMemBlock; // memory block allocated for SHashEntry + _hash_before_fn_t callbackFp; // function invoked before return the value to caller +}; /* * Function definition @@ -367,7 +377,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize taosHashRUnlock(pHashObj); - return pHashObj->enableUpdate ? 0 : -1; + return pHashObj->enableUpdate ? 0 : -2; } } @@ -464,7 +474,7 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** return data; } -int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) { return -1; } @@ -507,8 +517,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe prevNode->next = pNode->next; } - if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pe->num--; atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pNode); @@ -525,10 +533,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe return code; } -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); -} - void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; From 7b054eac789654ebde32f8eb1187c487c0d7e1a5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 13:28:03 +0800 Subject: [PATCH 74/82] tfs monitor --- include/libs/monitor/monitor.h | 2 +- source/common/src/tglobal.c | 64 +++++++++++++++++++++------------- source/util/src/tconfig.c | 11 ++---- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 00cb7c3dbc..972361fd68 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -116,7 +116,7 @@ typedef struct { typedef struct { char name[TSDB_FILENAME_LEN]; - int32_t level; + int8_t level; SDiskSize size; } SMonDiskDesc; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 93b66e3369..a7e99b594f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -155,21 +155,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } -static void taosSetTfsCfg(SConfig *pCfg) { +static int32_t taosSetTfsCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); - if (pItem == NULL) return; + memset(tsDataDir, 0, PATH_MAX); int32_t size = taosArrayGetSize(pItem->array); if (size <= 0) { tsDiskCfgNum = 1; taosAddDataDir(0, pItem->str, 0, 1); + tstrncpy(tsDataDir, pItem->str, PATH_MAX); + if (taosMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } else { tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; for (int32_t index = 0; index < tsDiskCfgNum; ++index) { SDiskCfg *pCfg = taosArrayGet(pItem->array, index); memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); + if (pCfg->level == 0 && pCfg->primary == 1) { + tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); + } + if (taosMkDir(pCfg->dir) != 0) { + uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } } + + if (tsDataDir[0] == 0) { + uError("datadir not set"); + return -1; + } + + return 0; } struct SConfig *taosGetCfg() { @@ -358,7 +377,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; } -static void taosSetClientCfg(SConfig *pCfg) { +static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); @@ -375,9 +394,13 @@ static void taosSetClientCfg(SConfig *pCfg) { snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); - tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); - taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + if (taosMkDir(tsTempDir) != 0) { + uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); + return -1; + } tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; @@ -392,6 +415,8 @@ static void taosSetClientCfg(SConfig *pCfg) { tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; + + return 0; } static void taosSetSystemCfg(SConfig *pCfg) { @@ -411,11 +436,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { tsVersion = 30000000; } -static void taosSetServerCfg(SConfig *pCfg) { - tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); - taosExpandDir(tsDataDir, tsDataDir, PATH_MAX); - - tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; +static int32_t taosSetServerCfg(SConfig *pCfg) { + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; @@ -444,6 +466,8 @@ static void taosSetServerCfg(SConfig *pCfg) { if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } + + return 0; } int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, @@ -504,8 +528,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU tsCfg = cfgInit(); if (tsc) { - if (taosAddClientLogCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1; + if (taosAddClientLogCfg(tsCfg) != 0) return -1; } else { if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddServerCfg(tsCfg) != 0) return -1; @@ -528,24 +552,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU } if (tsc) { - taosSetClientCfg(tsCfg); + if (taosSetClientCfg(tsCfg)) return -1; } else { - taosSetClientCfg(tsCfg); - taosSetServerCfg(tsCfg); - taosSetTfsCfg(tsCfg); + if (taosSetClientCfg(tsCfg)) return -1; + if (taosSetServerCfg(tsCfg)) return -1; + if (taosSetTfsCfg(tsCfg) != 0) return -1; } taosSetSystemCfg(tsCfg); - if (taosMkDir(tsTempDir) != 0) { - uError("failed to create dir:%s since %s", tsTempDir, terrstr()); - return -1; - } - - if (!tsc && taosMkDir(tsDataDir) != 0) { - uError("failed to create dir:%s since %s", tsDataDir, terrstr()); - return -1; - } - cfgDumpCfg(tsCfg, tsc, false); return 0; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 0f22d15cd4..d4bcc27f60 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -145,14 +145,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } -#if 0 - if (taosMkDir(fullDir) != 0) { - uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); - return -1; - } -#endif - - cfgFreeItem(pItem); + tfree(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -647,7 +640,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0) { + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); } } From d4a280d83f60b2870fc6331256d0e1e2bc3eca2c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 13:36:06 +0800 Subject: [PATCH 75/82] log level --- include/libs/monitor/monitor.h | 13 ++++++++++--- source/libs/monitor/src/monitor.c | 17 ++++++++++++++++- source/libs/monitor/test/monTest.cpp | 10 +++++----- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 972361fd68..a5372014d5 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -126,10 +126,17 @@ typedef struct { SMonDiskDesc tempdir; } SMonDiskInfo; +typedef enum { + MON_LEVEL_ERROR = 0, + MON_LEVEL_INFO = 1, + MON_LEVEL_DEBUG = 2, + MON_LEVEL_TRACE = 3, +} EMonLogLevel; + typedef struct { - int64_t ts; - int8_t level; - char content[MON_LOG_LEN]; + int64_t ts; + EMonLogLevel level; + char content[MON_LOG_LEN]; } SMonLogItem; typedef struct SMonInfo SMonInfo; diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 14375a4ef6..949fe3e5e7 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -270,6 +270,21 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); } +static const char *monLogLevelStr(EMonLogLevel level) { + switch (level) { + case MON_LEVEL_ERROR: + return "error"; + case MON_LEVEL_INFO: + return "info"; + case MON_LEVEL_DEBUG: + return "debug"; + case MON_LEVEL_TRACE: + return "trace"; + default: + return "undefine"; + } +} + static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; @@ -291,7 +306,7 @@ static void monSetLogInfo(SMonInfo *pMonitor) { taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); tjsonAddStringToObject(pLogJson, "ts", buf); - tjsonAddDoubleToObject(pLogJson, "level", pLogItem->level); + tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); tjsonAddStringToObject(pLogJson, "content", pLogItem->content); if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index ad48ed5407..49abd4803a 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -195,19 +195,19 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { void MonitorTest::AddLogInfo1() { SMonLogItem log1 = {0}; log1.ts = taosGetTimestampMs(); - log1.level = 1; + log1.level = MON_LEVEL_INFO; strcpy(log1.content, "1 -------------------------- a"); monAddLogItem(&log1); SMonLogItem log2 = {0}; log2.ts = taosGetTimestampMs(); - log2.level = 1; + log2.level = MON_LEVEL_ERROR; strcpy(log2.content, "1 ------------------------ b"); monAddLogItem(&log2); SMonLogItem log3 = {0}; log3.ts = taosGetTimestampMs(); - log3.level = 1; + log3.level = MON_LEVEL_DEBUG; strcpy(log3.content, "1 ------- c"); monAddLogItem(&log3); } @@ -215,13 +215,13 @@ void MonitorTest::AddLogInfo1() { void MonitorTest::AddLogInfo2() { SMonLogItem log1; log1.ts = taosGetTimestampMs(); - log1.level = 01; + log1.level = MON_LEVEL_ERROR; strcpy(log1.content, "2 ------- a"); monAddLogItem(&log1); SMonLogItem log2; log2.ts = taosGetTimestampMs(); - log2.level = 0; + log2.level = MON_LEVEL_ERROR; strcpy(log2.content, "2 ------- b"); monAddLogItem(&log2); } From 605da57489cb9fdc9dd2c867b45295dd6bc8fbd7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 13:41:47 +0800 Subject: [PATCH 76/82] fix compile error --- source/dnode/mnode/impl/src/mndProfile.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index cc2dcfab1a..dbf299e8da 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -924,4 +924,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { } } -int32_t mndGetNumOfConnections(SMnode *pMnode) { return taosHashGetSize(pMnode->profileMgmt.cache->pHashTable); } \ No newline at end of file +int32_t mndGetNumOfConnections(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + return taosCacheGetNumOfObj(pMgmt->cache); +} \ No newline at end of file From 51e2870cb8a13fc41c11732ad59a6513672f687f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 13:53:50 +0800 Subject: [PATCH 77/82] adjust log variables --- include/util/tlog.h | 35 +++++++++++---------- source/common/src/tglobal.c | 4 +-- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 2 +- source/libs/transport/test/transUT.cc | 2 +- source/util/src/tlog.c | 12 ++++--- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index 6e6795e9a2..9203921985 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,9 +22,24 @@ extern "C" { #endif +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U +#define DEBUG_SCREEN 64U +#define DEBUG_FILE 128U + +extern bool tsLogEmbedded; extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern int64_t tsNumOfErrorLogs; +extern int64_t tsNumOfInfoLogs; +extern int64_t tsNumOfDebugLogs; +extern int64_t tsNumOfTraceLogs; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; @@ -40,16 +55,6 @@ extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U -#define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U - int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); @@ -68,12 +73,10 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -extern int8_t tscEmbeddedInUtil; - -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a7e99b594f..264e54aafe 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -478,10 +478,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (pCfg == NULL) return -1; if (tsc) { - tscEmbeddedInUtil = 0; + tsLogEmbedded = 0; if (taosAddClientLogCfg(pCfg) != 0) return -1; } else { - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1; } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 00a79ebfd9..2e38ea7513 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; tsAsyncLog = 0; taosRemoveDir(path); diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6db709da51..5c364a4c33 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -148,7 +148,7 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; tsAsyncLog = 0; std::string path = "/tmp/transport"; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index d821e0440b..1ddb636058 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -66,13 +66,17 @@ typedef struct { static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1}; +static int64_t tsAsyncLogLostLines = 0; +static int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; -int8_t tscEmbeddedInUtil = 0; -int32_t tsLogKeepDays = 0; +bool tsLogEmbedded = 0; bool tsAsyncLog = true; int32_t tsNumOfLogLines = 10000000; -int64_t tsAsyncLogLostLines = 0; -int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; +int32_t tsLogKeepDays = 0; +int64_t tsNumOfErrorLogs = 0; +int64_t tsNumOfInfoLogs = 0; +int64_t tsNumOfDebugLogs = 0; +int64_t tsNumOfTraceLogs = 0; // log int32_t dDebugFlag = 135; From 1d9119df8729da83a4320d65ecd5a02fafa478bb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 14:34:35 +0800 Subject: [PATCH 78/82] record num of logs for monitoring --- include/libs/monitor/monitor.h | 14 ++----- include/libs/qcom/query.h | 16 ++++--- include/libs/wal/wal.h | 62 +++++++++++++-------------- include/util/tlog.h | 44 +++++++++---------- source/client/inc/clientLog.h | 14 +++---- source/dnode/mgmt/impl/inc/dndInt.h | 14 +++---- source/dnode/mnode/impl/inc/mndDef.h | 40 ------------------ source/dnode/mnode/impl/inc/mndInt.h | 7 ++++ source/dnode/mnode/sdb/inc/sdbInt.h | 12 +++--- source/dnode/vnode/src/inc/tqInt.h | 62 +++++++++++++-------------- source/dnode/vnode/src/inc/tsdbLog.h | 12 +++--- source/dnode/vnode/src/inc/vnd.h | 63 +++++++++++++--------------- source/libs/index/inc/indexInt.h | 60 +++++++++++++------------- source/libs/monitor/src/monitor.c | 18 ++++---- source/libs/monitor/test/monTest.cpp | 10 ++--- source/libs/sync/inc/syncInt.h | 62 +++++++++++++-------------- source/libs/tfs/inc/tfsInt.h | 14 +++---- source/libs/transport/inc/rpcLog.h | 63 +++++++++++++--------------- source/util/src/thttp.c | 2 +- source/util/src/tlog.c | 35 ++++++++++++++-- source/util/src/ttimer.c | 60 +++++++++++++------------- 21 files changed, 328 insertions(+), 356 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index a5372014d5..262861000d 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -18,6 +18,7 @@ #include "tarray.h" #include "tdef.h" +#include "tlog.h" #ifdef __cplusplus extern "C" { @@ -126,17 +127,10 @@ typedef struct { SMonDiskDesc tempdir; } SMonDiskInfo; -typedef enum { - MON_LEVEL_ERROR = 0, - MON_LEVEL_INFO = 1, - MON_LEVEL_DEBUG = 2, - MON_LEVEL_TRACE = 3, -} EMonLogLevel; - typedef struct { - int64_t ts; - EMonLogLevel level; - char content[MON_LOG_LEN]; + int64_t ts; + ELogLevel level; + char content[MON_LOG_LEN]; } SMonLogItem; typedef struct SMonInfo SMonInfo; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6d3f97fc4e..1f56254476 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -168,20 +168,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); - #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) - +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f90dbb97fe..32fcd95948 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -24,43 +24,41 @@ extern "C" { #endif -extern int32_t wDebugFlag; - -#define wFatal(...) \ - { \ - if (wDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \ - } \ +#define wFatal(...) \ + { \ + if (wDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define wError(...) \ - { \ - if (wDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ - } \ +#define wError(...) \ + { \ + if (wDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define wWarn(...) \ - { \ - if (wDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ - } \ +#define wWarn(...) \ + { \ + if (wDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define wInfo(...) \ - { \ - if (wDebugFlag & DEBUG_INFO) { \ - taosPrintLog("WAL ", 255, __VA_ARGS__); \ - } \ +#define wInfo(...) \ + { \ + if (wDebugFlag & DEBUG_INFO) { \ + taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define wDebug(...) \ - { \ - if (wDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ - } \ +#define wDebug(...) \ + { \ + if (wDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \ + } \ } -#define wTrace(...) \ - { \ - if (wDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ - } \ +#define wTrace(...) \ + { \ + if (wDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \ + } \ } #define WAL_HEAD_VER 0 diff --git a/include/util/tlog.h b/include/util/tlog.h index 9203921985..ff1c0044a4 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,15 +22,17 @@ extern "C" { #endif -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U -#define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +typedef enum { + DEBUG_FATAL = 1, + DEBUG_ERROR = 1, + DEBUG_WARN = 2, + DEBUG_INFO = 2, + DEBUG_DEBUG = 4, + DEBUG_TRACE = 8, + DEBUG_DUMP = 16, + DEBUG_SCREEN = 64, + DEBUG_FILE = 128 +} ELogLevel; extern bool tsLogEmbedded; extern bool tsAsyncLog; @@ -61,27 +63,27 @@ void taosResetLog(); void taosSetAllDebugFlag(int32_t flag); void taosDumpData(uint8_t *msg, int32_t len); -void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) +void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) + __attribute__((format(printf, 4, 5))) #endif ; -void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) +void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) + __attribute__((format(printf, 4, 5))) #endif ; -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} -#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} -#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } -#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } +#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } +#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } #ifdef __cplusplus } diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index bfa2c0319b..d47edcd795 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -22,13 +22,13 @@ extern "C" { #include "tlog.h" -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 9fabe40186..c7c30af057 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -47,14 +47,12 @@ extern "C" { #include "vnode.h" #include "tfs.h" -extern int32_t dDebugFlag; - -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index bb2367ef72..91d37c4eed 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -34,46 +34,6 @@ extern "C" { #endif -extern int32_t mDebugFlag; - -// mnode log function -#define mFatal(...) \ - { \ - if (mDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \ - } \ - } -#define mError(...) \ - { \ - if (mDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \ - } \ - } -#define mWarn(...) \ - { \ - if (mDebugFlag & DEBUG_WARN) { \ - taosPrintLog("MND WARN ", 255, __VA_ARGS__); \ - } \ - } -#define mInfo(...) \ - { \ - if (mDebugFlag & DEBUG_INFO) { \ - taosPrintLog("MND ", 255, __VA_ARGS__); \ - } \ - } -#define mDebug(...) \ - { \ - if (mDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ - } \ - } -#define mTrace(...) \ - { \ - if (mDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ - } \ - } - typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8e9f47d560..71448feb73 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -31,6 +31,13 @@ extern "C" { #endif +#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} +#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} + typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index c99dff57e1..d268b93051 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -28,12 +28,12 @@ extern "C" { #endif -#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} -#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} -#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} -#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} -#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} -#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} +#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} +#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} #define SDB_MAX_SIZE (32 * 1024) diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 30a83ca634..f416413859 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -25,43 +25,41 @@ extern "C" { #endif -extern int32_t tqDebugFlag; - -#define tqFatal(...) \ - { \ - if (tqDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ - } \ +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define tqError(...) \ - { \ - if (tqDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ - } \ +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define tqWarn(...) \ - { \ - if (tqDebugFlag & DEBUG_WARN) { \ - taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ - } \ +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define tqInfo(...) \ - { \ - if (tqDebugFlag & DEBUG_INFO) { \ - taosPrintLog("TQ ", 255, __VA_ARGS__); \ - } \ +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define tqDebug(...) \ - { \ - if (tqDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ - } \ +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \ + } \ } -#define tqTrace(...) \ - { \ - if (tqDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ - } \ +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \ + } \ } #define TQ_BUFFER_SIZE 8 diff --git a/source/dnode/vnode/src/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h index bde9b338a2..6ab17ec587 100644 --- a/source/dnode/vnode/src/inc/tsdbLog.h +++ b/source/dnode/vnode/src/inc/tsdbLog.h @@ -20,11 +20,11 @@ extern int32_t tsdbDebugFlag; -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) #endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 75088687a8..6f4f0049e3 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); -// For Log -extern int32_t vDebugFlag; - -#define vFatal(...) \ - do { \ - if (vDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \ - } \ +#define vFatal(...) \ + do { \ + if (vDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } while (0) -#define vError(...) \ - do { \ - if (vDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \ - } \ +#define vError(...) \ + do { \ + if (vDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } while (0) -#define vWarn(...) \ - do { \ - if (vDebugFlag & DEBUG_WARN) { \ - taosPrintLog("VND WARN ", 255, __VA_ARGS__); \ - } \ +#define vWarn(...) \ + do { \ + if (vDebugFlag & DEBUG_WARN) { \ + taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } while (0) -#define vInfo(...) \ - do { \ - if (vDebugFlag & DEBUG_INFO) { \ - taosPrintLog("VND ", 255, __VA_ARGS__); \ - } \ +#define vInfo(...) \ + do { \ + if (vDebugFlag & DEBUG_INFO) { \ + taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } while (0) -#define vDebug(...) \ - do { \ - if (vDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ - } \ +#define vDebug(...) \ + do { \ + if (vDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define vTrace(...) \ - do { \ - if (vDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ - } \ +#define vTrace(...) \ + do { \ + if (vDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \ + } \ } while (0) // vnodeCfg.h diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 0fcaf11087..6cb5142dd8 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -125,41 +125,41 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ - } \ +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ - } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("index WARN ", 255, __VA_ARGS__); \ - } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("index ", 255, __VA_ARGS__); \ - } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 949fe3e5e7..6882c66e7d 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -270,15 +270,15 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); } -static const char *monLogLevelStr(EMonLogLevel level) { +static const char *monLogLevelStr(ELogLevel level) { switch (level) { - case MON_LEVEL_ERROR: + case DEBUG_ERROR: return "error"; - case MON_LEVEL_INFO: + case DEBUG_INFO: return "info"; - case MON_LEVEL_DEBUG: + case DEBUG_DEBUG: return "debug"; - case MON_LEVEL_TRACE: + case DEBUG_TRACE: return "trace"; default: return "undefine"; @@ -318,25 +318,25 @@ static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pLogError = tjsonCreateObject(); if (pLogError == NULL) return; tjsonAddStringToObject(pLogError, "level", "error"); - tjsonAddDoubleToObject(pLogError, "total", 1); + tjsonAddDoubleToObject(pLogError, "total", tsNumOfErrorLogs); if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError); SJson *pLogInfo = tjsonCreateObject(); if (pLogInfo == NULL) return; tjsonAddStringToObject(pLogInfo, "level", "info"); - tjsonAddDoubleToObject(pLogInfo, "total", 1); + tjsonAddDoubleToObject(pLogInfo, "total", tsNumOfInfoLogs); if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo); SJson *pLogDebug = tjsonCreateObject(); if (pLogDebug == NULL) return; tjsonAddStringToObject(pLogDebug, "level", "debug"); - tjsonAddDoubleToObject(pLogDebug, "total", 1); + tjsonAddDoubleToObject(pLogDebug, "total", tsNumOfDebugLogs); if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug); SJson *pLogTrace = tjsonCreateObject(); if (pLogTrace == NULL) return; tjsonAddStringToObject(pLogTrace, "level", "trace"); - tjsonAddDoubleToObject(pLogTrace, "total", 1); + tjsonAddDoubleToObject(pLogTrace, "total", tsNumOfTraceLogs); if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 49abd4803a..91a502e8e2 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -195,19 +195,19 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { void MonitorTest::AddLogInfo1() { SMonLogItem log1 = {0}; log1.ts = taosGetTimestampMs(); - log1.level = MON_LEVEL_INFO; + log1.level = DEBUG_INFO; strcpy(log1.content, "1 -------------------------- a"); monAddLogItem(&log1); SMonLogItem log2 = {0}; log2.ts = taosGetTimestampMs(); - log2.level = MON_LEVEL_ERROR; + log2.level = DEBUG_ERROR; strcpy(log2.content, "1 ------------------------ b"); monAddLogItem(&log2); SMonLogItem log3 = {0}; log3.ts = taosGetTimestampMs(); - log3.level = MON_LEVEL_DEBUG; + log3.level = DEBUG_DEBUG; strcpy(log3.content, "1 ------- c"); monAddLogItem(&log3); } @@ -215,13 +215,13 @@ void MonitorTest::AddLogInfo1() { void MonitorTest::AddLogInfo2() { SMonLogItem log1; log1.ts = taosGetTimestampMs(); - log1.level = MON_LEVEL_ERROR; + log1.level = DEBUG_ERROR; strcpy(log1.content, "2 ------- a"); monAddLogItem(&log1); SMonLogItem log2; log2.ts = taosGetTimestampMs(); - log2.level = MON_LEVEL_ERROR; + log2.level = DEBUG_ERROR; strcpy(log2.content, "2 ------- b"); monAddLogItem(&log2); } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index aedb9662b1..e276bbd234 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -29,43 +29,41 @@ extern "C" { #include "tlog.h" #include "ttimer.h" -extern int32_t sDebugFlag; - -#define sFatal(...) \ - { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sFatal(...) \ + { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define sError(...) \ - { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sError(...) \ + { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SYN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define sWarn(...) \ - { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sWarn(...) \ + { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("SYN WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define sInfo(...) \ - { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("SYN INFO ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sInfo(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SYN INFO ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define sDebug(...) \ - { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("SYN DEBUG ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sDebug(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SYN DEBUG ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ + } \ } -#define sTrace(...) \ - { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("SYN TRACE ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sTrace(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SYN TRACE ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ + } \ } struct SRaft; diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index cfc246f07b..913f34d6c2 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -25,15 +25,13 @@ #include "thash.h" #include "tlog.h" -extern int32_t fsDebugFlag; - // For debug purpose -#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }} -#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} -#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} -#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} -#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} -#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} +#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", DEBUG_DEBUG, fsDebugFlag, __VA_ARGS__); }} +#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", DEBUG_TRACE, fsDebugFlag, __VA_ARGS__); }} typedef struct { int32_t level; diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 621504091c..a5db866932 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -22,44 +22,41 @@ extern "C" { #include "tlog.h" -extern int32_t rpcDebugFlag; - -// rpcDebugFlag = 143 -#define tFatal(...) \ - { \ - if (rpcDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tFatal(...) \ + { \ + if (rpcDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tError(...) \ - { \ - if (rpcDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tError(...) \ + { \ + if (rpcDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tWarn(...) \ - { \ - if (rpcDebugFlag & DEBUG_WARN) { \ - taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tWarn(...) \ + { \ + if (rpcDebugFlag & DEBUG_WARN) { \ + taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tInfo(...) \ - { \ - if (rpcDebugFlag & DEBUG_INFO) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tInfo(...) \ + { \ + if (rpcDebugFlag & DEBUG_INFO) { \ + taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tDebug(...) \ - { \ - if (rpcDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tDebug(...) \ + { \ + if (rpcDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tTrace(...) \ - { \ - if (rpcDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tTrace(...) \ + { \ + if (rpcDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); \ + } \ } #define tDump(x, y) \ { \ diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c index adf29b1aa9..0737f67ed1 100644 --- a/source/util/src/thttp.c +++ b/source/util/src/thttp.c @@ -63,7 +63,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, goto SEND_OVER; } - uInfo("send http to %s:%u, len:%d content: %s", server, port, contLen, pCont); + uTrace("send http to %s:%u, len:%d content: %s", server, port, contLen, pCont); code = 0; SEND_OVER: diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1ddb636058..bbc5b6e513 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -376,7 +376,27 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { return 0; } -void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { +static void taosUpdateLogNums(ELogLevel level) { + switch (level) { + case DEBUG_ERROR: + atomic_add_fetch_64(&tsNumOfErrorLogs, 1); + break; + case DEBUG_INFO: + atomic_add_fetch_64(&tsNumOfInfoLogs, 1); + break; + case DEBUG_DEBUG: + atomic_add_fetch_64(&tsNumOfDebugLogs, 1); + break; + case DEBUG_DUMP: + case DEBUG_TRACE: + atomic_add_fetch_64(&tsNumOfTraceLogs, 1); + break; + default: + break; + } +} + +void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { if (!osLogSpaceAvailable()) return; va_list argpointer; @@ -414,6 +434,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { buffer[len] = 0; if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { + taosUpdateLogNums(level); if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { @@ -427,11 +448,14 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) { + write(1, buffer, (uint32_t)len); + } } void taosDumpData(unsigned char *msg, int32_t len) { if (!osLogSpaceAvailable()) return; + taosUpdateLogNums(DEBUG_DUMP); char temp[256]; int32_t i, pos = 0, c = 0; @@ -453,7 +477,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } -void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { +void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { if (!osLogSpaceAvailable()) return; va_list argpointer; @@ -481,6 +505,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . buffer[len] = 0; if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { + taosUpdateLogNums(level); if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { @@ -494,7 +519,9 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) { + write(1, buffer, (uint32_t)len); + } } static void taosCloseLogByFd(TdFilePtr pFile) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index abb42ef28d..72c18518e3 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -19,41 +19,41 @@ #include "tlog.h" #include "tsched.h" -#define tmrFatal(...) \ - { \ - if (tmrDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrFatal(...) \ + { \ + if (tmrDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TMR FATAL ", DEBUG_FATAL, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrError(...) \ - { \ - if (tmrDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrError(...) \ + { \ + if (tmrDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TMR ERROR ", DEBUG_ERROR, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrWarn(...) \ - { \ - if (tmrDebugFlag & DEBUG_WARN) { \ - taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrWarn(...) \ + { \ + if (tmrDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TMR WARN ", DEBUG_WARN, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrInfo(...) \ - { \ - if (tmrDebugFlag & DEBUG_INFO) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrInfo(...) \ + { \ + if (tmrDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TMR ", DEBUG_INFO, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrDebug(...) \ - { \ - if (tmrDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrDebug(...) \ + { \ + if (tmrDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TMR ", DEBUG_DEBUG, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrTrace(...) \ - { \ - if (tmrDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrTrace(...) \ + { \ + if (tmrDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TMR ", DEBUG_TRACE, tmrDebugFlag, __VA_ARGS__); \ + } \ } #define TIMER_STATE_WAITING 0 From 5367ca7f2af839f14bd915f8d21c2c5db1abd2db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 15:15:29 +0800 Subject: [PATCH 79/82] record log --- include/libs/monitor/monitor.h | 8 +------ include/util/tlog.h | 3 +++ source/libs/monitor/inc/monInt.h | 18 ++++++++++----- source/libs/monitor/src/monitor.c | 34 ++++++++++++++++------------ source/libs/monitor/test/monTest.cpp | 33 ++++----------------------- source/util/src/tlog.c | 1 + 6 files changed, 42 insertions(+), 55 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 262861000d..0041f3ac5f 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -127,12 +127,6 @@ typedef struct { SMonDiskDesc tempdir; } SMonDiskInfo; -typedef struct { - int64_t ts; - ELogLevel level; - char content[MON_LOG_LEN]; -} SMonLogItem; - typedef struct SMonInfo SMonInfo; typedef struct { @@ -143,7 +137,7 @@ typedef struct { int32_t monInit(const SMonCfg *pCfg); void monCleanup(); -void monAddLogItem(SMonLogItem *pItem); +void monRecordLog(int64_t ts, ELogLevel level, const char *content); SMonInfo *monCreateMonitorInfo(); void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); diff --git a/include/util/tlog.h b/include/util/tlog.h index ff1c0044a4..d3ab9b0bfb 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -34,10 +34,13 @@ typedef enum { DEBUG_FILE = 128 } ELogLevel; +typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); + extern bool tsLogEmbedded; extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern LogFp tsLogFp; extern int64_t tsNumOfErrorLogs; extern int64_t tsNumOfInfoLogs; extern int64_t tsNumOfDebugLogs; diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 61f9980e4e..ca5aaf0a38 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -23,18 +23,24 @@ #include "tjson.h" typedef struct { - SRWLatch lock; - SArray *logs; // array of SMonLogItem - int32_t maxLogs; - const char *server; - uint16_t port; -} SMonitor; + int64_t ts; + ELogLevel level; + char content[MON_LOG_LEN]; +} SMonLogItem; typedef struct SMonInfo { SArray *logs; // array of SMonLogItem SJson *pJson; } SMonInfo; +typedef struct { + pthread_rwlock_t rwlock; + SArray *logs; // array of SMonLogItem + int32_t maxLogs; + const char *server; + uint16_t port; +} SMonitor; + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 6882c66e7d..a50b8c3e90 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -22,6 +22,21 @@ static SMonitor tsMonitor = {0}; +void monRecordLog(int64_t ts, ELogLevel level, const char *content) { + pthread_rwlock_rdlock(&tsMonitor.rwlock); + int32_t size = taosArrayGetSize(tsMonitor.logs); + if (size >= tsMonitor.maxLogs) { + uInfo("too many logs for monitor"); + } else { + SMonLogItem item = {.ts = ts, .level = level}; + SMonLogItem *pItem = taosArrayPush(tsMonitor.logs, &item); + if (pItem != NULL) { + tstrncpy(pItem->content, content, sizeof(item.content)); + } + } + pthread_rwlock_unlock(&tsMonitor.rwlock); +} + int32_t monInit(const SMonCfg *pCfg) { tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem)); if (tsMonitor.logs == NULL) { @@ -32,24 +47,15 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.maxLogs = pCfg->maxLogs; tsMonitor.server = pCfg->server; tsMonitor.port = pCfg->port; - taosInitRWLatch(&tsMonitor.lock); + tsLogFp = monRecordLog; + pthread_rwlock_init(&tsMonitor.rwlock, NULL); return 0; } void monCleanup() { taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; -} - -void monAddLogItem(SMonLogItem *pItem) { - taosWLockLatch(&tsMonitor.lock); - int32_t size = taosArrayGetSize(tsMonitor.logs); - if (size >= tsMonitor.maxLogs) { - uInfo("too many logs for monitor"); - } else { - taosArrayPush(tsMonitor.logs, pItem); - } - taosWUnLockLatch(&tsMonitor.lock); + pthread_rwlock_wrlock(&tsMonitor.rwlock); } SMonInfo *monCreateMonitorInfo() { @@ -59,10 +65,10 @@ SMonInfo *monCreateMonitorInfo() { return NULL; } - taosWLockLatch(&tsMonitor.lock); + pthread_rwlock_wrlock(&tsMonitor.rwlock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - taosWUnLockLatch(&tsMonitor.lock); + pthread_rwlock_unlock(&tsMonitor.rwlock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 91a502e8e2..3eaab45e3e 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -193,37 +193,14 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { } void MonitorTest::AddLogInfo1() { - SMonLogItem log1 = {0}; - log1.ts = taosGetTimestampMs(); - log1.level = DEBUG_INFO; - strcpy(log1.content, "1 -------------------------- a"); - monAddLogItem(&log1); - - SMonLogItem log2 = {0}; - log2.ts = taosGetTimestampMs(); - log2.level = DEBUG_ERROR; - strcpy(log2.content, "1 ------------------------ b"); - monAddLogItem(&log2); - - SMonLogItem log3 = {0}; - log3.ts = taosGetTimestampMs(); - log3.level = DEBUG_DEBUG; - strcpy(log3.content, "1 ------- c"); - monAddLogItem(&log3); + monRecordLog(taosGetTimestampMs(), DEBUG_INFO, "1 -------------------------- a"); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "1 ------------------------ b"); + monRecordLog(taosGetTimestampMs(), DEBUG_DEBUG, "1 ------- c"); } void MonitorTest::AddLogInfo2() { - SMonLogItem log1; - log1.ts = taosGetTimestampMs(); - log1.level = DEBUG_ERROR; - strcpy(log1.content, "2 ------- a"); - monAddLogItem(&log1); - - SMonLogItem log2; - log2.ts = taosGetTimestampMs(); - log2.level = DEBUG_ERROR; - strcpy(log2.content, "2 ------- b"); - monAddLogItem(&log2); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- a"); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- b"); } TEST_F(MonitorTest, 01_Full) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bbc5b6e513..f3457da5b0 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -73,6 +73,7 @@ bool tsLogEmbedded = 0; bool tsAsyncLog = true; int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; +LogFp tsLogFp = NULL; int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; int64_t tsNumOfDebugLogs = 0; From 42e243813eb4dfb76419273c5191e5013e70bbb0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 15:30:50 +0800 Subject: [PATCH 80/82] minor changes --- source/libs/monitor/inc/monInt.h | 10 +++++----- source/libs/monitor/src/monitor.c | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index ca5aaf0a38..65e47ef90c 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -34,11 +34,11 @@ typedef struct SMonInfo { } SMonInfo; typedef struct { - pthread_rwlock_t rwlock; - SArray *logs; // array of SMonLogItem - int32_t maxLogs; - const char *server; - uint16_t port; + pthread_mutex_t lock; + SArray *logs; // array of SMonLogItem + int32_t maxLogs; + const char *server; + uint16_t port; } SMonitor; #ifdef __cplusplus diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index a50b8c3e90..f278a41534 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -23,7 +23,7 @@ static SMonitor tsMonitor = {0}; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { - pthread_rwlock_rdlock(&tsMonitor.rwlock); + pthread_mutex_lock(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); if (size >= tsMonitor.maxLogs) { uInfo("too many logs for monitor"); @@ -34,7 +34,7 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) { tstrncpy(pItem->content, content, sizeof(item.content)); } } - pthread_rwlock_unlock(&tsMonitor.rwlock); + pthread_mutex_unlock(&tsMonitor.lock); } int32_t monInit(const SMonCfg *pCfg) { @@ -48,14 +48,14 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.server = pCfg->server; tsMonitor.port = pCfg->port; tsLogFp = monRecordLog; - pthread_rwlock_init(&tsMonitor.rwlock, NULL); + pthread_mutex_init(&tsMonitor.lock, NULL); return 0; } void monCleanup() { taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; - pthread_rwlock_wrlock(&tsMonitor.rwlock); + pthread_mutex_destroy(&tsMonitor.lock); } SMonInfo *monCreateMonitorInfo() { @@ -65,10 +65,10 @@ SMonInfo *monCreateMonitorInfo() { return NULL; } - pthread_rwlock_wrlock(&tsMonitor.rwlock); + pthread_mutex_lock(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - pthread_rwlock_unlock(&tsMonitor.rwlock); + pthread_mutex_unlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { From d414167214477e7d694585a1879c126653daefb0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Mar 2022 16:08:57 +0800 Subject: [PATCH 81/82] fix crash while acquire connection --- source/util/src/tcache.c | 9 +++++++-- source/util/src/tlog.c | 24 ++++++------------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 3920558276..51ea2b1f13 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -940,8 +940,13 @@ bool taosCacheIterNext(SCacheIter* pIter) { pIter->pCurrent[i] = NULL; } - while(1) { - SCacheEntry *pEntry = &pCacheObj->pEntryList[++pIter->entryIndex]; + while (1) { + pIter->entryIndex++; + if (pIter->entryIndex >= pCacheObj->capacity) { + return false; + } + + SCacheEntry *pEntry = &pCacheObj->pEntryList[pIter->entryIndex]; taosRLockLatch(&pEntry->latch); if (pEntry->num == 0) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f3457da5b0..1e43c85b87 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -17,12 +17,10 @@ #include "tlog.h" #include "tutil.h" -#define LOG_MAX_LINE_SIZE (1000) -#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 10) -#define LOG_MAX_LINE_CONTENT_SIZE (LOG_MAX_LINE_SIZE - 100) +#define LOG_MAX_LINE_SIZE (1024) +#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_DUMP_SIZE (65 * 1024) -#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 10) -#define LOG_MAX_LINE_DUMP_CONTENT_SIZE (LOG_MAX_LINE_DUMP_SIZE - 100) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB @@ -401,7 +399,7 @@ void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char if (!osLogSpaceAvailable()) return; va_list argpointer; - char buffer[LOG_MAX_LINE_BUFFER_SIZE] = {0}; + char buffer[LOG_MAX_LINE_BUFFER_SIZE]; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -416,17 +414,7 @@ void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - int32_t writeLen = vsnprintf(buffer + len, LOG_MAX_LINE_CONTENT_SIZE, format, argpointer); - if (writeLen <= 0) { - char tmp[LOG_MAX_LINE_DUMP_BUFFER_SIZE] = {0}; - writeLen = vsnprintf(tmp, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); - strncpy(buffer + len, tmp, LOG_MAX_LINE_CONTENT_SIZE); - len += LOG_MAX_LINE_CONTENT_SIZE; - } else if (writeLen >= LOG_MAX_LINE_CONTENT_SIZE) { - len += LOG_MAX_LINE_CONTENT_SIZE; - } else { - len += writeLen; - } + len += vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len, format, argpointer); va_end(argpointer); if (len > LOG_MAX_LINE_SIZE) len = LOG_MAX_LINE_SIZE; @@ -497,7 +485,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE, format, argpointer); va_end(argpointer); if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; From 71e4c4194165a9e2b718156eaa7bf9e71ea6822d Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 4 Mar 2022 16:45:20 +0800 Subject: [PATCH 82/82] [TD-13756]: file system remove func. --- include/os/osFile.h | 8 ++++++-- source/common/src/ttszip.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 8 ++++---- source/libs/index/test/fstTest.cc | 2 +- source/libs/tfs/src/tfs.c | 2 +- source/libs/transport/test/pushServer.c | 2 +- source/libs/transport/test/rserver.c | 2 +- source/libs/wal/src/walMeta.c | 2 +- source/libs/wal/src/walWrite.c | 8 ++++---- source/libs/wal/test/walMetaTest.cpp | 4 ++-- source/os/src/osDir.c | 4 ++-- source/os/src/osFile.c | 4 +++- source/util/src/tlog.c | 4 ++-- source/util/src/tpagedbuf.c | 2 +- 15 files changed, 32 insertions(+), 26 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 7e3a5277c8..6ddf1e33c6 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,6 +22,7 @@ extern "C" { #include "osSocket.h" +// If the error is in a third-party library, place this header file under the third-party library header file. #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID @@ -31,6 +32,8 @@ extern "C" { #define fstat FSTAT_FUNC_TAOS_FORBID #define close CLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID + #define fsync FSYNC_FUNC_TAOS_FORBID + // #define fflush FFLUSH_FUNC_TAOS_FORBID #endif #ifndef PATH_MAX @@ -47,13 +50,13 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_TEXT 0x0020 #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 -#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); #define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_READ_OK 0x2 #define TD_FILE_ACCESS_WRITE_OK 0x4 -bool taosCheckAccessFile(const char *pathname, int mode); +bool taosCheckAccessFile(const char *pathname, int mode); int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); @@ -80,6 +83,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile); int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); +int32_t taosRemoveFile(const char *path); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index f20e911681..464c29d287 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } if (!autoDelete) { - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } if (NULL == allocResForTSBuf(pTSBuf)) { @@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf->autoDelete) { // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7a95820115..37403f1a11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } else { // // remove meta.tmp file -// remove(mf.f.aname); +// taosRemoveFile(mf.f.aname); // taosHashCleanup(pfs->metaCacheComp); // pfs->metaCacheComp = NULL; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 411a166caa..a03739c90f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - (void)remove(tfname); + (void)taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } @@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index cb3206a611..618e20bc4b 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -19,7 +19,7 @@ static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { public: FstWriter() { - remove(fileName.c_str()); + taosRemoveFile(fileName.c_str()); _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024); _b = fstBuilderCreate(_wc, 0); } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 26fa90bdef..2579490791 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -202,7 +202,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -int32_t tfsRemoveFile(const STfsFile *pFile) { return remove(pFile->aname); } +int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { return taosCopyFile(pFile1->aname, pFile2->aname); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index a1c181ac98..c763b7bd8a 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 5432a07649..da5284b5c5 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -181,7 +181,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e64260c541..2a4f3497e4 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -379,7 +379,7 @@ int walSaveMeta(SWal* pWal) { // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } free(serialized); return 0; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4b1f0ba306..4ed3c63c18 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -55,9 +55,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); for (int i = pWal->writeCur; i < fileSetSize; i++) { walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // pop from fileInfoSet taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); @@ -174,9 +174,9 @@ int32_t walEndSnapshot(SWal *pWal) { for (int i = 0; i < deleteCnt; i++) { SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // make new array, remove files diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index b65a200ca1..230555e016 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -339,9 +339,9 @@ TEST_F(WalRetentionEnv, repairMeta1) { //getchar(); char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); - remove(buf); + taosRemoveFile(buf); sprintf(buf, "%s/meta-ver%d", pathName, 1); - remove(buf); + taosRemoveFile(buf); SetUp(); //getchar(); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index e0f5f80dab..7d7382d83f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -49,7 +49,7 @@ void taosRemoveDir(const char *dirname) { if (de->d_type & DT_DIR) { taosRemoveDir(filename); } else { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed\n", filename); } } @@ -102,7 +102,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (fileSec <= 100) continue; int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); } else { //printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 37004e9d70..652e0b5182 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -142,11 +142,13 @@ int64_t taosCopyFile(const char *from, const char *to) { _err: if (pFileFrom != NULL) taosCloseFile(&pFileFrom); if (pFileTo != NULL) taosCloseFile(&pFileTo); - remove(to); + taosRemoveFile(to); return -1; #endif } +int32_t taosRemoveFile(const char *path) { return remove(path); } + int32_t taosRenameFile(const char *oldName, const char *newName) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1e43c85b87..1c297851c3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -180,7 +180,7 @@ static void taosKeepOldLog(char *oldName) { char compressFileName[LOG_FILE_NAME_LEN + 20]; snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); if (taosCompressFile(fileName, compressFileName) == 0) { - (void)remove(fileName); + (void)taosRemoveFile(fileName); } } @@ -251,7 +251,7 @@ void taosResetLog() { tsLogObj.lines = tsLogObj.maxLines + 10; taosOpenNewLogFile(); - (void)remove(lastName); + (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 5bc4b81be7..8c2fd7809d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -563,7 +563,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages)); } - remove(pBuf->path); + taosRemoveFile(pBuf->path); tfree(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL);