From 00c5307d513db41ffbe06437953fcf6c121095fc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 10 Mar 2022 20:48:38 +0800 Subject: [PATCH 01/35] refactor --- include/common/taosdef.h | 9 +++++-- include/common/tcommon.h | 38 +++++++++++++---------------- source/common/src/tmsg.c | 1 - source/dnode/mnode/impl/src/mnode.c | 2 ++ 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 69c2618ac8..1584fcb4bf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -33,7 +33,7 @@ typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_STREAM_TABLE = 4, // table created by stream processing TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_TABLE_MAX = 6 } ETableType; @@ -50,7 +50,12 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; -typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; +typedef enum { + TD_ROW_DISCARD_UPDATE = 0, + TD_ROW_OVERWRITE_UPDATE = 1, + TD_ROW_PARTIAL_UPDATE = 2, +} TDUpdateConfig; + typedef enum { TSDB_STATIS_OK = 0, // statis part exist and load successfully TSDB_STATIS_NONE = 1, // statis part not exist diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 1d3ab4f340..3eb9e88ff5 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -135,6 +135,23 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) return (void*)buf; } +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return; + } + + // int32_t numOfOutput = pBlock->info.numOfCols; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < sz; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + // tfree(pBlock); +} + static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; @@ -177,23 +194,6 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { return buf; } -static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return; - } - - // int32_t numOfOutput = pBlock->info.numOfCols; - int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < sz; ++i) { - SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - // tfree(pBlock); -} - static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { if (pRsp->schemas) { if (pRsp->schemas->nCols) { @@ -203,10 +203,6 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); pRsp->pBlockData = NULL; - // for (int32_t i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { - // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); - // tDeleteSSDataBlock(pDataBlock); - //} } //====================================================================================================================== diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 135ff34207..26861e1ff8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2434,7 +2434,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; - tEndEncode(&encoder); int32_t tlen = encoder.pos; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index d3642f4204..f2baea1cd9 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -30,6 +30,7 @@ #include "mndShow.h" #include "mndSnode.h" #include "mndStb.h" +#include "mndStream.h" #include "mndSubscribe.h" #include "mndSync.h" #include "mndTelem.h" @@ -220,6 +221,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; From be532bd5ebdbceda0dcf26b6f5408cd809b36c39 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 12:29:44 +0800 Subject: [PATCH 02/35] sync refactor --- source/libs/sync/inc/syncEnv.h | 20 +++++-- source/libs/sync/src/syncEnv.c | 71 ++++++++++++++++-------- source/libs/sync/test/syncEnvTest.cpp | 26 ++++----- source/libs/sync/test/syncRpcMsgTest.cpp | 14 ++--- 4 files changed, 82 insertions(+), 49 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 9fbea03265..c3679636e6 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -29,6 +29,7 @@ extern "C" { #include "ttimer.h" #define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 #define ELECT_TIMER_MS_MIN 150 #define ELECT_TIMER_MS_MAX 300 @@ -38,17 +39,28 @@ extern "C" { #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) typedef struct SSyncEnv { - tmr_h pEnvTickTimer; + // tick timer + tmr_h pEnvTickTimer; + int32_t envTickTimerMS; + uint64_t envTickTimerLogicClock; + uint64_t envTickTimerLogicClockUser; + TAOS_TMR_CALLBACK FpEnvTickTimer; // Timer Fp + uint64_t envTickTimerCounter; + + // timer manager tmr_h pTimerManager; - char name[128]; + + // other resources shared by SyncNodes + // ... + } SSyncEnv; extern SSyncEnv* gSyncEnv; int32_t syncEnvStart(); int32_t syncEnvStop(); -tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void* param); -void syncEnvStopTimer(tmr_h* pTimer); +int32_t syncEnvStartTimer(); +int32_t syncEnvStopTimer(); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index cb38b3f6f8..2830d2d4aa 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -19,19 +19,18 @@ SSyncEnv *gSyncEnv = NULL; // local function ----------------- -static void syncEnvTick(void *param, void *tmrId); -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); -static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param); -static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer); +static SSyncEnv *doSyncEnvStart(); +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); +static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv); +static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv); +static void syncEnvTick(void *param, void *tmrId); // -------------------------------- int32_t syncEnvStart() { - int32_t ret; + int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); - gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + gSyncEnv = doSyncEnvStart(gSyncEnv); assert(gSyncEnv != NULL); - ret = doSyncEnvStart(gSyncEnv); return ret; } @@ -40,31 +39,46 @@ int32_t syncEnvStop() { return ret; } -tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param) { - return doSyncEnvStartTimer(gSyncEnv, fp, mseconds, param); +int32_t syncEnvStartTimer() { + int32_t ret = doSyncEnvStartTimer(gSyncEnv); + return ret; } -void syncEnvStopTimer(tmr_h *pTimer) { doSyncEnvStopTimer(gSyncEnv, pTimer); } +int32_t syncEnvStopTimer() { + int32_t ret = doSyncEnvStopTimer(gSyncEnv); + return ret; +} // local function ----------------- static void syncEnvTick(void *param, void *tmrId) { SSyncEnv *pSyncEnv = (SSyncEnv *)param; - sTrace("syncEnvTick ... name:%s ", pSyncEnv->name); + if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { + ++(pSyncEnv->envTickTimerCounter); + sTrace( + "syncEnvTick ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS); - pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); + // do something, tick ... + taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); + } } -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { - snprintf(pSyncEnv->name, sizeof(pSyncEnv->name), "SyncEnv_%p", pSyncEnv); +static SSyncEnv *doSyncEnvStart() { + SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + assert(pSyncEnv != NULL); + memset(pSyncEnv, 0, sizeof(pSyncEnv)); + + pSyncEnv->envTickTimerCounter = 0; + pSyncEnv->envTickTimerMS = ENV_TICK_TIMER_MS; + pSyncEnv->FpEnvTickTimer = syncEnvTick; + atomic_store_64(&pSyncEnv->envTickTimerLogicClock, 0); + atomic_store_64(&pSyncEnv->envTickTimerLogicClockUser, 0); // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - - // pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); - - sTrace("SyncEnv start ok, name:%s", pSyncEnv->name); - - return 0; + return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { @@ -72,8 +86,17 @@ static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } -static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param) { - return taosTmrStart(fp, mseconds, pSyncEnv, pSyncEnv->pTimerManager); +static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) { + int32_t ret = 0; + pSyncEnv->pEnvTickTimer = + taosTmrStart(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager); + atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser); + return ret; } -static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer) {} +static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) { + atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1); + taosTmrStop(pSyncEnv->pEnvTickTimer); + pSyncEnv->pEnvTickTimer = NULL; + return 0; +} diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 101c0efe9a..a7a819e046 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -14,15 +14,6 @@ void logTest() { sFatal("--- sync log test: fatal"); } -void *pTimer = NULL; -void *pTimerMgr = NULL; -int g = 300; - -static void timerFp(void *param, void *tmrId) { - printf("param:%p, tmrId:%p, pTimer:%p, pTimerMgr:%p \n", param, tmrId, pTimer, pTimerMgr); - taosTmrReset(timerFp, 1000, param, pTimerMgr, &pTimer); -} - int main() { // taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; @@ -34,13 +25,20 @@ int main() { ret = syncEnvStart(); assert(ret == 0); - // timer - pTimerMgr = taosTmrInit(1000, 50, 10000, "SYNC-ENV-TEST"); - taosTmrStart(timerFp, 1000, &g, pTimerMgr); + for (int i = 0; i < 5; ++i) { + ret = syncEnvStartTimer(); + assert(ret == 0); - while (1) { - taosMsleep(1000); + taosMsleep(5000); + + ret = syncEnvStopTimer(); + assert(ret == 0); + + taosMsleep(5000); } + ret = syncEnvStop(); + assert(ret == 0); + return 0; } diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 0331a29f22..61edbd3012 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -100,7 +100,7 @@ SyncAppendEntriesReply *createSyncAppendEntriesReply() { void test1() { SyncTimeout *pMsg = createSyncTimeout(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test1", &rpcMsg); syncTimeoutDestroy(pMsg); @@ -108,7 +108,7 @@ void test1() { void test2() { SyncPing *pMsg = createSyncPing(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test2", &rpcMsg); syncPingDestroy(pMsg); @@ -116,7 +116,7 @@ void test2() { void test3() { SyncPingReply *pMsg = createSyncPingReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test3", &rpcMsg); syncPingReplyDestroy(pMsg); @@ -132,7 +132,7 @@ void test4() { void test5() { SyncRequestVoteReply *pMsg = createSyncRequestVoteReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test5", &rpcMsg); syncRequestVoteReplyDestroy(pMsg); @@ -140,7 +140,7 @@ void test5() { void test6() { SyncAppendEntries *pMsg = createSyncAppendEntries(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test6", &rpcMsg); syncAppendEntriesDestroy(pMsg); @@ -148,7 +148,7 @@ void test6() { void test7() { SyncAppendEntriesReply *pMsg = createSyncAppendEntriesReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test7", &rpcMsg); syncAppendEntriesReplyDestroy(pMsg); @@ -156,7 +156,7 @@ void test7() { void test8() { SyncClientRequest *pMsg = createSyncClientRequest(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test8", &rpcMsg); syncClientRequestDestroy(pMsg); From 81be154e0d66492cbe8bd1d0b01378993f863836 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:02:15 +0800 Subject: [PATCH 03/35] sync refactor --- source/libs/sync/inc/syncEnv.h | 2 +- source/libs/sync/inc/syncIO.h | 18 +- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncEnv.c | 19 +- source/libs/sync/src/syncIO.c | 332 +++++++++++-------- source/libs/sync/src/syncMessage.c | 24 +- source/libs/sync/test/syncIOTickPingTest.cpp | 11 +- source/libs/sync/test/syncIOTickQTest.cpp | 15 +- source/libs/sync/test/syncUtilTest.cpp | 1 + 9 files changed, 266 insertions(+), 157 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index c3679636e6..40ff79287b 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -42,7 +42,7 @@ typedef struct SSyncEnv { // tick timer tmr_h pEnvTickTimer; int32_t envTickTimerMS; - uint64_t envTickTimerLogicClock; + uint64_t envTickTimerLogicClock; // if use queue, should pass logic clock into queue item uint64_t envTickTimerLogicClockUser; TAOS_TMR_CALLBACK FpEnvTickTimer; // Timer Fp uint64_t envTickTimerCounter; diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 160fefd086..352d30c8d7 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -29,6 +29,9 @@ extern "C" { #include "tqueue.h" #include "trpc.h" +#define TICK_Q_TIMER_MS 1000 +#define TICK_Ping_TIMER_MS 1000 + typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; @@ -38,9 +41,11 @@ typedef struct SSyncIO { void * clientRpc; SEpSet myAddr; - void *ioTimerTickQ; - void *ioTimerTickPing; - void *ioTimerManager; + tmr_h qTimer; + int32_t qTimerMS; + tmr_h pingTimer; + int32_t pingTimerMS; + tmr_h timerMgr; void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); @@ -59,11 +64,14 @@ extern SSyncIO *gSyncIO; int32_t syncIOStart(char *host, uint16_t port); int32_t syncIOStop(); -int32_t syncIOTickQ(); -int32_t syncIOTickPing(); int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg); int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg); +int32_t syncIOQTimerStart(); +int32_t syncIOQTimerStop(); +int32_t syncIOPingTimerStart(); +int32_t syncIOPingTimerStop(); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 2876577410..9bb5b6195e 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -39,6 +39,7 @@ typedef enum ESyncMessageType { SYNC_REQUEST_VOTE_REPLY = 111, SYNC_APPEND_ENTRIES = 113, SYNC_APPEND_ENTRIES_REPLY = 115, + SYNC_RESPONSE = 119, } ESyncMessageType; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 2830d2d4aa..dd7161800d 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -55,13 +55,19 @@ static void syncEnvTick(void *param, void *tmrId) { if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { ++(pSyncEnv->envTickTimerCounter); sTrace( - "syncEnvTick ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " - "envTickTimerMS:%d", + "syncEnvTick do ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d, tmrId:%p", pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS); + pSyncEnv->envTickTimerMS, tmrId); // do something, tick ... taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); + } else { + sTrace( + "syncEnvTick pass ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); } } @@ -88,15 +94,16 @@ static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) { int32_t ret = 0; - pSyncEnv->pEnvTickTimer = - taosTmrStart(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager); + taosTmrReset(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, + &pSyncEnv->pEnvTickTimer); atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser); return ret; } static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) { + int32_t ret = 0; atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1); taosTmrStop(pSyncEnv->pEnvTickTimer); pSyncEnv->pEnvTickTimer = NULL; - return 0; + return ret; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index af97c4663c..c8745ca138 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -16,6 +16,7 @@ #include "syncIO.h" #include #include "syncMessage.h" +#include "syncUtil.h" #include "tglobal.h" #include "ttimer.h" #include "tutil.h" @@ -23,33 +24,36 @@ SSyncIO *gSyncIO = NULL; // local function ------------ -static int32_t syncIOStartInternal(SSyncIO *io); -static int32_t syncIOStopInternal(SSyncIO *io); static SSyncIO *syncIOCreate(char *host, uint16_t port); static int32_t syncIODestroy(SSyncIO *io); +static int32_t syncIOStartInternal(SSyncIO *io); +static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void *syncIOConsumerFunc(void *param); +static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static int32_t syncIOTickQInternal(SSyncIO *io); -static void syncIOTickQFunc(void *param, void *tmrId); -static int32_t syncIOTickPingInternal(SSyncIO *io); -static void syncIOTickPingFunc(void *param, void *tmrId); +static int32_t syncIOStartQ(SSyncIO *io); +static int32_t syncIOStopQ(SSyncIO *io); +static int32_t syncIOStartPing(SSyncIO *io); +static int32_t syncIOStopPing(SSyncIO *io); +static void syncIOTickQ(void *param, void *tmrId); +static void syncIOTickPing(void *param, void *tmrId); // ---------------------------- // public function ------------ int32_t syncIOStart(char *host, uint16_t port) { + int32_t ret = 0; gSyncIO = syncIOCreate(host, port); assert(gSyncIO != NULL); taosSeedRand(taosGetTimestampSec()); - int32_t ret = syncIOStartInternal(gSyncIO); + ret = syncIOStartInternal(gSyncIO); assert(ret == 0); - sTrace("syncIOStart ok, gSyncIO:%p gSyncIO->clientRpc:%p", gSyncIO, gSyncIO->clientRpc); - return 0; + sTrace("syncIOStart ok, gSyncIO:%p", gSyncIO); + return ret; } int32_t syncIOStop() { @@ -61,37 +65,25 @@ int32_t syncIOStop() { return ret; } -int32_t syncIOTickQ() { - int32_t ret = syncIOTickQInternal(gSyncIO); - assert(ret == 0); - return ret; -} - -int32_t syncIOTickPing() { - int32_t ret = syncIOTickPingInternal(gSyncIO); - assert(ret == 0); - return ret; -} - int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { - sTrace( - "<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u, pMsg->ahandle:%p, pMsg->handle:%p, " - "pMsg->msgType:%d, pMsg->contLen:%d", - clientRpc, pEpSet->numOfEps, pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->ahandle, pMsg->handle, - pMsg->msgType, pMsg->contLen); - { - cJSON *pJson = syncRpcMsg2Json(pMsg); - char * serialized = cJSON_Print(pJson); - sTrace("process syncMessage send: pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } + assert(pEpSet->inUse == 0); + assert(pEpSet->numOfEps == 1); + + int32_t ret = 0; + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + syncRpcMsgPrint2(logBuf, pMsg); + pMsg->handle = NULL; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); - return 0; + return ret; } int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { + int32_t ret = 0; + char logBuf[128]; + syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg); + SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); @@ -99,11 +91,75 @@ int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { STaosQueue *pMsgQ = queue; taosWriteQitem(pMsgQ, pTemp); - return 0; + return ret; +} + +int32_t syncIOQTimerStart() { + int32_t ret = syncIOStartQ(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOQTimerStop() { + int32_t ret = syncIOStopQ(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOPingTimerStart() { + int32_t ret = syncIOStartPing(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOPingTimerStop() { + int32_t ret = syncIOStopPing(gSyncIO); + assert(ret == 0); + return ret; } // local function ------------ +static SSyncIO *syncIOCreate(char *host, uint16_t port) { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->myAddr.inUse = 0; + io->myAddr.numOfEps = 0; + addEpIntoEpSet(&io->myAddr, host, port); + + io->qTimerMS = TICK_Q_TIMER_MS; + io->pingTimerMS = TICK_Ping_TIMER_MS; + + return io; +} + +static int32_t syncIODestroy(SSyncIO *io) { + int32_t ret = 0; + int8_t start = atomic_load_8(&io->isStart); + assert(start == 0); + + if (io->serverRpc != NULL) { + rpcClose(io->serverRpc); + io->serverRpc = NULL; + } + + if (io->clientRpc != NULL) { + rpcClose(io->clientRpc); + io->clientRpc = NULL; + } + + taosCloseQueue(io->pMsgQ); + taosCloseQset(io->pQset); + + return ret; +} + static int32_t syncIOStartInternal(SSyncIO *io) { + int32_t ret = 0; taosBlockSIGPIPE(); rpcInit(); @@ -163,58 +219,24 @@ static int32_t syncIOStartInternal(SSyncIO *io) { } // start tmr thread - io->ioTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->timerMgr = taosTmrInit(1000, 50, 10000, "SYNC-IO"); - return 0; + atomic_store_8(&io->isStart, 1); + return ret; } static int32_t syncIOStopInternal(SSyncIO *io) { + int32_t ret = 0; atomic_store_8(&io->isStart, 0); pthread_join(io->consumerTid, NULL); - return 0; -} - -static SSyncIO *syncIOCreate(char *host, uint16_t port) { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->myAddr.inUse = 0; - addEpIntoEpSet(&io->myAddr, host, port); - - return io; -} - -static int32_t syncIODestroy(SSyncIO *io) { - int8_t start = atomic_load_8(&io->isStart); - assert(start == 0); - - if (io->serverRpc != NULL) { - free(io->serverRpc); - io->serverRpc = NULL; - } - - if (io->clientRpc != NULL) { - free(io->clientRpc); - io->clientRpc = NULL; - } - - taosCloseQueue(io->pMsgQ); - taosCloseQset(io->pQset); - - return 0; + taosTmrCleanUp(io->timerMgr); + return ret; } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; - + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; - int type; - + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -226,77 +248,67 @@ static void *syncIOConsumerFunc(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); + syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); - char *s = syncRpcMsg2Str(pRpcMsg); - sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen, - s); - free(s); - + // use switch case instead of if else if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { - SyncPing *pSyncMsg; + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncPing(io->pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + /* pSyncMsg = syncPingBuild(pRpcMsg->contLen); syncPingFromRpcMsg(pRpcMsg, pSyncMsg); // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); + */ } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { - SyncPingReply *pSyncMsg; - pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); - syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { - SyncRequestVote *pSyncMsg; - pSyncMsg = syncRequestVoteBuild(pRpcMsg->contLen); - syncRequestVoteFromRpcMsg(pRpcMsg, pSyncMsg); + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); io->FpOnSyncRequestVote(io->pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { - SyncRequestVoteReply *pSyncMsg; - pSyncMsg = syncRequestVoteReplyBuild(); - syncRequestVoteReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncRequestVoteReply(io->pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { - SyncAppendEntries *pSyncMsg; - pSyncMsg = syncAppendEntriesBuild(pRpcMsg->contLen); - syncAppendEntriesFromRpcMsg(pRpcMsg, pSyncMsg); + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); io->FpOnSyncAppendEntries(io->pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { - SyncAppendEntriesReply *pSyncMsg; - pSyncMsg = syncAppendEntriesReplyBuild(); - syncAppendEntriesReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncAppendEntriesReply(io->pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { - SyncTimeout *pSyncMsg; - pSyncMsg = syncTimeoutBuild(); - syncTimeoutFromRpcMsg(pRpcMsg, pSyncMsg); + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } } else { - ; + sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType); } } @@ -306,15 +318,16 @@ static void *syncIOConsumerFunc(void *param) { rpcFreeCont(pRpcMsg->pCont); if (pRpcMsg->handle != NULL) { - int msgSize = 128; + int msgSize = 32; memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = SYNC_RESPONSE; rpcMsg.pCont = rpcMallocCont(msgSize); rpcMsg.contLen = msgSize; snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); rpcMsg.handle = pRpcMsg->handle; rpcMsg.code = 0; - sTrace("syncIOConsumerFunc rpcSendResponse ... msgType:%d contLen:%d", pRpcMsg->msgType, rpcMsg.contLen); + syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); rpcSendResponse(&rpcMsg); } @@ -326,60 +339,102 @@ static void *syncIOConsumerFunc(void *param) { return NULL; } -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - return ret; -} - static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sTrace("<-- syncIOProcessRequest --> type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen, - (char *)pMsg->pCont); - + syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - taosWriteQitem(io->pMsgQ, pTemp); } static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sTrace("syncIOProcessReply: type:%d, contLen:%d msg:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont); + if (pMsg->msgType == SYNC_RESPONSE) { + sTrace("==syncIOProcessReply=="); + } else { + syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg); + } rpcFreeCont(pMsg->pCont); } -static int32_t syncIOTickQInternal(SSyncIO *io) { - io->ioTimerTickQ = taosTmrStart(syncIOTickQFunc, 1000, io, io->ioTimerManager); - return 0; +static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int32_t ret = 0; + return ret; } -static void syncIOTickQFunc(void *param, void *tmrId) { +static int32_t syncIOStartQ(SSyncIO *io) { + int32_t ret = 0; + taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer); + return ret; +} + +static int32_t syncIOStopQ(SSyncIO *io) { + int32_t ret = 0; + taosTmrStop(io->qTimer); + io->qTimer = NULL; + return ret; +} + +static int32_t syncIOStartPing(SSyncIO *io) { + int32_t ret = 0; + taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); + return ret; +} + +static int32_t syncIOStopPing(SSyncIO *io) { + int32_t ret = 0; + taosTmrStop(io->pingTimer); + io->pingTimer = NULL; + return ret; +} + +static void syncIOTickQ(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; - sTrace("<-- syncIOTickQFunc -->"); + + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + srcId.vgId = -1; + destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + destId.vgId = -1; + SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ"); SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickQ"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 55; - + syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); - + syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); - taosTmrReset(syncIOTickQFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickQ); + syncPingReplyDestroy(pMsg); + + taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer); } -static int32_t syncIOTickPingInternal(SSyncIO *io) { - io->ioTimerTickPing = taosTmrStart(syncIOTickPingFunc, 1000, io, io->ioTimerManager); - return 0; +static void syncIOTickPing(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + srcId.vgId = -1; + destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + destId.vgId = -1; + SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing"); + // SyncPing *pMsg = syncPingBuild3(&srcId, &destId); + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg); + rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); + syncPingDestroy(pMsg); + + taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } +#if 0 + + static void syncIOTickPingFunc(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; sTrace("<-- syncIOTickPingFunc -->"); @@ -393,4 +448,5 @@ static void syncIOTickPingFunc(void *param, void *tmrId) { rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 5a55bbc11f..8d2ff6a9a5 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -65,10 +65,32 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == SYNC_RESPONSE) { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + free(s); + s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + free(s); + } else { pRoot = syncRpcUnknownMsg2Json(); + char* s; + s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + free(s); + s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + free(s); } + cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); + cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); + cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); + cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); return pJson; @@ -77,7 +99,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* syncRpcUnknownMsg2Json() { cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "msgType", SYNC_UNKNOWN); - cJSON_AddStringToObject(pRoot, "data", "known message"); + cJSON_AddStringToObject(pRoot, "data", "unknown message"); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 8be93e6fc0..9c2342828e 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -25,8 +25,15 @@ int main() { ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); - ret = syncIOTickPing(); - assert(ret == 0); + for (int i = 0; i < 3; ++i) { + ret = syncIOPingTimerStart(); + assert(ret == 0); + taosMsleep(5000); + + ret = syncIOPingTimerStop(); + assert(ret == 0); + taosMsleep(5000); + } while (1) { taosSsleep(1); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 76f5e33e82..64b65f25c8 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -25,11 +25,18 @@ int main() { ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); - ret = syncIOTickQ(); + for (int i = 0; i < 3; ++i) { + ret = syncIOQTimerStart(); + assert(ret == 0); + taosMsleep(5000); + + ret = syncIOQTimerStop(); + assert(ret == 0); + taosMsleep(5000); + } + + ret = syncIOStop(); assert(ret == 0); - while (1) { - taosSsleep(1); - } return 0; } diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 9a1c113620..663db3a7b3 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -26,6 +26,7 @@ int main() { tsAsyncLog = 0; sDebugFlag = 143 + 64; logTest(); + electRandomMSTest(); return 0; From 2796c0d171ed245e86660adc4bf0ea603569ac0c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:13:49 +0800 Subject: [PATCH 04/35] sync comment --- source/libs/sync/src/syncIO.c | 6 +++--- source/libs/sync/src/syncMessage.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c8745ca138..327e692d21 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 8d2ff6a9a5..1e40df4419 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -89,7 +89,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); - cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); @@ -1305,4 +1305,4 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); free(serialized); -} \ No newline at end of file +} From 7607169b17f4b5f2463fbce4c27e03923e165b54 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:19:54 +0800 Subject: [PATCH 05/35] sync io --- source/libs/sync/src/syncIO.c | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 327e692d21..91dea79640 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -430,23 +430,4 @@ static void syncIOTickPing(void *param, void *tmrId) { syncPingDestroy(pMsg); taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); -} - -#if 0 - - -static void syncIOTickPingFunc(void *param, void *tmrId) { - SSyncIO *io = (SSyncIO *)param; - sTrace("<-- syncIOTickPingFunc -->"); - - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickPing"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; - - rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); - taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); -} -#endif \ No newline at end of file +} \ No newline at end of file From 64d979fd5c652b3e9d3fdf412904ea40e3684a9c Mon Sep 17 00:00:00 2001 From: ubuntu Date: Sat, 12 Mar 2022 19:07:53 +0800 Subject: [PATCH 06/35] update transport --- include/libs/transport/trpc.h | 7 +++ source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/transCli.c | 59 +++++++++++++++--------- source/libs/transport/src/transSrv.c | 2 +- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index fdc9368b76..8dfd736df6 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -71,6 +71,10 @@ typedef struct SRpcInit { // call back to keep conn or not bool (*pfp)(void *parent, tmsg_t msgType); + // to support Send messages multiple times on a link + // + void* (*mfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; @@ -89,6 +93,9 @@ void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(int64_t rid); +// just release client conn to rpc instance, no close sock +void rpcReleaseHandle(void *handle); + void rpcRefHandle(void *handle, int8_t type); void rpcUnrefHandle(void *handle, int8_t type); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 4e4dcf7aa4..3924a5cf1a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -64,6 +64,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); bool (*pfp)(void* parent, tmsg_t msgType); + void* (*mfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 727845b7a9..ce3c1c2dc8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -135,13 +135,12 @@ static void destroyThrdObj(SCliThrdObj* pThrd); } \ } while (0); +#define CONN_PERSIST_BY_APP(conn) do { if (conn->persist == false) { conn->persist = true; transRefCliHandle(conn);}} while(0) +#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) + static void* cliWorkThread(void* arg); -static void* cliNotifyApp() {} static void cliHandleResp(SCliConn* conn) { - SCliMsg* pMsg = conn->data; - STransConnCtx* pCtx = pMsg->ctx; - SCliThrdObj* pThrd = conn->hostThrd; SRpcInfo* pTransInst = pThrd->pTransInst; @@ -157,14 +156,24 @@ static void cliHandleResp(SCliConn* conn) { rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; rpcMsg.msgType = pHead->msgType; - rpcMsg.ahandle = pCtx->ahandle; + rpcMsg.ahandle = NULL; - if (pTransInst->pfp != NULL && (pTransInst->pfp)(pTransInst->parent, rpcMsg.msgType)) { + SCliMsg* pMsg = conn->data; + STransConnCtx *pCtx = pMsg ? pMsg->ctx : NULL; + if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { + rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; + } else { + rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + } + //if (rpcMsg.ahandle == NULL) { + // tDebug("%s cli conn %p handle except", CONN_GET_INST_LABEL(conn), conn); + // return; + //} + + if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; - transRefCliHandle(conn); - - conn->persist = 1; - tDebug("cli conn %p persist by app", conn); + CONN_PERSIST_BY_APP(conn); + tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, @@ -173,7 +182,7 @@ static void cliHandleResp(SCliConn* conn) { conn->secured = pHead->secured; - if (pCtx->pSem == NULL) { + if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL); } else { @@ -184,8 +193,7 @@ static void cliHandleResp(SCliConn* conn) { uv_read_start((uv_stream_t*)conn->stream, cliAllocBufferCb, cliRecvCb); - // user owns conn->persist = 1 - if (conn->persist == 0) { + if (CONN_NO_PERSIST_BY_APP(conn)) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); } destroyCmsg(conn->data); @@ -198,22 +206,29 @@ static void cliHandleResp(SCliConn* conn) { } static void cliHandleExcept(SCliConn* pConn) { if (pConn->data == NULL) { - // handle conn except in conn pool - transUnrefCliHandle(pConn); - return; + if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { + transUnrefCliHandle(pConn); + return; + } } SCliThrdObj* pThrd = pConn->hostThrd; SRpcInfo* pTransInst = pThrd->pTransInst; SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = pMsg->ctx; + STransConnCtx *pCtx = pMsg ? pMsg->ctx : NULL; SRpcMsg rpcMsg = {0}; - rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = pMsg->msg.msgType + 1; + rpcMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; + rpcMsg.ahandle = NULL; - if (pCtx->pSem == NULL) { + if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { + rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; + } else { + rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + } + + if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL); } else { @@ -358,6 +373,7 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { QUEUE_INIT(&conn->conn); conn->hostThrd = pThrd; + conn->persist = false; conn->broken = false; transRefCliHandle(conn); return conn; @@ -476,7 +492,6 @@ static SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = NULL; if (pMsg->msg.handle != NULL) { conn = (SCliConn*)(pMsg->msg.handle); - transUnrefCliHandle(conn); if (conn != NULL) { tTrace("%s cli conn %p reused", CONN_GET_INST_LABEL(conn), conn); } @@ -514,6 +529,7 @@ static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); } + conn->hThrdIdx = pCtx->hThrdIdx; } static void cliAsyncCb(uv_async_t* handle) { @@ -683,6 +699,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { // imp later } + tDebug("send request at thread:%d %p", index, pMsg); STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 1abca9ad97..c4c3d9ed0a 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -260,7 +260,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } tError("server conn %p read error: %s", conn, uv_err_name(nread)); - if (nread < 0 || nread == UV_EOF) { + if (nread < 0) { conn->broken = true; transUnrefSrvHandle(conn); From 421d087a8f11d75ec00323290d5b3039eb434196 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 12 Mar 2022 19:19:21 +0800 Subject: [PATCH 07/35] add tsdbBDBImpl files --- include/common/taosdef.h | 6 + source/dnode/vnode/inc/tsdb.h | 2 + source/dnode/vnode/src/inc/tsdbDBDef.h | 35 ++++ source/dnode/vnode/src/inc/tsdbDef.h | 16 +- source/dnode/vnode/src/inc/tsdbFile.h | 12 +- source/dnode/vnode/src/inc/tsdbSma.h | 50 +++++- source/dnode/vnode/src/tsdb/tsdbBDBImpl.c | 112 +++++++++++++ source/dnode/vnode/src/tsdb/tsdbMain.c | 29 +++- source/dnode/vnode/src/tsdb/tsdbSma.c | 196 +++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbWrite.c | 8 + source/dnode/vnode/test/tsdbSmaTest.cpp | 53 +++++- 11 files changed, 465 insertions(+), 54 deletions(-) create mode 100644 source/dnode/vnode/src/inc/tsdbDBDef.h diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 9e5e5ebdcf..8dc634571f 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -61,6 +61,12 @@ typedef enum { TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired } ETsdbSmaStat; +typedef enum { + TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA + TSDB_SMA_TYPE_TIME_RANGE = 1, // Time-range-wise SMA + TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA +} ETsdbSmaType; + extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index e67e0cae4b..25bac86f71 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -96,6 +96,8 @@ int tsdbCommit(STsdb *pTsdb); */ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); + /** * @brief Insert RSma(Time-range-wise Rollup SMA) data. * diff --git a/source/dnode/vnode/src/inc/tsdbDBDef.h b/source/dnode/vnode/src/inc/tsdbDBDef.h new file mode 100644 index 0000000000..cc40cec7d1 --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdbDBDef.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_DB_DEF_H_ +#define _TD_TSDB_DB_DEF_H_ + +#include "db.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDBFile SDBFile; +typedef DB_ENV* TDBEnv; + +int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); +void tsdbCloseDBF(SDBFile* pDBF); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TSDB_DB_DEF_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 1451ac9685..0956e418bb 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -43,6 +43,8 @@ extern "C" { struct STsdb { int32_t vgId; + bool repoLocked; + pthread_mutex_t mutex; char * path; STsdbCfg config; STsdbMemTable * mem; @@ -52,12 +54,18 @@ struct STsdb { STsdbFS * fs; SMeta * pMeta; STfs * pTfs; - SSmaStat * pSmaStat; + SSmaEnv * pTSmaEnv; + SSmaEnv * pRSmaEnv; + // SSmaStat * pSmaStat; }; -#define REPO_ID(r) ((r)->vgId) -#define REPO_CFG(r) (&(r)->config) -#define REPO_FS(r) (r)->fs +#define REPO_ID(r) ((r)->vgId) +#define REPO_CFG(r) (&(r)->config) +#define REPO_FS(r) (r)->fs +#define IS_REPO_LOCKED(r) (r)->repoLocked + +int tsdbLockRepo(STsdb *pTsdb); +int tsdbUnlockRepo(STsdb *pTsdb); static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) { return pTable->pSchema; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 5cc8cc045e..e65ef72623 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -329,21 +329,23 @@ static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { // =============== SDFileSet typedef struct { int fid; - int8_t state; // -128~127 - uint8_t ver; // 0~255, DFileSet version + int8_t state; // -128~127 + uint8_t ver; // 0~255, DFileSet version uint16_t reserve; SDFile files[TSDB_FILE_MAX]; } SDFileSet; typedef struct { - int fid; - int8_t state; - uint8_t ver; + int fid; + int8_t state; + uint8_t ver; + uint16_t reserve; #if 0 SDFInfo info; #endif STfsFile f; TdFilePtr pFile; + } SSFile; // files split by days with fid #define TSDB_LATEST_FSET_VER 0 diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 7fceb580d6..87e20b8aa9 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -17,27 +17,40 @@ #define _TD_TSDB_SMA_H_ typedef struct SSmaStat SSmaStat; +typedef struct SSmaEnv SSmaEnv; + + +struct SSmaEnv { + pthread_rwlock_t lock; + char * path; + SSmaStat * pStat; +}; + +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_PATH(env) ((env)->path) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) // insert/update interface int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); - // query interface // TODO: This is the basic params, and should wrap the params to a queryHandle. int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult); // management interface -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg); -int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg); +void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); +void * tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); #if 0 int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); #endif + + // internal func - - static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { int32_t len = 0; len += taosEncodeFixedI64(pData, tableUid); @@ -46,4 +59,31 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, return len; } +static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { + int code = pthread_rwlock_rdlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { + int code = pthread_rwlock_wrlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) { + int code = pthread_rwlock_unlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + #endif /* _TD_TSDB_SMA_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index f2f48bbc8a..7ea9f134cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -12,3 +12,115 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#define ALLOW_FORBID_FUNC +#include "db.h" + +#include "tcoding.h" +#include "thash.h" +#include "tsdbDBDef.h" + +#define IMPL_WITH_LOCK 1 + +struct SDBFile { + DB * pDB; + char *path; +}; + +static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path); +static void tsdbCloseBDBEnv(DB_ENV *pEnv); +static int tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup); +static void tsdbCloseBDBDb(DB *pDB); + +#define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) + +int tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) { + // TDBEnv is shared by a group of SDBFile + ASSERT(pEnv != NULL); + + // Open DBF + if (tsdbOpenBDBDb(&(pDBF->pDB), pEnv, pDBF->path, false) < 0) { + tsdbCloseBDBDb(pDBF->pDB); + return -1; + } + + return 0; +} + +static void *tsdbFreeDBF(SDBFile *pDBF) { + if (pDBF) { + free(pDBF); + } + return NULL; +} + +void tsdbCloseDBF(SDBFile *pDBF) { + if (pDBF->pDB) { + tsdbCloseBDBDb(pDBF->pDB); + pDBF->pDB = tsdbFreeDBF(pDBF); + } +} + +static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { + int ret = 0; + DB_ENV *pEnv = NULL; + + if (path == NULL) return 0; + + ret = db_env_create(&pEnv, 0); + if (ret != 0) { + BDB_PERR("Failed to create tsdb env", ret); + return -1; + } + + ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); + if (ret != 0) { + BDB_PERR("Failed to open tsdb env", ret); + return -1; + } + + *ppEnv = pEnv; + + return 0; +} + +static void tsdbCloseBDBEnv(DB_ENV *pEnv) { + if (pEnv) { + pEnv->close(pEnv, 0); + } +} + +static int tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup) { + int ret; + DB *pDB; + + ret = db_create(&(pDB), pEnv, 0); + if (ret != 0) { + BDB_PERR("Failed to create DBP", ret); + return -1; + } + + if (isDup) { + ret = pDB->set_flags(pDB, DB_DUPSORT); + if (ret != 0) { + BDB_PERR("Failed to set DB flags", ret); + return -1; + } + } + + ret = pDB->open(pDB, NULL, pFName, NULL, DB_BTREE, DB_CREATE, 0); + if (ret) { + BDB_PERR("Failed to open DBF", ret); + return -1; + } + + *ppDB = pDB; + + return 0; +} + +static void tsdbCloseBDBDb(DB *pDB) { + if (pDB) { + pDB->close(pDB, 0); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 0c82911226..afa8921c00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -80,6 +80,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, pTsdb->pmaf = pMAF; pTsdb->pMeta = pMeta; pTsdb->pTfs = pTfs; + pTsdb->pTSmaEnv = NULL; + pTsdb->pRSmaEnv = NULL; pTsdb->fs = tsdbNewFS(pTsdbCfg); @@ -88,8 +90,9 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { + tsdbFreeSmaEnv(pTsdb->pRSmaEnv); + tsdbFreeSmaEnv(pTsdb->pTSmaEnv); tsdbFreeFS(pTsdb->fs); - tsdbDestroySmaState(pTsdb->pSmaStat); tfree(pTsdb->path); free(pTsdb); } @@ -105,6 +108,30 @@ static void tsdbCloseImpl(STsdb *pTsdb) { tsdbCloseFS(pTsdb); // TODO } + +int tsdbLockRepo(STsdb *pTsdb) { + int code = pthread_mutex_lock(&pTsdb->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pTsdb->repoLocked = true; + return 0; +} + +int tsdbUnlockRepo(STsdb *pTsdb) { + ASSERT(IS_REPO_LOCKED(pTsdb)); + pTsdb->repoLocked = false; + int code = pthread_mutex_unlock(&pTsdb->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + #if 0 /* * Copyright (c) 2019 TAOS Data, Inc. diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index dc0d262725..ba8cde2121 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -16,6 +16,7 @@ #include "tsdbDef.h" #define SMA_STORAGE_TSDB_DAYS 30 +#define SMA_STORAGE_TSDB_TIMES 30 #define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 @@ -68,19 +69,104 @@ struct SSmaStat { }; // declaration of static functions -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); -static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit); -static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); -static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen); +static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); +static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); +static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); +static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); +static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); - static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { + SSmaEnv *pEnv = NULL; + + pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); + if (pEnv == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int code = pthread_rwlock_init(&(pEnv->lock), NULL); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + free(pEnv); + return NULL; + } + + ASSERT(path && (strlen(path) > 0)); + pEnv->path = strdup(path); + if (pEnv->path == NULL) { + tsdbFreeSmaEnv(pEnv); + return NULL; + } + + if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { + tsdbFreeSmaEnv(pEnv); + return NULL; + } + + return pEnv; +} + +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (pEnv && *pEnv) { + return TSDB_CODE_SUCCESS; + } + + if (tsdbLockRepo(pTsdb) != 0) { + return TSDB_CODE_FAILED; + } + + if (*pEnv == NULL) { + if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { + tsdbUnlockRepo(pTsdb); + return TSDB_CODE_FAILED; + } + } + + if (tsdbUnlockRepo(pTsdb) != 0) { + tsdbFreeSmaEnv(*pEnv); + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaEnv + * @return int32_t + */ +void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { + if (pSmaEnv) { + tsdbDestroySmaState(pSmaEnv->pStat); + tfree(pSmaEnv->pStat); + tfree(pSmaEnv->path); + pthread_rwlock_destroy(&(pSmaEnv->lock)); + } +} + +void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { + tsdbDestroySmaEnv(pSmaEnv); + tfree(pSmaEnv); + return NULL; +} + static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { ASSERT(pSmaStat != NULL); @@ -125,6 +211,12 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { return pItem; } +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaStat + * @return int32_t + */ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { if (pSmaStat) { // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. @@ -135,30 +227,42 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { item = taosHashIterate(pSmaStat->smaStatItems, item); } taosHashCleanup(pSmaStat->smaStatItems); - free(pSmaStat); } } /** * @brief Update expired window according to msg from stream computing module. - * - * @param pTsdb - * @param msg - * @return int32_t + * + * @param pTsdb + * @param smaType ETsdbSmaType + * @param msg + * @return int32_t */ -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { - if (msg == NULL) { +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSmaEnv * pEnv = NULL; + + if (!msg || !pTsdb->pMeta) { + terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - // lazy mode - if (tsdbInitSmaStat(&pTsdb->pSmaStat) != TSDB_CODE_SUCCESS) { + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + pEnv = pTsdb->pTSmaEnv; + } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + pEnv = pTsdb->pRSmaEnv; + } else { + ASSERT(0); + } + + char smaPath[TSDB_FILENAME_LEN] = "/proj/.sma/"; + if (tsdbInitSmaEnv(pTsdb, smaPath, &pEnv) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } // TODO: decode the msg => start int64_t indexUid = SMA_TEST_INDEX_UID; - const char * indexName = SMA_TEST_INDEX_NAME; + // const char * indexName = SMA_TEST_INDEX_NAME; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; int64_t now = taosGetTimestampMs(); @@ -167,9 +271,9 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { } // TODO: decode the msg <= end - SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems; + SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName)); + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (pItem == NULL) { pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state if (pItem == NULL) { @@ -188,7 +292,7 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { pItem->pSma = pSma; // TODO: change indexName to indexUid - if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) { + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { // If error occurs during put smaStatItem, free the resources of pItem taosHashCleanup(pItem->expiredWindows); free(pItem); @@ -207,7 +311,7 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { // windows failed to put into hash table. taosHashCleanup(pItem->expiredWindows); tfree(pItem->pSma); - taosHashRemove(pItemsHash, indexName, sizeof(indexName)); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); return TSDB_CODE_FAILED; } } @@ -215,11 +319,12 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { return TSDB_CODE_SUCCESS; } -static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, int64_t indexUid, TSKEY skey) { +static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY skey) { SSmaStatItem *pItem = NULL; - if (pTsdb->pSmaStat && pTsdb->pSmaStat->smaStatItems) { - pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, &indexUid, sizeof(indexUid)); + // TODO: If HASH_ENTRY_LOCK used, whether rwlock needed to handle cases of removing hashNode? + if (pStat && pStat->smaStatItems) { + pItem = (SSmaStatItem *)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); } if (pItem != NULL) { @@ -241,7 +346,7 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, int64_t indexUid, TSKEY skey * @param intervalUnit * @return int32_t */ -static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) { +static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? switch (intervalUnit) { case TD_TIME_UNIT_HOUR: @@ -422,12 +527,24 @@ static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWra } static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { - // TODO + STsdb *pTsdb = pSmaH->pTsdb; + pSmaH->pDFile = "tSma_interval_file_name"; return TSDB_CODE_SUCCESS; -} +} +static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel) { + STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb); + int32_t daysPerFile = pCfg->daysPerFile; + + if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { + int32_t days = 30 * (pSmaH->interval / tsTickPerDay[pCfg->precision]); + daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; + } + + return daysPerFile; +} /** * @brief Insert/Update Time-range-wise SMA data. @@ -454,23 +571,26 @@ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - // Step 1: Judge the storage level - int32_t storageLevel = tsdbJudgeStorageLevel(pData->interval, pData->intervalUnit); - int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; + if (!pTsdb->pTSmaEnv) { + terrno = TSDB_CODE_INVALID_PTR; + return terrno; + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t daysPerFile = tsdbGetTSmaDays(&tSmaH, storageLevel); + int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // - Set and open the DFile or the B+Tree file - - int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - - // Save all the TSma data to one file // TODO: tsdbStartTSmaCommit(); tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbInsertTSmaDataSection(&tSmaH, pData); // TODO:tsdbEndTSmaCommit(); // reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, pData->indexUid, pData->skey); + tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); return TSDB_CODE_SUCCESS; } @@ -496,7 +616,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level - int32_t storageLevel = tsdbJudgeStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -511,7 +631,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // TODO:tsdbEndTSmaCommit(); // reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, pData->indexUid, pData->skey); + tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); return TSDB_CODE_SUCCESS; } @@ -540,7 +660,7 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapp * @return int32_t */ static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { - int32_t storageLevel = 0; //tsdbJudgeStorageLevel(param->interval, param->intervalUnit); + int32_t storageLevel = 0; // tsdbGetSmaStorageLevel(param->interval, param->intervalUnit); int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile; pReadH->storageLevel = storageLevel; @@ -594,7 +714,7 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { */ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult) { SSmaStatItem *pItem = - (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, &pData->indexUid, sizeof(pData->indexUid)); + (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &pData->indexUid, sizeof(pData->indexUid)); if (pItem == NULL) { // mark all window as expired and notify query module to query raw TS data. return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 26d31af4f3..dede1502f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -51,6 +51,14 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { return code; } +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { + tsdbWarn("vgId:%d update expired window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} + /** * @brief Insert Time-range-wise Rollup Sma(RSma) data * diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index ac9a8fd3d0..159ad98219 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -221,12 +221,60 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { #if 1 TEST(testCase, tSmaInsertTest) { + // prepare meta + const char * smaIndexName1 = "sma_index_test_1"; + const char * smaIndexName2 = "sma_index_test_2"; + const char * timezone = "Asia/Shanghai"; + const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; + const char * tagsFilter = "I'm tags filter"; + const char * smaTestDir = "./smaTest"; + const tb_uid_t tbUid = 1234567890; + const int64_t indexUid1 = 2000000001; + const int64_t indexUid2 = 2000000002; + const uint32_t nCntTSma = 2; + // encode + STSma tSma = {0}; + tSma.version = 0; + tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.interval = 1; + tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.sliding = 0; + tSma.indexUid = indexUid1; + tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); + tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN); + tSma.tableUid = tbUid; + + tSma.exprLen = strlen(expr); + tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); + tstrncpy(tSma.expr, expr, tSma.exprLen + 1); + + tSma.tagsFilterLen = strlen(tagsFilter); + tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); + + SMeta * pMeta = NULL; + STSma * pSmaCfg = &tSma; + const SMetaCfg *pMetaCfg = &defaultMetaOptions; + + taosRemoveDir(smaTestDir); + + pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); + assert(pMeta != NULL); + // save index 1 + EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + + + // insert data const int64_t indexUid = 2000000002; STSmaDataWrapper *pSmaData = NULL; STsdb tsdb = {0}; STsdbCfg * pCfg = &tsdb.config; pCfg->daysPerFile = 1; + tsdb.pMeta = pMeta; + + char *msg = (char *)calloc(100, 1); + EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); // init int32_t allocCnt = 0; @@ -277,8 +325,11 @@ TEST(testCase, tSmaInsertTest) { // execute EXPECT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); - // release + // release data taosTZfree(buf); + // release meta + tdDestroyTSma(&tSma); + metaClose(pMeta); } #endif From 9bac1deeb9b3a9b2526491c798cea661a1a225db Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:19:54 +0800 Subject: [PATCH 08/35] sync io --- source/libs/sync/src/syncIO.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 91dea79640..c307ec5068 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { From 1336e08ad0ec5d157db2ec3256f360ce49c1fa39 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:27:14 +0800 Subject: [PATCH 09/35] sync io --- source/libs/sync/test/syncEnqTest.cpp | 83 ++++++++------- source/libs/sync/test/syncIOSendMsgTest.cpp | 111 +++++++++++++++----- 2 files changed, 131 insertions(+), 63 deletions(-) diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index e1706bb40b..57315f40ec 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,9 +1,10 @@ +#include #include #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" -#include "syncMessage.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -14,64 +15,69 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[3] = {7010, 7110, 7210}; +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; -SSyncNode* doSync(int myIndex) { - SSyncFSM* pFsm; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; - SSyncInfo syncInfo; - syncInfo.vgId = 1; +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; - pCfg->replicaNum = 3; + pCfg->replicaNum = replicaNum; - pCfg->nodeInfo[0].nodePort = ports[0]; - snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = ports[1]; - snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = ports[2]; - snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -void timerPingAll(void* param, void* tmrId) { - SSyncNode* pSyncNode = (SSyncNode*)param; - syncNodePingAll(pSyncNode); +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } } int main(int argc, char** argv) { - // taosInitLog((char*)"syncPingTest.log", 100000, 10); + // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - - int myIndex = 0; + myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); - if (myIndex > 2 || myIndex < 0) { - fprintf(stderr, "myIndex:%d error. should be 0 - 2", myIndex); - return 1; - } } int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); @@ -80,21 +86,22 @@ int main(int argc, char** argv) { ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(myIndex); - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"syncInitTest", pSyncNode); + + initRaftId(pSyncNode); + + //-------------------------------------------------------------- for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncEnqTest"); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); taosMsleep(1000); } - while (1) { - taosMsleep(1000); - } - return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index ed88fbb03e..0fc3ebfe4c 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,8 +1,10 @@ #include #include +#include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,37 +15,96 @@ void logTest() { sFatal("--- sync log test: fatal"); } -int main() { +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - - int32_t ret; - - ret = syncIOStart((char *)"127.0.0.1", 7010); - assert(ret == 0); - - for (int i = 0; i < 10; ++i) { - SEpSet epSet; - epSet.inUse = 0; - epSet.numOfEps = 0; - addEpIntoEpSet(&epSet, "127.0.0.1", 7010); - - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; - - syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); - taosSsleep(1); + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); } - while (1) { - taosSsleep(1); + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"syncInitTest", pSyncNode); + + initRaftId(pSyncNode); + + //-------------------------------------------------------------- + + for (int i = 0; i < 10; ++i) { + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncIOSendMsgTest"); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); + + SEpSet epSet; + syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, &rpcMsg); + + taosMsleep(1000); } return 0; From 51ceed983f49e971450f45057e205f503ea62ae8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:36:32 +0800 Subject: [PATCH 10/35] sync io test --- source/libs/sync/test/CMakeLists.txt | 20 ++++++++-------- ...MsgClientTest.cpp => syncIOClientTest.cpp} | 24 +++++++++---------- ...MsgServerTest.cpp => syncIOServerTest.cpp} | 0 3 files changed, 22 insertions(+), 22 deletions(-) rename source/libs/sync/test/{syncIOSendMsgClientTest.cpp => syncIOClientTest.cpp} (62%) rename source/libs/sync/test/{syncIOSendMsgServerTest.cpp => syncIOServerTest.cpp} (100%) diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 2a47b53945..6ade78936d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -4,8 +4,8 @@ add_executable(syncPingTimerTest "") add_executable(syncIOTickQTest "") add_executable(syncIOTickPingTest "") add_executable(syncIOSendMsgTest "") -add_executable(syncIOSendMsgClientTest "") -add_executable(syncIOSendMsgServerTest "") +add_executable(syncIOClientTest "") +add_executable(syncIOServerTest "") add_executable(syncRaftStoreTest "") add_executable(syncEnqTest "") add_executable(syncIndexTest "") @@ -51,13 +51,13 @@ target_sources(syncIOSendMsgTest PRIVATE "syncIOSendMsgTest.cpp" ) -target_sources(syncIOSendMsgClientTest +target_sources(syncIOClientTest PRIVATE - "syncIOSendMsgClientTest.cpp" + "syncIOClientTest.cpp" ) -target_sources(syncIOSendMsgServerTest +target_sources(syncIOServerTest PRIVATE - "syncIOSendMsgServerTest.cpp" + "syncIOServerTest.cpp" ) target_sources(syncRaftStoreTest PRIVATE @@ -167,12 +167,12 @@ target_include_directories(syncIOSendMsgTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOSendMsgClientTest +target_include_directories(syncIOClientTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOSendMsgServerTest +target_include_directories(syncIOServerTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" @@ -298,11 +298,11 @@ target_link_libraries(syncIOSendMsgTest sync gtest_main ) -target_link_libraries(syncIOSendMsgClientTest +target_link_libraries(syncIOClientTest sync gtest_main ) -target_link_libraries(syncIOSendMsgServerTest +target_link_libraries(syncIOServerTest sync gtest_main ) diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp similarity index 62% rename from source/libs/sync/test/syncIOSendMsgClientTest.cpp rename to source/libs/sync/test/syncIOClientTest.cpp index 250054fd5a..dffa8b5cb9 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -2,7 +2,8 @@ #include #include "syncIO.h" #include "syncInt.h" -#include "syncRaftStore.h" +#include "syncMessage.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -22,7 +23,7 @@ int main() { int32_t ret; - ret = syncIOStart((char *)"127.0.0.1", 7010); + ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); for (int i = 0; i < 10; ++i) { @@ -31,20 +32,19 @@ int main() { epSet.numOfEps = 0; addEpIntoEpSet(&epSet, "127.0.0.1", 7030); - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + srcId.vgId = 100; + destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + destId.vgId = 100; + + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOClientTest"); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); taosSsleep(1); } - while (1) { - taosSsleep(1); - } - return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp similarity index 100% rename from source/libs/sync/test/syncIOSendMsgServerTest.cpp rename to source/libs/sync/test/syncIOServerTest.cpp From 4caa94d5c9c80848c5c07ce530fcc6cf1ce5eb00 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 13 Mar 2022 18:55:12 +0800 Subject: [PATCH 11/35] [TD-13767]: redefine Locale api. --- include/os/osLocale.h | 6 +++++- source/os/src/osLocale.c | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/os/osLocale.h b/include/os/osLocale.h index 6e313eb8cd..ddafd2e93c 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -17,12 +17,16 @@ #define _TD_OS_LOCALE_H_ #include "os.h" -#include "osString.h" #ifdef __cplusplus extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define setlocale SETLOCALE_FUNC_TAOS_FORBID +#endif + char *taosCharsetReplace(char *charsetstr); void taosGetSystemLocale(char *outLocale, char *outCharset); void taosSetSystemLocale(const char *inLocale, const char *inCharSet); diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 47546f7deb..e9d6ed7c54 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "osLocale.h" From abbd818fe6e639cd8e7cc171446b6de6699095f2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 13 Mar 2022 23:32:33 +0800 Subject: [PATCH 12/35] import SmaEnv/SmaStat to facilitate save data --- include/util/taoserror.h | 1 + source/dnode/vnode/src/inc/tsdbDBDef.h | 8 ++ source/dnode/vnode/src/inc/tsdbDef.h | 3 +- source/dnode/vnode/src/inc/tsdbSma.h | 2 +- source/dnode/vnode/src/meta/metaBDBImpl.c | 37 +++--- source/dnode/vnode/src/tsdb/tsdbBDBImpl.c | 50 ++++++-- source/dnode/vnode/src/tsdb/tsdbSma.c | 149 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbWrite.c | 2 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 21 +-- source/util/src/terror.c | 1 + 10 files changed, 186 insertions(+), 88 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3a1343b384..1c61d738b5 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -353,6 +353,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0617) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/source/dnode/vnode/src/inc/tsdbDBDef.h b/source/dnode/vnode/src/inc/tsdbDBDef.h index cc40cec7d1..7740dd0fab 100644 --- a/source/dnode/vnode/src/inc/tsdbDBDef.h +++ b/source/dnode/vnode/src/inc/tsdbDBDef.h @@ -25,8 +25,16 @@ extern "C" { typedef struct SDBFile SDBFile; typedef DB_ENV* TDBEnv; +struct SDBFile { + DB* pDB; + char* path; +}; + int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); void tsdbCloseDBF(SDBFile* pDBF); +int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); +void tsdbCloseBDBEnv(DB_ENV* pEnv); +int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 0956e418bb..6f91b4d3ab 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -27,6 +27,7 @@ #include "ttime.h" #include "tsdb.h" +#include "tsdbDBDef.h" #include "tsdbCommit.h" #include "tsdbFS.h" #include "tsdbFile.h" @@ -37,6 +38,7 @@ #include "tsdbReadImpl.h" #include "tsdbSma.h" + #ifdef __cplusplus extern "C" { #endif @@ -56,7 +58,6 @@ struct STsdb { STfs * pTfs; SSmaEnv * pTSmaEnv; SSmaEnv * pRSmaEnv; - // SSmaStat * pSmaStat; }; #define REPO_ID(r) ((r)->vgId) diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 87e20b8aa9..c54fdf85a3 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -19,9 +19,9 @@ typedef struct SSmaStat SSmaStat; typedef struct SSmaEnv SSmaEnv; - struct SSmaEnv { pthread_rwlock_t lock; + TDBEnv dbEnv; char * path; SSmaStat * pStat; }; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index d9af526c2a..a729288e34 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -231,30 +231,31 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { void *pBuf = NULL, *qBuf = NULL; DBT key1 = {0}, value1 = {0}; - { - // save sma info - int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = calloc(len, 1); - if (pBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - key1.data = (void *)&pSmaCfg->indexUid; - key1.size = sizeof(pSmaCfg->indexUid); - - qBuf = pBuf; - tEncodeTSma(&qBuf, pSmaCfg); - - value1.data = pBuf; - value1.size = POINTER_DISTANCE(qBuf, pBuf); - value1.app_data = pSmaCfg; + // save sma info + int32_t len = tEncodeTSma(NULL, pSmaCfg); + pBuf = calloc(len, 1); + if (pBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + key1.data = (void *)&pSmaCfg->indexUid; + key1.size = sizeof(pSmaCfg->indexUid); + + qBuf = pBuf; + tEncodeTSma(&qBuf, pSmaCfg); + + value1.data = pBuf; + value1.size = POINTER_DISTANCE(qBuf, pBuf); + value1.app_data = pSmaCfg; + metaDBWLock(pMeta->pDB); pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0); metaDBULock(pMeta->pDB); + // release + tfree(pBuf); + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index 7ea9f134cf..4fc415cfd1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -16,30 +16,29 @@ #define ALLOW_FORBID_FUNC #include "db.h" +#include "taoserror.h" #include "tcoding.h" #include "thash.h" #include "tsdbDBDef.h" +#include "tsdbLog.h" #define IMPL_WITH_LOCK 1 -struct SDBFile { - DB * pDB; - char *path; -}; +static int tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup); +static void tsdbCloseBDBDb(DB *pDB); -static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path); -static void tsdbCloseBDBEnv(DB_ENV *pEnv); -static int tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup); -static void tsdbCloseBDBDb(DB *pDB); - -#define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) +#define BDB_PERR(info, code) fprintf(stderr, "%s:%d " info " reason: %s\n", __FILE__, __LINE__, db_strerror(code)) int tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) { // TDBEnv is shared by a group of SDBFile - ASSERT(pEnv != NULL); + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } // Open DBF if (tsdbOpenBDBDb(&(pDBF->pDB), pEnv, pDBF->path, false) < 0) { + terrno = TSDB_CODE_TDB_INIT_FAILED; tsdbCloseBDBDb(pDBF->pDB); return -1; } @@ -61,7 +60,7 @@ void tsdbCloseDBF(SDBFile *pDBF) { } } -static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { +int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { int ret = 0; DB_ENV *pEnv = NULL; @@ -75,7 +74,8 @@ static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); if (ret != 0) { - BDB_PERR("Failed to open tsdb env", ret); + // BDB_PERR("Failed to open tsdb env", ret); + tsdbWarn("Failed to open tsdb env for path %s since %d", path ? path : "NULL", ret); return -1; } @@ -84,7 +84,7 @@ static int tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { return 0; } -static void tsdbCloseBDBEnv(DB_ENV *pEnv) { +void tsdbCloseBDBEnv(DB_ENV *pEnv) { if (pEnv) { pEnv->close(pEnv, 0); } @@ -123,4 +123,26 @@ static void tsdbCloseBDBDb(DB *pDB) { if (pDB) { pDB->close(pDB, 0); } +} + +int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *key, uint32_t keySize, void *data, uint32_t dataSize) { + int ret; + DBT key1 = {0}, value1 = {0}; + + key1.data = key; + key1.size = keySize; + + value1.data = data; + value1.size = dataSize; + + // TODO: lock + ret = pDBF->pDB->put(pDBF->pDB, NULL, &key1, &value1, 0); + if (ret) { + BDB_PERR("Failed to put data to DBF", ret); + // TODO: unlock + return -1; + } + // TODO: unlock + + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index ba8cde2121..f96ea644b4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -24,16 +24,16 @@ #define SMA_STATE_ITEM_HASH_SLOT 32 #define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test -#define SMA_TEST_INDEX_UID 123456 // TODO: just for test +#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test typedef enum { SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2t200.dat SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/rsma/sma_index_uid/v2r200.dat } ESmaStorageLevel; typedef struct { - STsdb * pTsdb; - char * pDFile; // TODO: use the real DFile type, not char* - int32_t interval; // interval with the precision of DB + STsdb * pTsdb; + SDBFile *pDFile; + int32_t interval; // interval with the precision of DB // TODO } STSmaWriteH; @@ -74,10 +74,11 @@ static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); -static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen); +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel); @@ -114,6 +115,11 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } + if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { + tsdbFreeSmaEnv(pEnv); + return NULL; + } + return pEnv; } @@ -158,6 +164,7 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { tfree(pSmaEnv->pStat); tfree(pSmaEnv->path); pthread_rwlock_destroy(&(pSmaEnv->lock)); + tsdbCloseBDBEnv(pSmaEnv->dbEnv); } } @@ -213,9 +220,9 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { /** * @brief Release resources allocated for its member fields, not including itself. - * - * @param pSmaStat - * @return int32_t + * + * @param pSmaStat + * @return int32_t */ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { if (pSmaStat) { @@ -232,11 +239,11 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { /** * @brief Update expired window according to msg from stream computing module. - * - * @param pTsdb + * + * @param pTsdb * @param smaType ETsdbSmaType - * @param msg - * @return int32_t + * @param msg + * @return int32_t */ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { STsdbCfg *pCfg = REPO_CFG(pTsdb); @@ -247,21 +254,21 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { return TSDB_CODE_FAILED; } - if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - pEnv = pTsdb->pTSmaEnv; - } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - pEnv = pTsdb->pRSmaEnv; - } else { - ASSERT(0); - } - char smaPath[TSDB_FILENAME_LEN] = "/proj/.sma/"; if (tsdbInitSmaEnv(pTsdb, smaPath, &pEnv) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + pTsdb->pTSmaEnv = pEnv; + } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + pTsdb->pRSmaEnv = pEnv; + } else { + ASSERT(0); + } + // TODO: decode the msg => start - int64_t indexUid = SMA_TEST_INDEX_UID; + int64_t indexUid = SMA_TEST_INDEX_UID; // const char * indexName = SMA_TEST_INDEX_NAME; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; @@ -285,8 +292,11 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { // cache smaMeta STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); if (pSma == NULL) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); free(pItem); + tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, + tstrerror(terrno)); return TSDB_CODE_FAILED; } pItem->pSma = pSma; @@ -299,6 +309,11 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { return TSDB_CODE_FAILED; } } +#if 0 + SSmaStatItem *pItem1 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + int size1 = taosHashGetSize(pItem1->expiredWindows); + tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d before hashPut", REPO_ID(pTsdb), indexUid, size1); +#endif int8_t state = TSDB_SMA_STAT_EXPIRED; for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { @@ -316,6 +331,12 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { } } +#if 0 + SSmaStatItem *pItem2 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + int size2 = taosHashGetSize(pItem1->expiredWindows); + tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d after hashPut", REPO_ID(pTsdb), indexUid, size2); +#endif + return TSDB_CODE_SUCCESS; } @@ -326,7 +347,7 @@ static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY s if (pStat && pStat->smaStatItems) { pItem = (SSmaStatItem *)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); } - +#if 0 if (pItem != NULL) { // TODO: reset time window for the sma data blocks if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { @@ -336,6 +357,7 @@ static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY s } else { // error handling } +#endif return TSDB_CODE_SUCCESS; } @@ -394,10 +416,17 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { * @param dataLen * @return int32_t */ -static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) { +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { + SDBFile *pDBFile = pSmaH->pDFile; + // TODO: insert sma data blocks into B+Tree - tsdbDebug("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", - *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + tsdbDebug("insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", pDBFile->path, + *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + + if(tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0){ + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } @@ -486,8 +515,6 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p // TODO: check the data integrity - void *bTree = pSmaH->pDFile; - int32_t len = 0; while (true) { if (len >= pData->dataLen) { @@ -510,7 +537,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p pData->indexUid, pData->skey, pTbData->tableUid, pColData->colId); #endif tsdbEncodeTSmaKey(pTbData->tableUid, pColData->colId, pData->skey, (void **)&pSmaKey); - if (tsdbInsertTSmaBlocks(bTree, smaKey, pColData->data, pColData->blockSize) < 0) { + if (tsdbInsertTSmaBlocks(pSmaH, smaKey, SMA_KEY_LEN, pColData->data, pColData->blockSize) < 0) { tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } tbLen += (sizeof(STSmaColData) + pColData->blockSize); @@ -524,13 +551,28 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { pSmaH->pTsdb = pTsdb; pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->pDFile = (SDBFile *)calloc(1, sizeof(SDBFile *)); + if (!pSmaH->pDFile) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { + if (pSmaH) { + if (pSmaH->pDFile) { + tsdbCloseDBF(pSmaH->pDFile); + } + } } static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; - - pSmaH->pDFile = "tSma_interval_file_name"; - + ASSERT(pSmaH->pDFile->path == NULL && pSmaH->pDFile->pDB == NULL); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); + pSmaH->pDFile->path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; } @@ -559,21 +601,25 @@ static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel) { * @return int32_t */ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - STSmaWriteH tSmaH = {0}; - tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData); + if (!pTsdb->pTSmaEnv) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } if (pData->dataLen <= 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return TSDB_CODE_FAILED; } - if (!pTsdb->pTSmaEnv) { - terrno = TSDB_CODE_INVALID_PTR; - return terrno; + STSmaWriteH tSmaH = {0}; + + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + return TSDB_CODE_FAILED; } // Step 1: Judge the storage level and days @@ -585,27 +631,41 @@ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, tSmaH.pDFile) != 0) { + tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), + tSmaH.pDFile->path ? tSmaH.pDFile->path : "path is NULL", tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } - tsdbInsertTSmaDataSection(&tSmaH, pData); + if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { + tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } // TODO:tsdbEndTSmaCommit(); - // reset the SSmaStat + // Step 3: reset the SSmaStat tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); + tsdbDestroyTSmaWriteH(&tSmaH); return TSDB_CODE_SUCCESS; } static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { - // TODO - pSmaH->pDFile = "rSma_interval_file_name"; + STsdb *pTsdb = pSmaH->pTsdb; + + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); + pSmaH->pDFile->path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; } int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - STSmaWriteH tSmaH = {0}; + STSmaWriteH tSmaH = {0}; tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData); @@ -627,6 +687,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // Save all the TSma data to one file // TODO: tsdbStartTSmaCommit(); tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbInsertTSmaDataSection(&tSmaH, pData); // TODO:tsdbEndTSmaCommit(); diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index dede1502f4..9cccea9853 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -54,7 +54,7 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { - tsdbWarn("vgId:%d update expired window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; } diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 159ad98219..f815291c77 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -223,14 +223,12 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { TEST(testCase, tSmaInsertTest) { // prepare meta const char * smaIndexName1 = "sma_index_test_1"; - const char * smaIndexName2 = "sma_index_test_2"; const char * timezone = "Asia/Shanghai"; const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; const char * tagsFilter = "I'm tags filter"; const char * smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; - const int64_t indexUid2 = 2000000002; const uint32_t nCntTSma = 2; // encode STSma tSma = {0}; @@ -263,15 +261,20 @@ TEST(testCase, tSmaInsertTest) { // save index 1 EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); - // insert data - const int64_t indexUid = 2000000002; STSmaDataWrapper *pSmaData = NULL; STsdb tsdb = {0}; STsdbCfg * pCfg = &tsdb.config; - pCfg->daysPerFile = 1; tsdb.pMeta = pMeta; + tsdb.vgId = 2; + tsdb.config.daysPerFile = 10; // default days is 10 + tsdb.config.keep1 = 30; + tsdb.config.keep2 = 90; + tsdb.config.keep = 365; + tsdb.config.precision = TSDB_TIME_PRECISION_MILLI; + tsdb.config.update = TD_ROW_OVERWRITE_UPDATE; + tsdb.config.compression = TWO_STAGE_COMP; char *msg = (char *)calloc(100, 1); EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); @@ -283,16 +286,16 @@ TEST(testCase, tSmaInsertTest) { void * buf = NULL; EXPECT_EQ(tsdbMakeRoom(&buf, allocStep), 0); int32_t bufSize = taosTSizeof(buf); - int32_t numOfTables = 25; - col_id_t numOfCols = 4096; + int32_t numOfTables = 5; + col_id_t numOfCols = 10; EXPECT_GT(numOfCols, 0); pSmaData = (STSmaDataWrapper *)buf; printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData); - pSmaData->skey = 1646987196; + pSmaData->skey = 1646987196000; pSmaData->interval = 10; pSmaData->intervalUnit = TD_TIME_UNIT_MINUTE; - pSmaData->indexUid = indexUid; + pSmaData->indexUid = indexUid1; int32_t len = sizeof(STSmaDataWrapper); for (int32_t t = 0; t < numOfTables; ++t) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c1cb4f8a41..f97df62ccc 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -349,6 +349,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") From 1a7891776766e2b358dabb4823aa9724764f70c2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 14:05:40 +0800 Subject: [PATCH 13/35] sync index --- source/libs/sync/inc/syncRaftLog.h | 3 ++ source/libs/sync/inc/syncUtil.h | 2 + source/libs/sync/src/syncMain.c | 54 +++++++++++----------- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/sync/src/syncReplication.c | 19 ++++++++ source/libs/sync/src/syncUtil.c | 10 ++++ source/libs/sync/test/syncLogStoreTest.cpp | 11 ++++- 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 59b5fa94db..d979e0df15 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -27,6 +27,9 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" +#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_INVALID -1 + typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 2bbc1948dd..bc38acdfe6 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -52,6 +52,8 @@ const char* syncUtilState2String(ESyncState state); bool syncUtilCanPrint(char c); char* syncUtilprintBin(char* ptr, uint32_t len); char* syncUtilprintBin2(char* ptr, uint32_t len); +SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b); +SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 860dd96cdf..63fb4062e0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -327,33 +327,6 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } -// for debug -------------- -void syncNodePrint(SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); - fflush(NULL); - free(serialized); -} - -void syncNodePrint2(char* s, SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); - fflush(NULL); - free(serialized); -} - -void syncNodeLog(SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); -} - -void syncNodeLog2(char* s, SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); -} - int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); @@ -491,6 +464,33 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { return 0; } +// for debug -------------- +void syncNodePrint(SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + free(serialized); +} + +void syncNodePrint2(char* s, SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + free(serialized); +} + +void syncNodeLog(SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); + free(serialized); +} + +void syncNodeLog2(char* s, SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + free(serialized); +} + // ------ local funciton --------- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 27c8a26154..6ebeba1991 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -130,7 +130,7 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index dfbe5db0ed..d4c57630ad 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -14,7 +14,9 @@ */ #include "syncReplication.h" +#include "syncIndexMgr.h" #include "syncMessage.h" +#include "syncRaftEntry.h" // TLA+ Spec // AppendEntries(i, j) == @@ -42,7 +44,24 @@ // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* pDestId = &(pSyncNode->peersId[i]); + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; + SyncTerm preLogTerm = 0; + if (preLogIndex >= 0) { + SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); + preLogTerm = pPreEntry->term; + } else { + preLogTerm = 0; + } + // SyncTerm lastIndex = + // pSyncNode->pLogStore->getLastIndex < nextIndex ? pSyncNode->pLogStore->getLastIndex : nextIndex; + } + return ret; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index b78971bf37..4e56a9670f 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -184,4 +184,14 @@ char* syncUtilprintBin2(char* ptr, uint32_t len) { p += n; } return s; +} + +SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b) { + SyncIndex r = a < b ? a : b; + return r; +} + +SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) { + SyncIndex r = a > b ? a : b; + return r; } \ No newline at end of file diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 602fdee8c2..1b05f76fa2 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -81,7 +81,10 @@ SSyncNode* syncNodeInit() { SSyncNode* syncInitTest() { return syncNodeInit(); } void logStoreTest() { - logStorePrint(pSyncNode->pLogStore); + logStorePrint2((char*)"logStoreTest2", pSyncNode->pLogStore); + + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID); + for (int i = 0; i < 5; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); @@ -97,6 +100,10 @@ void logStoreTest() { // syncEntryPrint2((char*)"write entry:", pEntry); pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); syncEntryDestory(pEntry); + + if (i == 0) { + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN); + } } logStorePrint(pSyncNode->pLogStore); @@ -129,6 +136,8 @@ int main(int argc, char** argv) { ret = syncEnvStart(); assert(ret == 0); + taosRemoveDir("./wal_test"); + pSyncNode = syncInitTest(); assert(pSyncNode != NULL); From 0deef5aa1ea0a9732059f2576a42c4d1cc7441d4 Mon Sep 17 00:00:00 2001 From: ubuntu Date: Mon, 14 Mar 2022 16:03:01 +0800 Subject: [PATCH 14/35] update trans --- source/libs/transport/inc/transComm.h | 21 +++++- source/libs/transport/src/trans.c | 27 ++++++- source/libs/transport/src/transCli.c | 104 ++++++++++++-------------- source/libs/transport/src/transSrv.c | 29 ++++--- 4 files changed, 106 insertions(+), 75 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index f2ac77fe61..06d5a7da91 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -120,6 +120,10 @@ typedef struct { // SEpSet* pSet; // for synchronous API } SRpcReqContext; +typedef SRpcMsg STransMsg; +typedef SRpcInfo STrans; +typedef SRpcConnInfo STransHandleInfo; + typedef struct { SEpSet epSet; // ip list provided by app void* ahandle; // handle provided by app @@ -134,7 +138,7 @@ typedef struct { int8_t connType; // connection type int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API + STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API int hThrdIdx; @@ -143,6 +147,7 @@ typedef struct { // SEpSet* pSet; // for synchronous API } STransConnCtx; + #pragma pack(push, 1) typedef struct { @@ -243,10 +248,24 @@ bool transReadComplete(SConnBuffer* connBuf); int transSetConnOption(uv_tcp_t* stream); + void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); void transUnrefCliHandle(void* handle); + +void transSendRequest(void *shandle, const char *ip, uint32_t port, STransMsg *pMsg); +void transSendRecv(void* shandle, const char *ip, uint32_t port, STransMsg *pMsg, STransMsg *pRsp); +void transSendResponse(const STransMsg* pMsg); +int transGetConnInfo(void *thandle, STransHandleInfo *pInfo); + + +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void transCloseClient(void *arg); +void transCloseServer(void *arg); + #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4d244665c7..015018f73f 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -18,8 +18,9 @@ #include "transComm.h" void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { - taosInitServer, taosInitClient}; -void (*taosCloseHandle[])(void* arg) = {taosCloseServer, taosCloseClient}; + transInitServer, transInitClient}; + +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void* rpcOpen(const SRpcInit* pInit) { SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); @@ -34,11 +35,12 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; pRpc->pfp = pInit->pfp; + pRpc->mfp = pInit->mfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { - pRpc->numOfThreads = pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } pRpc->connType = pInit->connType; @@ -116,6 +118,24 @@ int32_t rpcInit() { return 0; } +void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg* pMsg, int64_t *pRid) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + transSendRequest(shandle, ip, port, pMsg); +} +void rpcSendRecv(void* shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + transSendRecv(shandle, ip, port, pMsg, pRsp); +} + +void rpcSendResponse(const SRpcMsg *pMsg) { + transSendResponse(pMsg); +} +int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { + return transGetConnInfo((void *)thandle, pInfo); +} + void rpcCleanup(void) { // impl later // @@ -129,6 +149,7 @@ void rpcRefHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); (*taosRefHandle[type])(handle); } + void rpcUnrefHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); (*taosUnRefHandle[type])(handle); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ce3c1c2dc8..c55d699e4f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -42,7 +42,7 @@ typedef struct SCliConn { typedef struct SCliMsg { STransConnCtx* ctx; - SRpcMsg msg; + STransMsg msg; queue q; uint64_t st; } SCliMsg; @@ -105,9 +105,9 @@ static void cliHandleExcept(SCliConn* conn); static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliSendQuit(SCliThrdObj* thrd); -static void destroyUserdata(SRpcMsg* userdata); +static void destroyUserdata(STransMsg* userdata); -static int cliRBChoseIdx(SRpcInfo* pTransInst); +static int cliRBChoseIdx(STrans* pTransInst); static void destroyCmsg(SCliMsg* cmsg); static void transDestroyConnCtx(STransConnCtx* ctx); @@ -118,7 +118,7 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) -#define CONN_GET_INST_LABEL(conn) (((SRpcInfo*)(((SCliThrdObj*)conn->hostThrd)->pTransInst))->label) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)conn->hostThrd)->pTransInst))->label) #define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ do { \ if (thrd->quit) { \ @@ -135,14 +135,14 @@ static void destroyThrdObj(SCliThrdObj* pThrd); } \ } while (0); -#define CONN_PERSIST_BY_APP(conn) do { if (conn->persist == false) { conn->persist = true; transRefCliHandle(conn);}} while(0) +#define CONN_SET_PERSIST_BY_APP(conn) do { if (conn->persist == false) { conn->persist = true; transRefCliHandle(conn);}} while(0) #define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) static void* cliWorkThread(void* arg); -static void cliHandleResp(SCliConn* conn) { +void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; - SRpcInfo* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); @@ -151,7 +151,7 @@ static void cliHandleResp(SCliConn* conn) { // buf's mem alread translated to rpcMsg.pCont transClearBuffer(&conn->readBuf); - SRpcMsg rpcMsg = {0}; + STransMsg rpcMsg = {0}; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; @@ -172,7 +172,7 @@ static void cliHandleResp(SCliConn* conn) { if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; - CONN_PERSIST_BY_APP(conn); + CONN_SET_PERSIST_BY_APP(conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -204,7 +204,8 @@ static void cliHandleResp(SCliConn* conn) { // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } } -static void cliHandleExcept(SCliConn* pConn) { + +void cliHandleExcept(SCliConn* pConn) { if (pConn->data == NULL) { if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { transUnrefCliHandle(pConn); @@ -212,12 +213,12 @@ static void cliHandleExcept(SCliConn* pConn) { } } SCliThrdObj* pThrd = pConn->hostThrd; - SRpcInfo* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; SCliMsg* pMsg = pConn->data; STransConnCtx *pCtx = pMsg ? pMsg->ctx : NULL; - SRpcMsg rpcMsg = {0}; + STransMsg rpcMsg = {0}; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; rpcMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; rpcMsg.ahandle = NULL; @@ -243,9 +244,9 @@ static void cliHandleExcept(SCliConn* pConn) { transUnrefCliHandle(pConn); } -static void cliTimeoutCb(uv_timer_t* handle) { +void cliTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; - SRpcInfo* pRpc = pThrd->pTransInst; + STrans* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pRpc->label); @@ -267,11 +268,12 @@ static void cliTimeoutCb(uv_timer_t* handle) { pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } -static void* createConnPool(int size) { + +void* createConnPool(int size) { // thread local, no lock return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } -static void* destroyConnPool(void* pool) { +void* destroyConnPool(void* pool) { SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { @@ -316,7 +318,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); - SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + STrans* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); @@ -411,16 +413,16 @@ static void cliSendCb(uv_write_t* req, int status) { uv_read_start((uv_stream_t*)pConn->stream, cliAllocBufferCb, cliRecvCb); } -static void cliSend(SCliConn* pConn) { +void cliSend(SCliConn* pConn) { CONN_HANDLE_BROKEN(pConn); SCliMsg* pCliMsg = pConn->data; STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; - SRpcInfo* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; - SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); + STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); int msgLen = transMsgLenFromCont(pMsg->contLen); @@ -458,7 +460,8 @@ static void cliSend(SCliConn* pConn) { _RETURE: return; } -static void cliConnCb(uv_connect_t* req, int status) { + +void cliConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; if (status != 0) { @@ -488,7 +491,8 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { pThrd->quit = true; uv_stop(pThrd->loop); } -static SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { + +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = NULL; if (pMsg->msg.handle != NULL) { conn = (SCliConn*)(pMsg->msg.handle); @@ -502,13 +506,14 @@ static SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { } return conn; } -static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { + +void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { uint64_t et = taosGetTimestampUs(); uint64_t el = et - pMsg->st; - tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((SRpcInfo*)pThrd->pTransInst)->label, el); + tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el); STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { @@ -567,10 +572,10 @@ static void* cliWorkThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SCliObj* cli = calloc(1, sizeof(SCliObj)); - SRpcInfo* pRpc = shandle; + STrans* pRpc = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); @@ -589,7 +594,7 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, return cli; } -static void destroyUserdata(SRpcMsg* userdata) { +static void destroyUserdata(STransMsg* userdata) { if (userdata->pCont == NULL) { return; } @@ -645,12 +650,20 @@ static void transDestroyConnCtx(STransConnCtx* ctx) { free(ctx); } // -static void cliSendQuit(SCliThrdObj* thrd) { +void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully SCliMsg* msg = calloc(1, sizeof(SCliMsg)); transSendAsync(thrd->asyncPool, &msg->q); } -void taosCloseClient(void* arg) { +int cliRBChoseIdx(STrans* pTransInst) { + int64_t index = pTransInst->index; + if (pTransInst->index++ >= pTransInst->numOfThreads) { + pTransInst->index = 0; + } + return index % pTransInst->numOfThreads; +} + +void transCloseClient(void* arg) { SCliObj* cli = arg; for (int i = 0; i < cli->numOfThreads; i++) { cliSendQuit(cli->pThreadObj[i]); @@ -659,13 +672,6 @@ void taosCloseClient(void* arg) { free(cli->pThreadObj); free(cli); } -static int cliRBChoseIdx(SRpcInfo* pTransInst) { - int64_t index = pTransInst->index; - if (pTransInst->index++ >= pTransInst->numOfThreads) { - pTransInst->index = 0; - } - return index % pTransInst->numOfThreads; -} void transRefCliHandle(void* handle) { if (handle == NULL) { return; @@ -681,17 +687,11 @@ void transUnrefCliHandle(void* handle) { if (ref == 0) { cliDestroyConn((SCliConn*)handle, true); } - - // unref cli handle } -void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { - // impl later - char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); - uint32_t port = pEpSet->eps[pEpSet->inUse].port; - SRpcInfo* pTransInst = (SRpcInfo*)shandle; - - int index = CONN_HOST_THREAD_INDEX(pMsg->handle); +void transSendRequest(void *shandle, const char *ip, uint32_t port, STransMsg *pMsg) { + STrans* pTransInst = (STrans*)shandle; + int index = CONN_HOST_THREAD_INDEX((SCliConn *)pMsg->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } @@ -718,13 +718,8 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); } - -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { - char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); - uint32_t port = pEpSet->eps[pEpSet->inUse].port; - - SRpcInfo* pTransInst = (SRpcInfo*)shandle; - +void transSendRecv(void* shandle, const char *ip, uint32_t port, STransMsg *pReq, STransMsg *pRsp) { + STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX(pReq->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); @@ -751,7 +746,6 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { tsem_wait(pSem); tsem_destroy(pSem); free(pSem); - - return; } + #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c4c3d9ed0a..f9c014902c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -37,8 +37,7 @@ typedef struct SSrvConn { struct sockaddr_in addr; struct sockaddr_in locaddr; - // SRpcMsg sendMsg; - // del later + char secured; int spi; char info[64]; @@ -49,7 +48,7 @@ typedef struct SSrvConn { typedef struct SSrvMsg { SSrvConn* pConn; - SRpcMsg msg; + STransMsg msg; queue q; } SSrvMsg; @@ -207,20 +206,20 @@ static void uvHandleReq(SSrvConn* pConn) { pConn->inType = pHead->msgType; - SRpcInfo* pRpc = (SRpcInfo*)p->shandle; + STrans* pRpc = (STrans*)p->shandle; pHead->code = htonl(pHead->code); int32_t dlen = 0; if (transDecompressMsg(NULL, 0, NULL)) { // add compress later - // pHead = rpcDecompressRpcMsg(pHead); + // pHead = rpcDecompresSTransMsg(pHead); } else { pHead->msgLen = htonl(pHead->msgLen); // impl later // } - SRpcMsg rpcMsg; + STransMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; @@ -319,7 +318,7 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { tTrace("server conn %p prepare to send resp", smsg->pConn); SSrvConn* pConn = smsg->pConn; - SRpcMsg* pMsg = &smsg->msg; + STransMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; @@ -547,7 +546,7 @@ static bool addHandleToWorkloop(void* arg) { return false; } - // SRpcInfo* pRpc = pThrd->shandle; + // STrans* pRpc = pThrd->shandle; uv_pipe_init(pThrd->loop, pThrd->pipe, 1); uv_pipe_open(pThrd->pipe, pThrd->fd); @@ -668,7 +667,7 @@ static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { return msgLen; } -void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SServerObj* srv = calloc(1, sizeof(SServerObj)); srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); srv->numOfThreads = numOfThreads; @@ -720,7 +719,7 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, return srv; End: - taosCloseServer(srv); + transCloseServer(srv); return NULL; } @@ -740,7 +739,7 @@ void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { transSendAsync(pThrd->asyncPool, &srvMsg->q); } -void taosCloseServer(void* arg) { +void transCloseServer(void* arg) { // impl later SServerObj* srv = arg; for (int i = 0; i < srv->numOfThreads; i++) { @@ -786,7 +785,7 @@ void transUnrefSrvHandle(void* handle) { } // unref srv handle } -void rpcSendResponse(const SRpcMsg* pMsg) { +void transSendResponse(const STransMsg* pMsg) { if (pMsg->handle == NULL) { return; } @@ -799,14 +798,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } - -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { +int transGetConnInfo(void *thandle, STransHandleInfo *pInfo) { SSrvConn* pConn = thandle; - struct sockaddr_in addr = pConn->addr; + pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); pInfo->clientPort = ntohs(addr.sin_port); - tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); return 0; } From ed788d3991f3b93aeeac8b040d2ddb32619d8c08 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 14 Mar 2022 16:11:17 +0800 Subject: [PATCH 15/35] formate --- source/libs/transport/inc/transComm.h | 20 ++++----- source/libs/transport/src/transCli.c | 62 +++++++++++++++------------ source/libs/transport/src/transSrv.c | 10 ++--- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 06d5a7da91..99f890d3a0 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -120,8 +120,8 @@ typedef struct { // SEpSet* pSet; // for synchronous API } SRpcReqContext; -typedef SRpcMsg STransMsg; -typedef SRpcInfo STrans; +typedef SRpcMsg STransMsg; +typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; typedef struct { @@ -139,7 +139,7 @@ typedef struct { int64_t rid; // refId returned by taosAddRef STransMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API + tsem_t* pSem; // for synchronous API int hThrdIdx; char* ip; @@ -147,7 +147,6 @@ typedef struct { // SEpSet* pSet; // for synchronous API } STransConnCtx; - #pragma pack(push, 1) typedef struct { @@ -248,24 +247,21 @@ bool transReadComplete(SConnBuffer* connBuf); int transSetConnOption(uv_tcp_t* stream); - void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); void transUnrefCliHandle(void* handle); - -void transSendRequest(void *shandle, const char *ip, uint32_t port, STransMsg *pMsg); -void transSendRecv(void* shandle, const char *ip, uint32_t port, STransMsg *pMsg, STransMsg *pRsp); +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg); +void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); void transSendResponse(const STransMsg* pMsg); -int transGetConnInfo(void *thandle, STransHandleInfo *pInfo); - +int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); -void transCloseClient(void *arg); -void transCloseServer(void *arg); +void transCloseClient(void* arg); +void transCloseServer(void* arg); #endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c55d699e4f..4af02a982e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -42,7 +42,7 @@ typedef struct SCliConn { typedef struct SCliMsg { STransConnCtx* ctx; - STransMsg msg; + STransMsg msg; queue q; uint64_t st; } SCliMsg; @@ -122,7 +122,7 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ do { \ if (thrd->quit) { \ - cliHandleExcept(conn); \ + cliHandleExcept(conn); \ goto _RETURE; \ } \ } while (0) @@ -130,19 +130,25 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_HANDLE_BROKEN(conn) \ do { \ if (conn->broken) { \ - cliHandleExcept(conn); \ + cliHandleExcept(conn); \ goto _RETURE; \ } \ } while (0); -#define CONN_SET_PERSIST_BY_APP(conn) do { if (conn->persist == false) { conn->persist = true; transRefCliHandle(conn);}} while(0) +#define CONN_SET_PERSIST_BY_APP(conn) \ + do { \ + if (conn->persist == false) { \ + conn->persist = true; \ + transRefCliHandle(conn); \ + } \ + } while (0) #define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) static void* cliWorkThread(void* arg); -void cliHandleResp(SCliConn* conn) { +void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); @@ -156,16 +162,16 @@ void cliHandleResp(SCliConn* conn) { rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; rpcMsg.msgType = pHead->msgType; - rpcMsg.ahandle = NULL; + rpcMsg.ahandle = NULL; - SCliMsg* pMsg = conn->data; - STransConnCtx *pCtx = pMsg ? pMsg->ctx : NULL; + SCliMsg* pMsg = conn->data; + STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; } else { - rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } - //if (rpcMsg.ahandle == NULL) { + // if (rpcMsg.ahandle == NULL) { // tDebug("%s cli conn %p handle except", CONN_GET_INST_LABEL(conn), conn); // return; //} @@ -207,26 +213,26 @@ void cliHandleResp(SCliConn* conn) { void cliHandleExcept(SCliConn* pConn) { if (pConn->data == NULL) { - if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { - transUnrefCliHandle(pConn); - return; - } + if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { + transUnrefCliHandle(pConn); + return; + } } SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; SCliMsg* pMsg = pConn->data; - STransConnCtx *pCtx = pMsg ? pMsg->ctx : NULL; + STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg rpcMsg = {0}; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; rpcMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; - rpcMsg.ahandle = NULL; + rpcMsg.ahandle = NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; } else { - rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } if (pCtx == NULL || pCtx->pSem == NULL) { @@ -246,7 +252,7 @@ void cliHandleExcept(SCliConn* pConn) { void cliTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; - STrans* pRpc = pThrd->pTransInst; + STrans* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pRpc->label); @@ -420,7 +426,7 @@ void cliSend(SCliConn* pConn) { STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); @@ -513,7 +519,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el); STransConnCtx* pCtx = pMsg->ctx; - STrans* pTransInst = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { @@ -534,7 +540,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); } - + conn->hThrdIdx = pCtx->hThrdIdx; } static void cliAsyncCb(uv_async_t* handle) { @@ -543,7 +549,7 @@ static void cliAsyncCb(uv_async_t* handle) { SCliMsg* pMsg = NULL; // batch process to avoid to lock/unlock frequently - queue wq; + queue wq; pthread_mutex_lock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); pthread_mutex_unlock(&item->mtx); @@ -689,9 +695,9 @@ void transUnrefCliHandle(void* handle) { } } -void transSendRequest(void *shandle, const char *ip, uint32_t port, STransMsg *pMsg) { +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) { STrans* pTransInst = (STrans*)shandle; - int index = CONN_HOST_THREAD_INDEX((SCliConn *)pMsg->handle); + int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } @@ -718,9 +724,9 @@ void transSendRequest(void *shandle, const char *ip, uint32_t port, STransMsg *p SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); } -void transSendRecv(void* shandle, const char *ip, uint32_t port, STransMsg *pReq, STransMsg *pRsp) { +void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)shandle; - int index = CONN_HOST_THREAD_INDEX(pReq->handle); + int index = CONN_HOST_THREAD_INDEX(pReq->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index f9c014902c..c236a69f4e 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -48,7 +48,7 @@ typedef struct SSrvConn { typedef struct SSrvMsg { SSrvConn* pConn; - STransMsg msg; + STransMsg msg; queue q; } SSrvMsg; @@ -317,8 +317,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); - SSrvConn* pConn = smsg->pConn; - STransMsg* pMsg = &smsg->msg; + SSrvConn* pConn = smsg->pConn; + STransMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; @@ -798,8 +798,8 @@ void transSendResponse(const STransMsg* pMsg) { tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } -int transGetConnInfo(void *thandle, STransHandleInfo *pInfo) { - SSrvConn* pConn = thandle; +int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { + SSrvConn* pConn = thandle; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); From 7d897369d5259aad38e41dcfa3ab0cc2094d2ea4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 16:27:25 +0800 Subject: [PATCH 16/35] sync index --- source/libs/sync/inc/syncInt.h | 28 ++- source/libs/sync/src/syncMain.c | 408 ++++++++++++++++---------------- 2 files changed, 220 insertions(+), 216 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index df277e2d7e..8866603b8e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -67,9 +67,6 @@ extern "C" { } \ } -struct SRaft; -typedef struct SRaft SRaft; - struct SyncTimeout; typedef struct SyncTimeout SyncTimeout; @@ -117,8 +114,10 @@ typedef struct SSyncNode { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; - SWal* pWal; - void* rpcClient; + + // sync io + SWal* pWal; + void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); void* queue; int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg); @@ -164,7 +163,7 @@ typedef struct SSyncNode { int32_t pingTimerMS; uint64_t pingTimerLogicClock; uint64_t pingTimerLogicClockUser; - TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp + TAOS_TMR_CALLBACK FpPingTimerCB; // Timer Fp uint64_t pingTimerCounter; // elect timer @@ -172,7 +171,7 @@ typedef struct SSyncNode { int32_t electTimerMS; uint64_t electTimerLogicClock; uint64_t electTimerLogicClockUser; - TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp + TAOS_TMR_CALLBACK FpElectTimerCB; // Timer Fp uint64_t electTimerCounter; // heartbeat timer @@ -180,7 +179,7 @@ typedef struct SSyncNode { int32_t heartbeatTimerMS; uint64_t heartbeatTimerLogicClock; uint64_t heartbeatTimerLogicClockUser; - TAOS_TMR_CALLBACK FpHeartbeatTimer; // Timer Fp + TAOS_TMR_CALLBACK FpHeartbeatTimerCB; // Timer Fp uint64_t heartbeatTimerCounter; // callback @@ -194,16 +193,17 @@ typedef struct SSyncNode { } SSyncNode; +// open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); +// ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg); -int32_t syncNodePingAll(SSyncNode* pSyncNode); -int32_t syncNodePingPeers(SSyncNode* pSyncNode); int32_t syncNodePingSelf(SSyncNode* pSyncNode); +int32_t syncNodePingPeers(SSyncNode* pSyncNode); +int32_t syncNodePingAll(SSyncNode* pSyncNode); +// timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); @@ -211,6 +211,10 @@ int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); + +// utils -------------- +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 63fb4062e0..4caa376599 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -31,13 +31,16 @@ static int32_t tsNodeRefId = -1; // ------ local funciton --------- +// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +// on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +// raft algorithm ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); @@ -48,16 +51,20 @@ static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { - sTrace("syncInit ok"); - return 0; + int32_t ret = syncEnvStart(); + return ret; } -void syncCleanUp() { sTrace("syncCleanUp ok"); } +void syncCleanUp() { + int32_t ret = syncEnvStop(); + assert(ret == 0); +} int64_t syncStart(const SSyncInfo* pSyncInfo) { + int32_t ret = 0; SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); - return 0; + return ret; } void syncStop(int64_t rid) { @@ -65,9 +72,13 @@ void syncStop(int64_t rid) { syncNodeClose(pSyncNode); } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { + int32_t ret = 0; + return ret; +} int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = 0; SSyncNode* pSyncNode = NULL; // get pointer from rid if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); @@ -75,11 +86,13 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncClientRequestDestroy(pSyncMsg); + ret = 0; + } else { sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); - return -1; // need define err code !! + ret = -1; // need define err code !! } - return 0; + return ret; } ESyncState syncGetMyRole(int64_t rid) { @@ -89,6 +102,7 @@ ESyncState syncGetMyRole(int64_t rid) { void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} +// open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); @@ -162,7 +176,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->pingTimerMS = PING_TIMER_MS; atomic_store_64(&pSyncNode->pingTimerLogicClock, 0); atomic_store_64(&pSyncNode->pingTimerLogicClockUser, 0); - pSyncNode->FpPingTimer = syncNodeEqPingTimer; + pSyncNode->FpPingTimerCB = syncNodeEqPingTimer; pSyncNode->pingTimerCounter = 0; // init elect timer @@ -170,7 +184,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->electTimerMS = syncUtilElectRandomMS(); atomic_store_64(&pSyncNode->electTimerLogicClock, 0); atomic_store_64(&pSyncNode->electTimerLogicClockUser, 0); - pSyncNode->FpElectTimer = syncNodeEqElectTimer; + pSyncNode->FpElectTimerCB = syncNodeEqElectTimer; pSyncNode->electTimerCounter = 0; // init heartbeat timer @@ -178,7 +192,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->heartbeatTimerMS = HEARTBEAT_TIMER_MS; atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, 0); atomic_store_64(&pSyncNode->heartbeatTimerLogicClockUser, 0); - pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer; + pSyncNode->FpHeartbeatTimerCB = syncNodeEqHeartbeatTimer; pSyncNode->heartbeatTimerCounter = 0; // init callback @@ -194,10 +208,146 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { } void syncNodeClose(SSyncNode* pSyncNode) { + int32_t ret; assert(pSyncNode != NULL); + + ret = raftStoreClose(pSyncNode->pRaftStore); + assert(ret == 0); + + voteGrantedDestroy(pSyncNode->pVotesGranted); + votesRespondDestory(pSyncNode->pVotesRespond); + syncIndexMgrDestroy(pSyncNode->pNextIndex); + syncIndexMgrDestroy(pSyncNode->pMatchIndex); + logStoreDestory(pSyncNode->pLogStore); + + syncNodeStopPingTimer(pSyncNode); + syncNodeStopElectTimer(pSyncNode); + syncNodeStopHeartbeatTimer(pSyncNode); + free(pSyncNode); } +// ping -------------- +int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + syncPingLog2((char*)"==syncNodePing==", pMsg); + int32_t ret = 0; + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); + + ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + return ret; +} + +int32_t syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + assert(ret == 0); + + syncPingDestroy(pMsg); + return ret; +} + +int32_t syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); + assert(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +int32_t syncNodePingAll(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); + assert(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +// timer control -------------- +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pPingTimer); + pSyncNode->pPingTimer = NULL; + return ret; +} + +int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { + int32_t ret = 0; + pSyncNode->electTimerMS = ms; + taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pElectTimer); + atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pElectTimer); + pSyncNode->pElectTimer = NULL; + return ret; +} + +int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { + int32_t ret = 0; + syncNodeStopElectTimer(pSyncNode); + syncNodeStartElectTimer(pSyncNode, ms); + return ret; +} + +int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pHeartbeatTimer); + pSyncNode->pHeartbeatTimer = NULL; + return ret; +} + +// utils -------------- +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + syncUtilraftId2EpSet(destRaftId, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + cJSON* syncNode2Json(const SSyncNode* pSyncNode) { char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); @@ -253,12 +403,22 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); + char tmpBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf)); + cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf); // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); // ping timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); @@ -268,8 +428,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer); - cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter); cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); @@ -281,8 +441,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer); - cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter); cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); @@ -294,8 +454,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter); cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); @@ -327,143 +487,6 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); - return 0; -} - -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); - return 0; -} - -int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - sTrace("syncNodePing pSyncNode:%p ", pSyncNode); - int32_t ret = 0; - - SRpcMsg rpcMsg; - syncPing2RpcMsg(pMsg, &rpcMsg); - syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); - - { - cJSON* pJson = syncPing2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("syncNodePing pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - { - SyncPing* pMsg2 = rpcMsg.pCont; - cJSON* pJson = syncPing2Json(pMsg2); - char* serialized = cJSON_Print(pJson); - sTrace("syncNodePing rpcMsg.pCont:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - return ret; -} - -int32_t syncNodePingAll(SSyncNode* pSyncNode) { - sTrace("syncNodePingAll pSyncNode:%p ", pSyncNode); - int32_t ret = 0; - for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); - } -} - -int32_t syncNodePingPeers(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); - } -} - -int32_t syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); -} - -int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { - atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); - pSyncNode->pingTimerMS = PING_TIMER_MS; - if (pSyncNode->pPingTimer == NULL) { - pSyncNode->pPingTimer = - taosTmrStart(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pPingTimer); - } - return 0; -} - -int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1); - pSyncNode->pingTimerMS = TIMER_MAX_MS; - return 0; -} - -int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { - pSyncNode->electTimerMS = ms; - atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); - if (pSyncNode->pElectTimer == NULL) { - pSyncNode->pElectTimer = - taosTmrStart(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pElectTimer); - } - return 0; -} - -int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); - pSyncNode->electTimerMS = TIMER_MAX_MS; - return 0; -} - -int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { - syncNodeStopElectTimer(pSyncNode); - syncNodeStartElectTimer(pSyncNode, ms); - return 0; -} - -int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { - atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); - if (pSyncNode->pHeartbeatTimer == NULL) { - pSyncNode->pHeartbeatTimer = - taosTmrStart(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pHeartbeatTimer); - } - return 0; -} - -int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); - pSyncNode->heartbeatTimerMS = TIMER_MAX_MS; - return 0; -} - // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -492,61 +515,23 @@ void syncNodeLog2(char* s, SSyncNode* pObj) { } // ------ local funciton --------- -static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { - int32_t ret = 0; - sTrace("<-- syncNodeOnPingCb -->"); - - { - cJSON* pJson = syncPing2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnPingCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); - SRpcMsg rpcMsg; - syncPingReply2RpcMsg(pMsgReply, &rpcMsg); - syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); - - return ret; -} - -static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { - int32_t ret = 0; - sTrace("<-- syncNodeOnPingReplyCb -->"); - - { - cJSON* pJson = syncPingReply2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnPingReplyCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - return ret; -} - +// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { - sTrace("<-- syncNodeEqPingTimer -->"); - SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock), pSyncNode->pingTimerMS, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); - // reset timer ms - // pSyncNode->pingTimerMS += 100; - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("syncNodeEqPingTimer: pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", pSyncNode->pingTimerLogicClock, - pSyncNode->pingTimerLogicClockUser); + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", + pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -555,19 +540,18 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { if (atomic_load_64(&pSyncNode->electTimerLogicClockUser) <= atomic_load_64(&pSyncNode->electTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_ELECTION, atomic_load_64(&pSyncNode->electTimerLogicClock), pSyncNode->electTimerMS, pSyncNode); - - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); // reset timer ms pSyncNode->electTimerMS = syncUtilElectRandomMS(); - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("syncNodeEqElectTimer: electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", + sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); } } @@ -579,23 +563,39 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), pSyncNode->heartbeatTimerMS, pSyncNode); - SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); - // reset timer ms - // pSyncNode->heartbeatTimerMS += 100; - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { - sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } } +// on message ---- +static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { + int32_t ret = 0; + syncPingLog2("==syncNodeOnPingCb==", pMsg); + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pMsgReply, &rpcMsg); + syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + + return ret; +} + +static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { + int32_t ret = 0; + syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); + return ret; +} + +// raft algorithm ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; From a471152b9696701db65cc78c6cf82160a57b95d2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 16:34:01 +0800 Subject: [PATCH 17/35] sync index --- source/libs/sync/src/syncMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4caa376599..15369cc32d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -40,7 +40,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -// raft algorithm ---- +// raft state change ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); @@ -595,7 +595,7 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { return ret; } -// raft algorithm ---- +// raft state change ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; From 9734b9b0ec59f853a9828a638d4b14de6d805e55 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 17:02:43 +0800 Subject: [PATCH 18/35] sync index --- source/libs/sync/src/syncMain.c | 47 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 15369cc32d..7f183021cf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -17,11 +17,13 @@ #include "sync.h" #include "syncAppendEntries.h" #include "syncAppendEntriesReply.h" +#include "syncElection.h" #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncReplication.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncTimeout.h" @@ -41,13 +43,15 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); // raft state change ---- -static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); +static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); + static void syncNodeCandidate2Leader(SSyncNode* pSyncNode); +static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); static void syncNodeLeader2Follower(SSyncNode* pSyncNode); static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); + // --------------------------------- int32_t syncInit() { @@ -596,12 +600,15 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { } // raft state change ---- -static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { +static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; + raftStorePersist(pSyncNode->pRaftStore); + + syncNodeBecomeFollower(pSyncNode); + pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; raftStorePersist(pSyncNode->pRaftStore); - syncNodeBecomeFollower(pSyncNode); } } @@ -610,9 +617,11 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { pSyncNode->leaderCache = EMPTY_RAFT_ID; } + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); + int32_t electMS = syncUtilElectRandomMS(); - syncNodeStartElectTimer(pSyncNode, electMS); + syncNodeRestartElectTimer(pSyncNode, electMS); } // TLA+ Spec @@ -637,13 +646,23 @@ static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_LEADER; pSyncNode->leaderCache = pSyncNode->myRaftId; - // next Index +=1 - // match Index = 0; + for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { + pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + } + + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; + } syncNodeStopElectTimer(pSyncNode); syncNodeStartHeartbeatTimer(pSyncNode); + syncNodeReplicate(pSyncNode); +} - // appendEntries; +static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + assert(voteGrantedMajority(pSyncNode->pVotesGranted)); + syncNodeBecomeLeader(pSyncNode); } static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -651,8 +670,12 @@ static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; } -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {} +static void syncNodeLeader2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncNodeBecomeFollower(pSyncNode); +} -static void syncNodeLeader2Follower(SSyncNode* pSyncNode) {} - -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {} +static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + syncNodeBecomeFollower(pSyncNode); +} From 65870538fd90fc575a3abbe61cb8dfc317b2cf1f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 14 Mar 2022 17:38:10 +0800 Subject: [PATCH 19/35] test insert and query --- source/dnode/vnode/inc/tsdb.h | 7 +- source/dnode/vnode/src/inc/tsdbDBDef.h | 1 + source/dnode/vnode/src/inc/tsdbSma.h | 17 +- source/dnode/vnode/src/tsdb/tsdbBDBImpl.c | 43 ++- source/dnode/vnode/src/tsdb/tsdbSma.c | 308 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbWrite.c | 5 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 69 ++++- 7 files changed, 314 insertions(+), 136 deletions(-) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 25bac86f71..87edfb8dde 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -95,7 +95,6 @@ int tsdbCommit(STsdb *pTsdb); * @return int32_t */ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); - int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); /** @@ -107,6 +106,12 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); */ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); +// TODO: This is the basic params, and should wrap the params to a queryHandle. +int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, + int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, + int32_t nMaxResult); + + // STsdbCfg int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); diff --git a/source/dnode/vnode/src/inc/tsdbDBDef.h b/source/dnode/vnode/src/inc/tsdbDBDef.h index 7740dd0fab..2e37b0ba45 100644 --- a/source/dnode/vnode/src/inc/tsdbDBDef.h +++ b/source/dnode/vnode/src/inc/tsdbDBDef.h @@ -35,6 +35,7 @@ void tsdbCloseDBF(SDBFile* pDBF); int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); void tsdbCloseBDBEnv(DB_ENV* pEnv); int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); +void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index c54fdf85a3..649b5a2d47 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -27,29 +27,18 @@ struct SSmaEnv { }; #define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) -// insert/update interface -int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); -int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); - -// query interface -// TODO: This is the basic params, and should wrap the params to a queryHandle. -int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult); - -// management interface -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg); -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); -void * tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); +void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); #if 0 int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); #endif - - // internal func static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { int32_t len = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index 4fc415cfd1..cf3351c5d8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -29,7 +29,7 @@ static void tsdbCloseBDBDb(DB *pDB); #define BDB_PERR(info, code) fprintf(stderr, "%s:%d " info " reason: %s\n", __FILE__, __LINE__, db_strerror(code)) -int tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) { +int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) { // TDBEnv is shared by a group of SDBFile if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; @@ -46,18 +46,12 @@ int tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) { return 0; } -static void *tsdbFreeDBF(SDBFile *pDBF) { - if (pDBF) { - free(pDBF); - } - return NULL; -} - void tsdbCloseDBF(SDBFile *pDBF) { if (pDBF->pDB) { tsdbCloseBDBDb(pDBF->pDB); - pDBF->pDB = tsdbFreeDBF(pDBF); + pDBF->pDB = NULL; } + tfree(pDBF->path); } int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { @@ -145,4 +139,35 @@ int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *key, uint32_t keySize, void *data, // TODO: unlock return 0; +} + +void *tsdbGetSmaDataByKey(SDBFile *pDBF, void* key, uint32_t keySize, uint32_t *valueSize) { + void *result = NULL; + DBT key1 = {0}; + DBT value1 = {0}; + int ret; + + // Set key/value + key1.data = key; + key1.size = keySize; + + // Query + // TODO: lock + ret = pDBF->pDB->get(pDBF->pDB, NULL, &key1, &value1, 0); + // TODO: unlock + if (ret != 0) { + return NULL; + } + + result = calloc(1, value1.size); + + if (result == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + *valueSize = value1.size; + memcpy(result, value1.data, value1.size); + + return result; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index f96ea644b4..0dbcb29f80 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -15,6 +15,7 @@ #include "tsdbDef.h" +#undef SMA_PRINT_DEBUG_LOG #define SMA_STORAGE_TSDB_DAYS 30 #define SMA_STORAGE_TSDB_TIMES 30 #define SMA_STORAGE_SPLIT_HOURS 24 @@ -31,24 +32,23 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb * pTsdb; - SDBFile *pDFile; - int32_t interval; // interval with the precision of DB - // TODO + STsdb * pTsdb; + SDBFile dFile; + int32_t interval; // interval with the precision of DB } STSmaWriteH; typedef struct { int32_t iter; + int32_t fid; } SmaFsIter; typedef struct { STsdb * pTsdb; - char * pDFile; // TODO: use the real DFile type, not char* + SDBFile dFile; int32_t interval; // interval with the precision of DB int32_t blockSize; // size of SMA block item int8_t storageLevel; int8_t days; SmaFsIter smaFsIter; - // TODO } STSmaReadH; typedef struct { @@ -69,23 +69,30 @@ struct SSmaStat { }; // declaration of static functions +static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); +// TODO: This is the basic params, and should wrap the params to a queryHandle. +static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, + int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, + int32_t nMaxResult); +static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg); + static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); -static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel); +static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); -static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin); -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); +static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { SSmaEnv *pEnv = NULL; @@ -408,25 +415,35 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { } /** - * @brief Insert TSma data blocks to B+Tree + * @brief Insert TSma data blocks to DB File build by B+Tree * - * @param bTree + * @param pSmaH * @param smaKey + * @param keyLen * @param pData * @param dataLen * @return int32_t */ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { - SDBFile *pDBFile = pSmaH->pDFile; + SDBFile *pDBFile = &pSmaH->dFile; // TODO: insert sma data blocks into B+Tree - tsdbDebug("insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", pDBFile->path, - *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", + REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), + *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); - if(tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0){ + if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { return TSDB_CODE_FAILED; } +#ifdef SMA_PRINT_DEBUG_LOG + uint32_t valueSize = 0; + void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + ASSERT(data != NULL); + for (uint32_t v = 0; v < valueSize; v += 8) { + tsdbWarn("vgId:%d sma data - val[%d] is %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + } +#endif return TSDB_CODE_SUCCESS; } @@ -458,41 +475,41 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit } } - switch (intervalUnit) { - case TD_TIME_UNIT_MILLISEC: - if (TSDB_TIME_PRECISION_MILLI == precision) { - return interval; - } else if (TSDB_TIME_PRECISION_MICRO == precision) { - return interval * 1e3; - } else { // nano second - return interval * 1e6; - } - break; - case TD_TIME_UNIT_MICROSEC: - if (TSDB_TIME_PRECISION_MILLI == precision) { + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us return interval / 1e3; - } else if (TSDB_TIME_PRECISION_MICRO == precision) { - return interval; - } else { // nano second - return interval * 1e3; - } - break; - case TD_TIME_UNIT_NANOSEC: - if (TSDB_TIME_PRECISION_MILLI == precision) { + } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second return interval / 1e6; - } else if (TSDB_TIME_PRECISION_MICRO == precision) { - return interval / 1e3; - } else { // nano second + } else { return interval; } break; - default: - if (TSDB_TIME_PRECISION_MILLI == precision) { + case TSDB_TIME_PRECISION_MICRO: + if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us + return interval; + } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + return interval / 1e3; + } else { return interval * 1e3; - } else if (TSDB_TIME_PRECISION_MICRO == precision) { + } + break; + case TSDB_TIME_PRECISION_NANO: + if (TD_TIME_UNIT_MICROSEC == intervalUnit) { + return interval * 1e3; + } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + return interval; + } else { return interval * 1e6; - } else { // nano second - return interval * 1e9; + } + break; + default: // ms + if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us + return interval / 1e3; + } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + return interval / 1e6; + } else { + return interval; } break; } @@ -551,37 +568,38 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { pSmaH->pTsdb = pTsdb; pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); - pSmaH->pDFile = (SDBFile *)calloc(1, sizeof(SDBFile *)); - if (!pSmaH->pDFile) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } return TSDB_CODE_SUCCESS; } static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { if (pSmaH) { - if (pSmaH->pDFile) { - tsdbCloseDBF(pSmaH->pDFile); - } + tsdbCloseDBF(&pSmaH->dFile); } } static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; - ASSERT(pSmaH->pDFile->path == NULL && pSmaH->pDFile->pDB == NULL); + ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); char tSmaFile[TSDB_FILENAME_LEN] = {0}; snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); - pSmaH->pDFile->path = strdup(tSmaFile); + pSmaH->dFile.path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; } -static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel) { - STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb); +/** + * @brief + * + * @param pTsdb + * @param interval Interval calculated by DB's precision + * @param storageLevel + * @return int32_t + */ +static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel) { + STsdbCfg *pCfg = REPO_CFG(pTsdb); int32_t daysPerFile = pCfg->daysPerFile; if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t days = 30 * (pSmaH->interval / tsTickPerDay[pCfg->precision]); + int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]); daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; } @@ -600,7 +618,7 @@ static int32_t tsdbGetTSmaDays(STSmaWriteH *pSmaH, int32_t storageLevel) { * @param msg * @return int32_t */ -int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { +static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; @@ -624,16 +642,16 @@ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 1: Judge the storage level and days int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); - int32_t daysPerFile = tsdbGetTSmaDays(&tSmaH, storageLevel); + int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); - if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, tSmaH.pDFile) != 0) { + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.pDFile->path ? tSmaH.pDFile->path : "path is NULL", tstrerror(terrno)); + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tsdbDestroyTSmaWriteH(&tSmaH); return TSDB_CODE_FAILED; } @@ -657,12 +675,12 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, char tSmaFile[TSDB_FILENAME_LEN] = {0}; snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); - pSmaH->pDFile->path = strdup(tSmaFile); + pSmaH->dFile.path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; } -int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; STSmaWriteH tSmaH = {0}; @@ -698,35 +716,35 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { } /** - * @brief Init of tSma ReadH + * @brief * * @param pSmaH * @param pTsdb - * @param param - * @param pData + * @param interval + * @param intervalUnit * @return int32_t */ -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); - // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit); + pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel); } /** * @brief Init of tSma FS * * @param pReadH - * @param param - * @param queryWin + * @param skey * @return int32_t */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { - int32_t storageLevel = 0; // tsdbGetSmaStorageLevel(param->interval, param->intervalUnit); - int32_t daysPerFile = - storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile; - pReadH->storageLevel = storageLevel; - pReadH->days = daysPerFile; - pReadH->smaFsIter.iter = 0; +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); + pSmaH->dFile.path = strdup(tSmaFile); + pSmaH->smaFsIter.iter = 0; + pSmaH->smaFsIter.fid = fid; } /** @@ -738,17 +756,18 @@ static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { * @return true * @return false */ -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { +static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; int32_t nSmaFs = taosArrayGetSize(smaFs); - pReadH->pDFile = NULL; + tsdbCloseDBF(&pReadH->dFile); +#if 0 while (pReadH->smaFsIter.iter < nSmaFs) { void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); if (pSmaFile) { // match(indexName, queryWindow) // TODO: select the file by index_name ... - pReadH->pDFile = pSmaFile; + pReadH->dFile = pSmaFile; ++pReadH->smaFsIter.iter; break; } @@ -759,41 +778,82 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) { tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); return true; } +#endif return false; } /** - * @brief Return the data between queryWin and fill the pData. + * @brief * - * @param pTsdb - * @param param + * @param pTsdb Return the data between queryWin and fill the pData. * @param pData - * @param queryWin + * @param indexUid + * @param interval + * @param intervalUnit + * @param tableUid + * @param colId + * @param pQuerySKey * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. * @return int32_t */ -int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult) { - SSmaStatItem *pItem = - (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &pData->indexUid, sizeof(pData->indexUid)); +static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, + int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, + int32_t nMaxResult) { + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); if (pItem == NULL) { // mark all window as expired and notify query module to query raw TS data. return TSDB_CODE_SUCCESS; } - int32_t nQueryWin = 0; +#if 0 + int32_t nQueryWin = taosArrayGetSize(pQuerySKey); for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY thisWindow = n; - if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) { + TSKEY skey = taosArrayGet(pQuerySKey, n); + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY)) != NULL) { // TODO: mark this window as expired. } } - +#endif +#if 0 + if (taosHashGet(pItem->expiredWindows, &querySkey, sizeof(TSKEY)) != NULL) { + // TODO: mark this window as expired. + } +#endif STSmaReadH tReadH = {0}; - tsdbInitTSmaReadH(&tReadH, pTsdb, pData); + tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); + tsdbCloseDBF(&tReadH.dFile); - tsdbInitTSmaFile(&tReadH, queryWin); + tsdbInitTSmaFile(&tReadH, querySkey); + if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { + tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + tsdbEncodeTSmaKey(tableUid, colId, querySkey, (void **)&pSmaKey); + + tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), + tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), + *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); + + void * result = NULL; + uint32_t valueSize = 0; + if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { + tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 + " since %s", + REPO_ID(pTsdb), indexUid, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), + *(int64_t *)POINTER_SHIFT(smaKey, 10), tstrerror(terrno)); + tsdbCloseDBF(&tReadH.dFile); + return TSDB_CODE_FAILED; + } +#ifdef SMA_PRINT_DEBUG_LOG + for (uint32_t v = 0; v < valueSize; v += 8) { + tsdbWarn("vgId:%d v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); + } +#endif +#if 0 int32_t nResult = 0; int64_t lastKey = 0; @@ -815,8 +875,9 @@ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow * } } } - +#endif // read data from file and fill the result + tsdbCloseDBF(&tReadH.dFile); return TSDB_CODE_SUCCESS; } @@ -854,4 +915,55 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { // } return TSDB_CODE_SUCCESS; } -#endif \ No newline at end of file +#endif + +/** + * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine + * + * @param pTsdb + * @param param + * @param msg + * @return int32_t + * TODO: Who is responsible for resource allocate and release? + */ +int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { + tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} + +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { + tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert Time-range-wise Rollup Sma(RSma) data + * + * @param pTsdb + * @param param + * @param msg + * @return int32_t + */ +int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { + tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} + +int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, + tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, int32_t nMaxResult) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySkey, + nMaxResult)) < 0) { + tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 9cccea9853..3ccb483fe4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,6 +34,7 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); } +#if 0 /** * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine * @@ -73,4 +74,6 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index f815291c77..18dca33bda 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -33,7 +33,7 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } -TEST(testCase, tSmaEncodeDecodeTest) { +TEST(testCase, tSma_Meta_Encode_Decode_Test) { // encode STSma tSma = {0}; tSma.version = 0; @@ -87,8 +87,9 @@ TEST(testCase, tSmaEncodeDecodeTest) { tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } + #if 1 -TEST(testCase, tSma_DB_Put_Get_Del_Test) { +TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { const char * smaIndexName1 = "sma_index_test_1"; const char * smaIndexName2 = "sma_index_test_2"; const char * timezone = "Asia/Shanghai"; @@ -220,16 +221,21 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { #endif #if 1 -TEST(testCase, tSmaInsertTest) { - // prepare meta +TEST(testCase, tSma_Data_Insert_Query_Test) { + // step 1: prepare meta const char * smaIndexName1 = "sma_index_test_1"; const char * timezone = "Asia/Shanghai"; const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; - const char * tagsFilter = "I'm tags filter"; + const char * tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; const char * smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; + const int64_t interval1 = 1; + const int8_t intervalUnit1 = TD_TIME_UNIT_DAY; const uint32_t nCntTSma = 2; + TSKEY skey1 = 1646987196; + const int64_t testSmaData1 = 100; + const int64_t testSmaData2 = 200; // encode STSma tSma = {0}; tSma.version = 0; @@ -261,7 +267,7 @@ TEST(testCase, tSmaInsertTest) { // save index 1 EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); - // insert data + // step 2: insert data STSmaDataWrapper *pSmaData = NULL; STsdb tsdb = {0}; STsdbCfg * pCfg = &tsdb.config; @@ -276,6 +282,21 @@ TEST(testCase, tSmaInsertTest) { tsdb.config.update = TD_ROW_OVERWRITE_UPDATE; tsdb.config.compression = TWO_STAGE_COMP; + switch (tsdb.config.precision) { + case TSDB_TIME_PRECISION_MILLI: + skey1 *= 1e3; + break; + case TSDB_TIME_PRECISION_MICRO: + skey1 *= 1e6; + break; + case TSDB_TIME_PRECISION_NANO: + skey1 *= 1e9; + break; + default: // ms + skey1 *= 1e3; + break; + } + char *msg = (char *)calloc(100, 1); EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); @@ -286,21 +307,21 @@ TEST(testCase, tSmaInsertTest) { void * buf = NULL; EXPECT_EQ(tsdbMakeRoom(&buf, allocStep), 0); int32_t bufSize = taosTSizeof(buf); - int32_t numOfTables = 5; - col_id_t numOfCols = 10; + int32_t numOfTables = 10; + col_id_t numOfCols = 4096; EXPECT_GT(numOfCols, 0); pSmaData = (STSmaDataWrapper *)buf; printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData); - pSmaData->skey = 1646987196000; - pSmaData->interval = 10; - pSmaData->intervalUnit = TD_TIME_UNIT_MINUTE; + pSmaData->skey = skey1; + pSmaData->interval = interval1; + pSmaData->intervalUnit = intervalUnit1; pSmaData->indexUid = indexUid1; int32_t len = sizeof(STSmaDataWrapper); for (int32_t t = 0; t < numOfTables; ++t) { STSmaTbData *pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len); - pTbData->tableUid = t; + pTbData->tableUid = tbUid + t; int32_t tableDataLen = sizeof(STSmaTbData); for (col_id_t c = 0; c < numOfCols; ++c) { @@ -313,8 +334,17 @@ TEST(testCase, tSmaInsertTest) { } STSmaColData *pColData = (STSmaColData *)POINTER_SHIFT(pSmaData, len + tableDataLen); pColData->colId = c + PRIMARYKEY_TIMESTAMP_COL_ID; - pColData->blockSize = ((c & 1) == 0) ? 8 : 16; + // TODO: fill col data + if ((c & 1) == 0) { + pColData->blockSize = 8; + memcpy(pColData->data, &testSmaData1, 8); + } else { + pColData->blockSize = 16; + memcpy(pColData->data, &testSmaData1, 8); + memcpy(POINTER_SHIFT(pColData->data, 8), &testSmaData2, 8); + } + tableDataLen += (sizeof(STSmaColData) + pColData->blockSize); } pTbData->dataLen = (tableDataLen - sizeof(STSmaTbData)); @@ -328,6 +358,19 @@ TEST(testCase, tSmaInsertTest) { // execute EXPECT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); + // step 3: query + uint32_t checkDataCnt = 0; + for (int32_t t = 0; t < numOfTables; ++t) { + for (col_id_t c = 0; c < numOfCols; ++c) { + EXPECT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, + c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1), + TSDB_CODE_SUCCESS); + ++checkDataCnt; + } + } + + printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); + // release data taosTZfree(buf); // release meta From 7d070abbe6a1448a70e86286a6497e9673b5e5a7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 14 Mar 2022 17:43:25 +0800 Subject: [PATCH 20/35] resource release --- source/dnode/vnode/src/tsdb/tsdbSma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 0dbcb29f80..7335e4f585 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -848,6 +848,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } + tfree(result); #ifdef SMA_PRINT_DEBUG_LOG for (uint32_t v = 0; v < valueSize; v += 8) { tsdbWarn("vgId:%d v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); From 8ae9fb6ae133f33efaf659e8732864b3c89c542b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 18:22:39 +0800 Subject: [PATCH 21/35] sync index --- source/libs/sync/inc/syncRaftStore.h | 6 +++++ source/libs/sync/inc/syncUtil.h | 1 + source/libs/sync/src/syncMain.c | 34 +++++++++++++++++++++++----- source/libs/sync/src/syncRaftStore.c | 29 ++++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 2 ++ 5 files changed, 66 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 4058d3bd1c..30f7c5d9f7 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -43,6 +43,12 @@ int32_t raftStorePersist(SRaftStore *pRaftStore); int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); +bool raftStoreHasVoted(SRaftStore *pRaftStore); +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); +void raftStoreClearVote(SRaftStore *pRaftStore); +void raftStoreNextTerm(SRaftStore *pRaftStore); +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); + // for debug ------------------- void raftStorePrint(SRaftStore *pObj); void raftStorePrint2(char *s, SRaftStore *pObj); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index bc38acdfe6..1b702c2528 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -34,6 +34,7 @@ void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); +bool syncUtilEmptyId(const SRaftId* pId); // ---- SSyncBuffer ---- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7f183021cf..da1286bd32 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -21,6 +21,7 @@ #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncMessage.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncReplication.h" @@ -52,6 +53,9 @@ static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); static void syncNodeLeader2Follower(SSyncNode* pSyncNode); static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); +// raft vote ---- +static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); +static void syncNodeVoteForSelf(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { @@ -602,13 +606,9 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // raft state change ---- static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { - pSyncNode->pRaftStore->currentTerm = term; - raftStorePersist(pSyncNode->pRaftStore); - + raftStoreSetTerm(pSyncNode->pRaftStore, term); syncNodeBecomeFollower(pSyncNode); - - pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; - raftStorePersist(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode->pRaftStore); } } @@ -679,3 +679,25 @@ static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); syncNodeBecomeFollower(pSyncNode); } + +// raft vote ---- +static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { + assert(term == pSyncNode->pRaftStore->currentTerm); + assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); + + raftStoreVote(pSyncNode->pRaftStore, pRaftId); +} + +static void syncNodeVoteForSelf(SSyncNode* pSyncNode) { + syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); + + SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->myRaftId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->voteGranted = true; + + voteGrantedVote(pSyncNode->pVotesGranted, pMsg); + votesRespondAdd(pSyncNode->pVotesRespond, pMsg); + syncRequestVoteReplyDestroy(pMsg); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3a26caa161..5ad618b9c0 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -15,6 +15,8 @@ #include "syncRaftStore.h" #include "cJSON.h" +#include "syncEnv.h" +#include "syncUtil.h" // private function static int32_t raftStoreInit(SRaftStore *pRaftStore); @@ -135,6 +137,33 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } +bool raftStoreHasVoted(SRaftStore *pRaftStore) { + bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); + return b; +} + +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { + assert(!raftStoreHasVoted(pRaftStore)); + assert(!syncUtilEmptyId(pRaftId)); + pRaftStore->voteFor = *pRaftId; + raftStorePersist(pRaftStore); +} + +void raftStoreClearVote(SRaftStore *pRaftStore) { + pRaftStore->voteFor = EMPTY_RAFT_ID; + raftStorePersist(pRaftStore); +} + +void raftStoreNextTerm(SRaftStore *pRaftStore) { + ++(pRaftStore->currentTerm); + raftStorePersist(pRaftStore); +} + +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { + pRaftStore->currentTerm = term; + raftStorePersist(pRaftStore); +} + // for debug ------------------- void raftStorePrint(SRaftStore *pObj) { char serialized[RAFT_STORE_BLOCK_SIZE]; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4e56a9670f..ba8a76c190 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -74,6 +74,8 @@ bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { return ret; } +bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } + // ---- SSyncBuffer ----- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { syncBuf->len = len; From 9aa0a8444897dd0eace2c41e2f19e7a37189b8da Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 14 Mar 2022 18:43:40 +0800 Subject: [PATCH 22/35] [add db options] --- tests/script/tsim/db/basic6.sim | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index eb12da2ccb..08ce9955b8 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -14,13 +14,19 @@ $st = $stPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 cache 16 vgroups 4 +# quorum presicion +sql create database $db vgroups 8 replica 1 days 20 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +if $rows != 1 then + return -1 +endi if $data00 != $db then return -1 endi -if $data02 != 4 then +if $data02 != 8 then return -1 endi if $data03 != 0 then @@ -32,16 +38,23 @@ endi if $data06 != 20 then return -1 endi -if $data08 != 16 then +if $data07 != 3650,3650,3650 then + return -1 +endi +if $data08 != 32 then + return -1 +endi +if $data09 != 12 then return -1 endi print =============== step2 -sql create database $db +sql_error create database $db +sql create database if not exists $db sql show databases if $rows != 1 then return -1 -endi +endi print =============== step3 sql drop database $db From 7c868aea85f1255f093ad397b3a41cf477512e1e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 18:44:53 +0800 Subject: [PATCH 23/35] sync index --- source/libs/sync/inc/syncInt.h | 15 ++ source/libs/sync/src/syncElection.c | 13 ++ source/libs/sync/src/syncMain.c | 219 ++++++++++++++-------------- source/libs/sync/src/syncTimeout.c | 10 +- 4 files changed, 135 insertions(+), 122 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8866603b8e..4a1557addd 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -209,6 +209,7 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); +int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); @@ -218,6 +219,20 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); +// raft state change -------------- +void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); +void syncNodeBecomeFollower(SSyncNode* pSyncNode); +void syncNodeBecomeLeader(SSyncNode* pSyncNode); + +void syncNodeCandidate2Leader(SSyncNode* pSyncNode); +void syncNodeFollower2Candidate(SSyncNode* pSyncNode); +void syncNodeLeader2Follower(SSyncNode* pSyncNode); +void syncNodeCandidate2Follower(SSyncNode* pSyncNode); + +// raft vote -------------- +void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); +void syncNodeVoteForSelf(SSyncNode* pSyncNode); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 223431336e..d06253c049 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -16,6 +16,7 @@ #include "syncElection.h" #include "syncMessage.h" #include "syncRaftStore.h" +#include "syncVoteMgr.h" // TLA+ Spec // RequestVote(i, j) == @@ -49,10 +50,22 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + syncNodeFollower2Candidate(pSyncNode); + } assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); // start election + raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode->pRaftStore); + voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->pRaftStore->currentTerm); + votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); + + syncNodeVoteForSelf(pSyncNode); int32_t ret = syncNodeRequestVotePeers(pSyncNode); + assert(ret == 0); + syncNodeResetElectTimer(pSyncNode); + return ret; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index da1286bd32..5ec9df1e70 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -42,20 +42,6 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); - -// raft state change ---- -static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); -static void syncNodeBecomeFollower(SSyncNode* pSyncNode); -static void syncNodeBecomeLeader(SSyncNode* pSyncNode); - -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode); -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); -static void syncNodeLeader2Follower(SSyncNode* pSyncNode); -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); - -// raft vote ---- -static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); -static void syncNodeVoteForSelf(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { @@ -325,6 +311,13 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { return ret; } +int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + int32_t electMS = syncUtilElectRandomMS(); + ret = syncNodeRestartElectTimer(pSyncNode, electMS); + return ret; +} + int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, @@ -495,6 +488,105 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +// raft state change -------------- +void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { + if (term > pSyncNode->pRaftStore->currentTerm) { + raftStoreSetTerm(pSyncNode->pRaftStore, term); + syncNodeBecomeFollower(pSyncNode); + raftStoreClearVote(pSyncNode->pRaftStore); + } +} + +void syncNodeBecomeFollower(SSyncNode* pSyncNode) { + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + pSyncNode->leaderCache = EMPTY_RAFT_ID; + } + + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + syncNodeStopHeartbeatTimer(pSyncNode); + + int32_t electMS = syncUtilElectRandomMS(); + syncNodeRestartElectTimer(pSyncNode, electMS); +} + +// TLA+ Spec +// \* Candidate i transitions to leader. +// BecomeLeader(i) == +// /\ state[i] = Candidate +// /\ votesGranted[i] \in Quorum +// /\ state' = [state EXCEPT ![i] = Leader] +// /\ nextIndex' = [nextIndex EXCEPT ![i] = +// [j \in Server |-> Len(log[i]) + 1]] +// /\ matchIndex' = [matchIndex EXCEPT ![i] = +// [j \in Server |-> 0]] +// /\ elections' = elections \cup +// {[eterm |-> currentTerm[i], +// eleader |-> i, +// elog |-> log[i], +// evotes |-> votesGranted[i], +// evoterLog |-> voterLog[i]]} +// /\ UNCHANGED <> +// +void syncNodeBecomeLeader(SSyncNode* pSyncNode) { + pSyncNode->state = TAOS_SYNC_STATE_LEADER; + pSyncNode->leaderCache = pSyncNode->myRaftId; + + for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { + pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + } + + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; + } + + syncNodeStopElectTimer(pSyncNode); + syncNodeStartHeartbeatTimer(pSyncNode); + syncNodeReplicate(pSyncNode); +} + +void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + assert(voteGrantedMajority(pSyncNode->pVotesGranted)); + syncNodeBecomeLeader(pSyncNode); +} + +void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); + pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; +} + +void syncNodeLeader2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncNodeBecomeFollower(pSyncNode); +} + +void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + syncNodeBecomeFollower(pSyncNode); +} + +// raft vote -------------- +void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { + assert(term == pSyncNode->pRaftStore->currentTerm); + assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); + + raftStoreVote(pSyncNode->pRaftStore, pRaftId); +} + +void syncNodeVoteForSelf(SSyncNode* pSyncNode) { + syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); + + SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->myRaftId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->voteGranted = true; + + voteGrantedVote(pSyncNode->pVotesGranted, pMsg); + votesRespondAdd(pSyncNode->pVotesRespond, pMsg); + syncRequestVoteReplyDestroy(pMsg); +} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -602,102 +694,3 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); return ret; } - -// raft state change ---- -static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); - syncNodeBecomeFollower(pSyncNode); - raftStoreClearVote(pSyncNode->pRaftStore); - } -} - -static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - pSyncNode->leaderCache = EMPTY_RAFT_ID; - } - - pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - syncNodeStopHeartbeatTimer(pSyncNode); - - int32_t electMS = syncUtilElectRandomMS(); - syncNodeRestartElectTimer(pSyncNode, electMS); -} - -// TLA+ Spec -// \* Candidate i transitions to leader. -// BecomeLeader(i) == -// /\ state[i] = Candidate -// /\ votesGranted[i] \in Quorum -// /\ state' = [state EXCEPT ![i] = Leader] -// /\ nextIndex' = [nextIndex EXCEPT ![i] = -// [j \in Server |-> Len(log[i]) + 1]] -// /\ matchIndex' = [matchIndex EXCEPT ![i] = -// [j \in Server |-> 0]] -// /\ elections' = elections \cup -// {[eterm |-> currentTerm[i], -// eleader |-> i, -// elog |-> log[i], -// evotes |-> votesGranted[i], -// evoterLog |-> voterLog[i]]} -// /\ UNCHANGED <> -// -static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { - pSyncNode->state = TAOS_SYNC_STATE_LEADER; - pSyncNode->leaderCache = pSyncNode->myRaftId; - - for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { - pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; - } - - for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { - pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; - } - - syncNodeStopElectTimer(pSyncNode); - syncNodeStartHeartbeatTimer(pSyncNode); - syncNodeReplicate(pSyncNode); -} - -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - assert(voteGrantedMajority(pSyncNode->pVotesGranted)); - syncNodeBecomeLeader(pSyncNode); -} - -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); - pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; -} - -static void syncNodeLeader2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); - syncNodeBecomeFollower(pSyncNode); -} - -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - syncNodeBecomeFollower(pSyncNode); -} - -// raft vote ---- -static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - assert(term == pSyncNode->pRaftStore->currentTerm); - assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); - - raftStoreVote(pSyncNode->pRaftStore, pRaftId); -} - -static void syncNodeVoteForSelf(SSyncNode* pSyncNode) { - syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); - - SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); - pMsg->srcId = pSyncNode->myRaftId; - pMsg->destId = pSyncNode->myRaftId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; - pMsg->voteGranted = true; - - voteGrantedVote(pSyncNode->pVotesGranted, pMsg); - votesRespondAdd(pSyncNode->pVotesRespond, pMsg); - syncRequestVoteReplyDestroy(pMsg); -} \ No newline at end of file diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 7cbfd6d40a..3a48b0cbb3 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -19,15 +19,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { int32_t ret = 0; - sTrace("<-- syncNodeOnTimeoutCb -->"); - - { - cJSON* pJson = syncTimeout2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnTimeoutCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } + syncTimeoutLog2("==syncNodeOnTimeoutCb==", pMsg); if (pMsg->timeoutType == SYNC_TIMEOUT_PING) { if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) { From 5c7b09f7068f7b3dc3be50f681b2c86e3616ce82 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 14 Mar 2022 19:14:37 +0800 Subject: [PATCH 24/35] [add normal table case] --- tests/script/tsim/table/basic1.sim | 71 ++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index 9e94c3a311..09e6ede77d 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -61,10 +61,50 @@ if $rows != 7 then endi print $data00 $data01 $data02 -print $data10 $data11 $data22 -print $data20 $data11 $data22 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +print =============== create normal table +sql create database ndb +sql use ndb +sql create table nt0 (ts timestamp, i int) +sql create table if not exists nt0 (ts timestamp, i int) +sql create table nt1 (ts timestamp, i int) +sql create table if not exists nt1 (ts timestamp, i int) +sql create table if not exists nt3 (ts timestamp, i int) + +sql show tables +if $rows != 3 then + return -1 +endi + +sql insert into nt0 values(now+1s, 1)(now+2s, 2)(now+3s, 3) +sql insert into nt1 values(now+1s, 1)(now+2s, 2)(now+3s, 3) + +sql select * from nt1 +if $rows != 3 then + return -1 +endi + +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + +if $data01 != 1 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + print =============== insert data +sql use d1 sql insert into c1 values(now+1s, 1) sql insert into c1 values(now+2s, 2) sql insert into c1 values(now+3s, 3) @@ -95,7 +135,7 @@ endi print $data00 $data01 print $data10 $data11 -print $data20 $data11 +print $data20 $data21 if $data01 != 1 then return -1 @@ -160,7 +200,7 @@ endi print $data00 $data01 print $data10 $data11 -print $data20 $data11 +print $data20 $data21 if $data01 != 1 then return -1 @@ -210,4 +250,27 @@ if $rows != 21 then return -1 endi +print =============== query data from normal table after restart dnode +sql use ndb +sql select * from nt1 +if $rows != 3 then + return -1 +endi + +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + +if $data01 != 1 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 36d4ef0af6cbe03e7a5418135fb4851e90382658 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 14 Mar 2022 20:32:19 +0800 Subject: [PATCH 25/35] refactor stream worker --- include/common/tmsg.h | 11 +++ include/dnode/snode/snode.h | 4 ++ source/dnode/mgmt/impl/inc/dndEnv.h | 6 +- source/dnode/mgmt/impl/inc/dndInt.h | 2 +- source/dnode/mgmt/impl/src/dndSnode.c | 92 +++++++++++++++++++++++--- source/dnode/mgmt/impl/src/dndWorker.c | 2 +- source/dnode/snode/src/snode.c | 12 ++++ source/dnode/vnode/src/tq/tqRead.c | 4 +- source/util/src/tworker.c | 2 +- 9 files changed, 118 insertions(+), 17 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5c2c89b24..5a60761f11 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1118,6 +1118,17 @@ int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateS int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq); void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq); +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + int64_t streamId; + char* sql; + char* executorMsg; +} SMVCreateStreamReq, SMSCreateStreamReq; + +typedef struct { + int64_t streamId; +} SMVCreateStreamRsp, SMSCreateStreamRsp; + typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; int8_t igExists; diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index c9fab140cc..21a93532e0 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -80,6 +80,10 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); */ int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); + +int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); + /** * @brief Drop a snode. * diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 13ef101908..aeea5386b4 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -90,9 +90,11 @@ typedef struct { int32_t refCount; int8_t deployed; int8_t dropped; + int8_t uniqueWorkerInUse; SSnode *pSnode; SRWLatch latch; - SDnodeWorker writeWorker; + SArray *uniqueWorkers; // SArray + SDnodeWorker sharedWorker; } SSnodeMgmt; typedef struct { @@ -153,4 +155,4 @@ int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); } #endif -#endif /*_TD_DND_ENV_H_*/ \ No newline at end of file +#endif /*_TD_DND_ENV_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 4ca6b97ad4..a8530037da 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -70,4 +70,4 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); } #endif -#endif /*_TD_DND_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_INT_H_*/ diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 4906aef246..d192d9df01 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -19,7 +19,20 @@ #include "dndTransport.h" #include "dndWorker.h" -static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); +typedef struct { + int32_t vgId; + int32_t refCount; + int32_t snVersion; + int8_t dropped; + char *path; + SSnode *pImpl; + STaosQueue *pSharedQ; + STaosQueue *pUniqueQ; +} SSnodeObj; + +static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg); + +static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); static SSnode *dndAcquireSnode(SDnode *pDnode) { SSnodeMgmt *pMgmt = &pDnode->smgmt; @@ -152,8 +165,18 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { static int32_t dndStartSnodeWorker(SDnode *pDnode) { SSnodeMgmt *pMgmt = &pDnode->smgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, dndProcessSnodeQueue) != 0) { - dError("failed to start snode write worker since %s", terrstr()); + pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); + for (int32_t i = 0; i < 2; i++) { + SDnodeWorker uniqueWorker; + if (dndInitWorker(pDnode, &uniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { + dError("failed to start snode unique worker since %s", terrstr()); + return -1; + } + taosArrayPush(pMgmt->uniqueWorkers, &uniqueWorker); + } + if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", 4, 4, + dndProcessSnodeSharedQueue)) { + dError("failed to start snode shared worker since %s", terrstr()); return -1; } @@ -169,9 +192,13 @@ static void dndStopSnodeWorker(SDnode *pDnode) { while (pMgmt->refCount > 0) { taosMsleep(10); - } + } - dndCleanupWorker(&pMgmt->writeWorker); + for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { + SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); + dndCleanupWorker(worker); + } + taosArrayDestroy(pMgmt->uniqueWorkers); } static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { @@ -292,17 +319,36 @@ int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } } -static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { +static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { SSnodeMgmt *pMgmt = &pDnode->smgmt; - SRpcMsg *pRsp = NULL; int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; SSnode *pSnode = dndAcquireSnode(pDnode); if (pSnode != NULL) { - code = sndProcessMsg(pSnode, pMsg, &pRsp); + for (int32_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + + sndProcessUMsg(pSnode, pMsg); + + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + } + dndReleaseSnode(pDnode, pSnode); +} + +static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { + SSnodeMgmt *pMgmt = &pDnode->smgmt; + int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode != NULL) { + code = sndProcessSMsg(pSnode, pMsg); } dndReleaseSnode(pDnode, pSnode); +#if 0 if (pMsg->msgType & 1u) { if (pRsp != NULL) { pRsp->ahandle = pMsg->ahandle; @@ -314,11 +360,32 @@ static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { rpcSendResponse(&rpcRsp); } } +#endif rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } +static void dndWriteSnodeMsgToRandomWorker(SDnode *pDnode, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode != NULL) { + int32_t index = (pDnode->smgmt.uniqueWorkerInUse + 1) % taosArrayGetSize(pDnode->smgmt.uniqueWorkers); + SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, index); + code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); + } + dndReleaseSnode(pDnode, pSnode); + + if (code != 0) { + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pMsg->pCont); + } +} + static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; @@ -337,8 +404,13 @@ static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpc } } -void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.writeWorker, pMsg); +void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + // judge from msg to write to unique queue + dndWriteSnodeMsgToRandomWorker(pDnode, pMsg); +} + +void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); } int32_t dndInitSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index 5ccf6640c0..38f8737b2b 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -109,4 +109,4 @@ int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 01500fbc54..91008dd03a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -31,3 +31,15 @@ int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } void sndDestroy(const char *path) {} + +int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { + // stream deployment + // stream stop/resume + // operator exec + return 0; +} + +int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { + // operator exec + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 92a111298f..a2342ec85a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -83,8 +83,8 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { } int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - /*int32_t sversion = pHandle->pBlock->sversion;*/ - /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ + // currently only rows are used + pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); pBlockInfo->rows = pHandle->pBlock->numOfRows; pBlockInfo->uid = pHandle->pBlock->uid; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1657a85ee8..1fa70da870 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -188,7 +188,7 @@ void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueu int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; - pool->workers = calloc(sizeof(SWWorker), pool->max); + pool->workers = calloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; From f924c1b879f8c38013e31de19e124e96c1d82b4d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 20:43:35 +0800 Subject: [PATCH 26/35] sync index --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncMessage.h | 3 +- source/libs/sync/src/syncAppendEntries.c | 7 ++- source/libs/sync/src/syncAppendEntriesReply.c | 45 ++++++++++++++++++- source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncMain.c | 2 + source/libs/sync/src/syncMessage.c | 8 ++-- source/libs/sync/src/syncReplication.c | 29 +++++++++--- source/libs/sync/src/syncRequestVote.c | 36 ++++++++++++++- source/libs/sync/src/syncRequestVoteReply.c | 35 ++++++++++++++- source/libs/sync/test/syncRequestVoteTest.cpp | 2 +- source/libs/sync/test/syncRpcMsgTest.cpp | 2 +- 12 files changed, 155 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 4a1557addd..15c719b76e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -232,6 +232,7 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode); // raft vote -------------- void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); +void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 9bb5b6195e..7dfea31f5c 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -196,7 +196,7 @@ typedef struct SyncRequestVote { SRaftId srcId; SRaftId destId; // private data - SyncTerm currentTerm; + SyncTerm term; SyncIndex lastLogIndex; SyncTerm lastLogTerm; } SyncRequestVote; @@ -287,6 +287,7 @@ typedef struct SyncAppendEntriesReply { SRaftId srcId; SRaftId destId; // private data + SyncTerm term; bool success; SyncIndex matchIndex; } SyncAppendEntriesReply; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index ba10234a1d..55d369a115 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -80,4 +80,9 @@ // /\ UNCHANGED <> // /\ UNCHANGED <> // -int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {} +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + return ret; +} diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 0a5120c8dc..61eb4884e2 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,6 +14,12 @@ */ #include "syncAppendEntriesReply.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == @@ -28,4 +34,41 @@ // /\ Discard(m) // /\ UNCHANGED <> // -int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {} +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + return ret; + } + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex = reply.matchIndex + 1 + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex = reply.matchIndex + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncNodeMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + return ret; +} diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index d06253c049..77c3d07698 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -38,7 +38,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { SyncRequestVote* pMsg = syncRequestVoteBuild(); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->peersId[i]; - pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->pRaftStore->currentTerm; pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5ec9df1e70..aaf6535f40 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -587,6 +587,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncRequestVoteReplyDestroy(pMsg); } +void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 1e40df4419..2447ea2218 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -820,8 +820,8 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm); - cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); @@ -1264,9 +1264,11 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); - cJSON_AddStringToObject(pRoot, "match_index", u64buf); + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index d4c57630ad..b935943a1d 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -17,6 +17,8 @@ #include "syncIndexMgr.h" #include "syncMessage.h" #include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncUtil.h" // TLA+ Spec // AppendEntries(i, j) == @@ -50,16 +52,31 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; - SyncTerm preLogTerm = 0; - if (preLogIndex >= 0) { + + SyncTerm preLogTerm = 0; + if (preLogIndex >= SYNC_INDEX_BEGIN) { SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); preLogTerm = pPreEntry->term; - } else { - preLogTerm = 0; } - // SyncTerm lastIndex = - // pSyncNode->pLogStore->getLastIndex < nextIndex ? pSyncNode->pLogStore->getLastIndex : nextIndex; + + SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); + assert(nextIndex == lastIndex); + + SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); + assert(pEntry != NULL); + + SyncAppendEntries* pMsg = syncAppendEntriesBuild(pEntry->bytes); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + pMsg->dataLen = pEntry->bytes; + // add pEntry into msg + + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); } return ret; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 354c559a90..be4f40aaad 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,6 +14,10 @@ */ #include "syncRequestVote.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == @@ -37,4 +41,34 @@ // m) // /\ UNCHANGED <> // -int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {} +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + } + + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 72223ea83c..7cdeace166 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,6 +14,10 @@ */ #include "syncRequestVoteReply.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == @@ -32,4 +36,33 @@ // /\ Discard(m) // /\ UNCHANGED <> // -int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {} +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + return ret; + } + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + voteGrantedVote(ths->pVotesGranted, pMsg); + if (voteGrantedMajority(ths->pVotesGranted)) { + if (ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + ths->pVotesGranted->toLeader = true; + } + } + } + } + return ret; +} diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 7f75ee937b..22f47046de 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -20,7 +20,7 @@ SyncRequestVote *createMsg() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->currentTerm = 11; + pMsg->term = 11; pMsg->lastLogIndex = 22; pMsg->lastLogTerm = 33; return pMsg; diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 61edbd3012..ee6f6c0800 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -57,7 +57,7 @@ SyncRequestVote *createSyncRequestVote() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->currentTerm = 11; + pMsg->term = 11; pMsg->lastLogIndex = 22; pMsg->lastLogTerm = 33; return pMsg; From 6044c7511800f470ac27eb74d019cccec113570e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 14 Mar 2022 20:51:13 +0800 Subject: [PATCH 27/35] fix worker init --- source/dnode/mgmt/impl/src/dndSnode.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index d192d9df01..b27a25680a 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -167,12 +167,15 @@ static int32_t dndStartSnodeWorker(SDnode *pDnode) { SSnodeMgmt *pMgmt = &pDnode->smgmt; pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); for (int32_t i = 0; i < 2; i++) { - SDnodeWorker uniqueWorker; - if (dndInitWorker(pDnode, &uniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { + SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); + if (pUniqueWorker == NULL) { + return -1; + } + if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { dError("failed to start snode unique worker since %s", terrstr()); return -1; } - taosArrayPush(pMgmt->uniqueWorkers, &uniqueWorker); + taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker); } if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", 4, 4, dndProcessSnodeSharedQueue)) { From af3a8be54e69598c4f71a694e96a1955822c6f8c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 14 Mar 2022 22:21:30 +0800 Subject: [PATCH 28/35] add test UT --- source/libs/transport/src/trans.c | 35 +++--- source/libs/transport/src/transCli.c | 72 ++++++------ source/libs/transport/src/transComm.c | 98 +--------------- source/libs/transport/src/transSrv.c | 47 ++++---- source/libs/transport/test/transUT.cc | 161 ++++++++++++++++++-------- 5 files changed, 185 insertions(+), 228 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 015018f73f..58809ee3be 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -53,7 +53,6 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->secret) { memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret)); } - return pRpc; } void rpcClose(void* arg) { @@ -113,34 +112,19 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -int32_t rpcInit() { - // impl later - return 0; -} - -void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg* pMsg, int64_t *pRid) { +void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg); + transSendRequest(shandle, ip, port, pMsg); } -void rpcSendRecv(void* shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; transSendRecv(shandle, ip, port, pMsg, pRsp); } -void rpcSendResponse(const SRpcMsg *pMsg) { - transSendResponse(pMsg); -} -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { - return transGetConnInfo((void *)thandle, pInfo); -} - -void rpcCleanup(void) { - // impl later - // - return; -} +void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); } void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; @@ -155,4 +139,13 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } +int32_t rpcInit() { + // impl later + return 0; +} +void rpcCleanup(void) { + // impl later + return; +} + #endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4af02a982e..2f6ff3763f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -84,7 +84,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void cliTimeoutCb(uv_timer_t* handle); // alloc buf for recv -static void cliAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after read nbytes from socket static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); // callback after write data to socket @@ -154,50 +154,50 @@ void cliHandleResp(SCliConn* conn) { pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); - // buf's mem alread translated to rpcMsg.pCont - transClearBuffer(&conn->readBuf); - - STransMsg rpcMsg = {0}; - rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = transContFromHead((char*)pHead); - rpcMsg.code = pHead->code; - rpcMsg.msgType = pHead->msgType; - rpcMsg.ahandle = NULL; + STransMsg transMsg = {0}; + transMsg.contLen = transContLenFromMsg(pHead->msgLen); + transMsg.pCont = transContFromHead((char*)pHead); + transMsg.code = pHead->code; + transMsg.msgType = pHead->msgType; + transMsg.ahandle = NULL; SCliMsg* pMsg = conn->data; STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; + transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; } else { - rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } // if (rpcMsg.ahandle == NULL) { // tDebug("%s cli conn %p handle except", CONN_GET_INST_LABEL(conn), conn); // return; //} - if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, rpcMsg.msgType)) { - rpcMsg.handle = conn; + // buf's mem alread translated to transMsg.pCont + transClearBuffer(&conn->readBuf); + + if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) { + transMsg.handle = conn; CONN_SET_PERSIST_BY_APP(conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), rpcMsg.contLen); + inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); conn->secured = pHead->secured; if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); - (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL); + (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn); - memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg)); + memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); tsem_post(pCtx->pSem); } - uv_read_start((uv_stream_t*)conn->stream, cliAllocBufferCb, cliRecvCb); + uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); if (CONN_NO_PERSIST_BY_APP(conn)) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); @@ -224,23 +224,23 @@ void cliHandleExcept(SCliConn* pConn) { SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; - STransMsg rpcMsg = {0}; - rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; - rpcMsg.ahandle = NULL; + STransMsg transMsg = {0}; + transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; + transMsg.ahandle = NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { - rpcMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, rpcMsg.msgType) : NULL; + transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; } else { - rpcMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL); + (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn); - memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg)); + memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); tsem_post(pCtx->pSem); } destroyCmsg(pConn->data); @@ -252,9 +252,9 @@ void cliHandleExcept(SCliConn* pConn) { void cliTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; - STrans* pRpc = pThrd->pTransInst; + STrans* pTransInst = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; - tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pRpc->label); + tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pTransInst->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { @@ -271,8 +271,8 @@ void cliTimeoutCb(uv_timer_t* handle) { p = taosHashIterate((SHashObj*)pThrd->pool, p); } - pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); - uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); + uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0); } void* createConnPool(int size) { @@ -324,15 +324,15 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); - STrans* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; - conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); } -static void cliAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); @@ -416,7 +416,7 @@ static void cliSendCb(uv_write_t* req, int status) { cliHandleExcept(pConn); return; } - uv_read_start((uv_stream_t*)pConn->stream, cliAllocBufferCb, cliRecvCb); + uv_read_start((uv_stream_t*)pConn->stream, cliAllocRecvBufferCb, cliRecvCb); } void cliSend(SCliConn* pConn) { @@ -581,14 +581,14 @@ static void* cliWorkThread(void* arg) { void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SCliObj* cli = calloc(1, sizeof(SCliObj)); - STrans* pRpc = shandle; + STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { SCliThrdObj* pThrd = createThrdObj(); - pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; int err = pthread_create(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c83f76c2ec..367cb33fc9 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -16,20 +16,6 @@ #include "transComm.h" -int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - int ret = -1; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; - - return ret; -} int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; int ret = -1; @@ -44,17 +30,7 @@ int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { return ret; } -void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - memcpy(pAuth, context.digest, sizeof(context.digest)); -} void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; @@ -67,45 +43,6 @@ void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { memcpy(pAuth, context.digest, sizeof(context.digest)); } -int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { - SRpcHead* pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - - if (!NEEDTO_COMPRESSS_MSG(contLen)) { - return contLen; - } - - char* buf = malloc(contLen + overhead + 8); // 8 extra bytes - if (buf == NULL) { - tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); - return contLen; - } - - int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); - tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - - /* - * only the compressed size is less than the value of contLen - overhead, the compression is applied - * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message - */ - if (compLen > 0 && compLen < contLen - overhead) { - SRpcComp* pComp = (SRpcComp*)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); - memcpy(pCont + overhead, buf, compLen); - - pHead->comp = 1; - tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); - finalLen = compLen + overhead; - } else { - finalLen = contLen; - } - - free(buf); - return finalLen; -} - bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return false; // SRpcHead* pHead = rpcHeadFromCont(pCont); @@ -154,39 +91,6 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { return false; } -SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead* pNewHead = NULL; - uint8_t* pCont = pHead->content; - SRpcComp* pComp = (SRpcComp*)pHead->content; - - if (pHead->comp) { - // decompress the content - assert(pComp->reserved == 0); - int contLen = htonl(pComp->contLen); - - // prepare the temporary buffer to decompress message - char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - - if (pNewHead) { - int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen); - assert(origLen == contLen); - - memcpy(pNewHead, pHead, sizeof(SRpcHead)); - pNewHead->msgLen = rpcMsgLenFromCont(origLen); - /// rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; - tTrace("decomp malloc mem:%p", temp); - } else { - tError("failed to allocate memory to decompress msg, contLen:%d", contLen); - } - } - - return pHead; -} - void transConnCtxDestroy(STransConnCtx* ctx) { free(ctx->ip); free(ctx); @@ -315,7 +219,7 @@ int transSendAsync(SAsyncPool* pool, queue* q) { if (el > 50) { // tInfo("lock and unlock cost: %d", (int)el); } - return uv_async_send(async); } + #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c236a69f4e..cb3bbaefec 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -58,12 +58,13 @@ typedef struct SWorkThrdObj { uv_os_fd_t fd; uv_loop_t* loop; SAsyncPool* asyncPool; - // uv_async_t* workerAsync; // + queue msg; - queue conn; pthread_mutex_t msgMtx; - void* pTransInst; - bool quit; + + queue conn; + void* pTransInst; + bool quit; } SWorkThrdObj; typedef struct SServerObj { @@ -90,7 +91,7 @@ static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void uvOnTimeoutCb(uv_timer_t* handle); static void uvOnSendCb(uv_write_t* req, int status); @@ -120,7 +121,7 @@ static void* acceptThread(void* arg); static bool addHandleToWorkloop(void* arg); static bool addHandleToAcceptloop(void* arg); -void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSrvConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); @@ -162,7 +163,7 @@ static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); code = TSDB_CODE_RPC_INVALID_TIME_STAMP; } else { - if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + if (transAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { // tDebug("%s, authentication failed, msg discarded", pConn->info); code = TSDB_CODE_RPC_AUTH_FAILURE; } else { @@ -203,10 +204,6 @@ static void uvHandleReq(SSrvConn* pConn) { memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } - - pConn->inType = pHead->msgType; - - STrans* pRpc = (STrans*)p->shandle; pHead->code = htonl(pHead->code); int32_t dlen = 0; @@ -219,21 +216,24 @@ static void uvHandleReq(SSrvConn* pConn) { // } - STransMsg rpcMsg; - rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = pHead->content; - rpcMsg.msgType = pHead->msgType; - rpcMsg.code = pHead->code; - rpcMsg.ahandle = NULL; - rpcMsg.handle = pConn; + STransMsg transMsg; + transMsg.contLen = transContLenFromMsg(pHead->msgLen); + transMsg.pCont = pHead->content; + transMsg.msgType = pHead->msgType; + transMsg.code = pHead->code; + transMsg.ahandle = NULL; + transMsg.handle = pConn; transClearBuffer(&pConn->readBuf); - + pConn->inType = pHead->msgType; transRefSrvHandle(pConn); - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(rpcMsg.msgType), + + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port), rpcMsg.contLen); - (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); + ntohs(pConn->locaddr.sin_port), transMsg.contLen); + + STrans* pTransInst = (STrans*)p->shandle; + (*((STrans*)p->shandle)->cfp)(pTransInst->parent, &transMsg, NULL); // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth // validate msg type @@ -525,7 +525,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { return; } - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnRecvCb); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); } else { tDebug("failed to create new connection"); @@ -546,7 +546,6 @@ static bool addHandleToWorkloop(void* arg) { return false; } - // STrans* pRpc = pThrd->shandle; uv_pipe_init(pThrd->loop, pThrd->pipe, 1); uv_pipe_open(pThrd->pipe, pThrd->fd); diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 5edddb006b..fa20327003 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -29,24 +29,25 @@ const char *ckey = "ckey"; class Server; int port = 7000; // server process +typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); class Client { public: void Init(int nThread) { - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = (char *)label; - rpcInit.numOfThreads = nThread; - rpcInit.cfp = processResp; - rpcInit.user = (char *)user; - rpcInit.secret = (char *)secret; - rpcInit.ckey = (char *)ckey; - rpcInit.spi = 1; - rpcInit.parent = this; - rpcInit.connType = TAOS_CONN_CLIENT; - this->transCli = rpcOpen(&rpcInit); + memset(&rpcInit_, 0, sizeof(rpcInit_)); + rpcInit_.localPort = 0; + rpcInit_.label = (char *)label; + rpcInit_.numOfThreads = nThread; + rpcInit_.cfp = processResp; + rpcInit_.user = (char *)user; + rpcInit_.secret = (char *)secret; + rpcInit_.ckey = (char *)ckey; + rpcInit_.spi = 1; + rpcInit_.parent = this; + rpcInit_.connType = TAOS_CONN_CLIENT; + this->transCli = rpcOpen(&rpcInit_); tsem_init(&this->sem, 0, 0); } void SetResp(SRpcMsg *pMsg) { @@ -55,9 +56,27 @@ class Client { } SRpcMsg *Resp() { return &this->resp; } - void Restart() { + void Restart(CB cb) { rpcClose(this->transCli); - this->transCli = rpcOpen(&rpcInit); + rpcInit_.cfp = cb; + this->transCli = rpcOpen(&rpcInit_); + } + void setPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) { + rpcClose(this->transCli); + rpcInit_.pfp = pfp; + this->transCli = rpcOpen(&rpcInit_); + } + void setConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) { + rpcClose(this->transCli); + rpcInit_.mfp = mfp; + this->transCli = rpcOpen(&rpcInit_); + } + void setPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { + rpcClose(this->transCli); + + rpcInit_.pfp = pfp; + rpcInit_.mfp = mfp; + this->transCli = rpcOpen(&rpcInit_); } void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { @@ -79,7 +98,7 @@ class Client { private: tsem_t sem; - SRpcInit rpcInit; + SRpcInit rpcInit_; void * transCli; SRpcMsg resp; }; @@ -133,39 +152,56 @@ static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { client->SetResp(pMsg); client->SemPost(); } + +static void initEnv() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + std::string path = "/tmp/transport"; + taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } +} class TransObj { public: TransObj() { - dDebugFlag = 143; - vDebugFlag = 0; - mDebugFlag = 143; - cDebugFlag = 0; - jniDebugFlag = 0; - tmrDebugFlag = 143; - uDebugFlag = 143; - rpcDebugFlag = 143; - qDebugFlag = 0; - wDebugFlag = 0; - sDebugFlag = 0; - tsdbDebugFlag = 0; - tsLogEmbedded = 1; - tsAsyncLog = 0; - - std::string path = "/tmp/transport"; - taosRemoveDir(path.c_str()); - taosMkDir(path.c_str()); - - tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1) != 0) { - printf("failed to init log file\n"); - } + initEnv(); cli = new Client; cli->Init(1); srv = new Server; srv->Start(); } - void RestartCli() { cli->Restart(); } + + void RestartCli(CB cb) { cli->Restart(cb); } void StopSrv() { srv->Stop(); } + void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) { + // do nothing + cli->setPersistFP(pfp); + } + void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) { + // do nothing + cli->setConstructFP(mfp); + } + void SetMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { + // do nothing + cli->setPAndMFp(pfp, mfp); + } void RestartSrv() { srv->Restart(); } void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } ~TransObj() { @@ -191,16 +227,16 @@ class TransEnv : public ::testing::Test { TransObj *tr = NULL; }; -// TEST_F(TransEnv, 01sendAndRec) { -// for (int i = 0; i < 1; i++) { -// SRpcMsg req = {0}, resp = {0}; -// req.msgType = 0; -// req.pCont = rpcMallocCont(10); -// req.contLen = 10; -// tr->cliSendAndRecv(&req, &resp); -// assert(resp.code == 0); -// } -//} +TEST_F(TransEnv, 01sendAndRec) { + for (int i = 0; i < 1; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } +} TEST_F(TransEnv, 02StopServer) { for (int i = 0; i < 1; i++) { @@ -218,6 +254,31 @@ TEST_F(TransEnv, 02StopServer) { tr->StopSrv(); // tr->RestartSrv(); tr->cliSendAndRecv(&req, &resp); - assert(resp.code != 0); } +TEST_F(TransEnv, clientUserDefined) {} + +TEST_F(TransEnv, cliPersistHandle) { + // impl late +} +TEST_F(TransEnv, srvPersistHandle) { + // impl later +} + +TEST_F(TransEnv, srvPersisHandleExcept) { + // conn breken + // +} +TEST_F(TransEnv, cliPersisHandleExcept) { + // conn breken +} + +TEST_F(TransEnv, multiCliPersisHandleExcept) { + // conn breken +} +TEST_F(TransEnv, multiSrvPersisHandleExcept) { + // conn breken +} +TEST_F(TransEnv, queryExcept) { + // query and conn is broken +} From 17e607103366f515f0410e55dccedb788ea7ea2d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 15 Mar 2022 10:47:11 +0800 Subject: [PATCH 29/35] sync index --- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncAppendEntries.c | 117 +++++++++++++++++++++++ source/libs/sync/src/syncMessage.c | 3 + 3 files changed, 121 insertions(+) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 7dfea31f5c..5785089a20 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -255,6 +255,7 @@ typedef struct SyncAppendEntries { SRaftId srcId; SRaftId destId; // private data + SyncTerm term; SyncIndex prevLogIndex; SyncTerm prevLogTerm; SyncIndex commitIndex; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 55d369a115..87d6669f59 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,6 +14,11 @@ */ #include "syncAppendEntries.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -84,5 +89,117 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + assert(pMsg->dataLen >= 0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); + assert(pEntry != NULL); + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogIndex == localPreLogTerm)); + + // reject + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + syncNodeBecomeFollower(ths); + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + bool matchSuccess = false; + if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && + ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { + matchSuccess = true; + } + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); + assert(pEntry != NULL); + if (pMsg->prevLogTerm == pEntry->term) { + matchSuccess = true; + } + syncEntryDestory(pEntry); + } + + if (matchSuccess) { + // delete conflict entries + if (ths->pLogStore->getLastIndex(ths->pLogStore) > pMsg->prevLogIndex) { + SyncIndex fromIndex = pMsg->prevLogIndex + 1; + ths->pLogStore->truncate(ths->pLogStore, fromIndex); + } + + // append one entry + if (pMsg->dataLen > 0) { + SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncEntryDestory(pEntry); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + pReply->matchIndex = pMsg->prevLogIndex + 1; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + + syncAppendEntriesReplyDestroy(pReply); + } else { + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + } + + if (pMsg->commitIndex > ths->commitIndex) { + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + // commit + ths->commitIndex = pMsg->commitIndex; + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + } + } + } + return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 2447ea2218..509ede274b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1108,6 +1108,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); From 09e2f36ddd4c6370c272721f4fb4ec2ea706ca90 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 15 Mar 2022 14:04:16 +0800 Subject: [PATCH 30/35] sync append entries --- source/libs/sync/inc/syncElection.h | 1 - source/libs/sync/inc/syncInt.h | 4 ++++ source/libs/sync/inc/syncReplication.h | 1 - source/libs/sync/src/syncMain.c | 18 ++++++++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 019c291efc..85a82dcfb7 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -39,7 +39,6 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode); - int32_t syncNodeElect(SSyncNode* pSyncNode); int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 15c719b76e..5a9af83827 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -70,6 +70,9 @@ extern "C" { struct SyncTimeout; typedef struct SyncTimeout SyncTimeout; +struct SyncClientRequest; +typedef struct SyncClientRequest SyncClientRequest; + struct SyncPing; typedef struct SyncPing SyncPing; @@ -185,6 +188,7 @@ typedef struct SSyncNode { // callback int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); + int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg); int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index aca6205b9d..6fe18dae38 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -52,7 +52,6 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); - int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index aaf6535f40..dd2c142104 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -42,6 +42,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); // --------------------------------- int32_t syncInit() { @@ -192,6 +193,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init callback pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; + pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; @@ -696,3 +698,19 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); return ret; } + +static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { + int32_t ret = 0; + syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncNodeReplicate(ths); + syncEntryDestory(pEntry); + } else { + // ths->pFsm->FpCommitCb(-1) + } + + return ret; +} \ No newline at end of file From a69330b68d6fd1026cc8da9b0a8199433164d8b4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 15 Mar 2022 14:07:45 +0800 Subject: [PATCH 31/35] sync append entries (sync-io) --- source/libs/sync/inc/syncIO.h | 7 ++++--- source/libs/sync/src/syncIO.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 352d30c8d7..09e93fda1c 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -34,11 +34,11 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; - STaosQset * pQset; + STaosQset *pQset; pthread_t consumerTid; - void * serverRpc; - void * clientRpc; + void *serverRpc; + void *clientRpc; SEpSet myAddr; tmr_h qTimer; @@ -50,6 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c307ec5068..8176ac417a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -273,6 +273,13 @@ static void *syncIOConsumerFunc(void *param) { syncPingReplyDestroy(pSyncMsg); } + } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) { + if (io->FpOnSyncClientRequest != NULL) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + } + } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); From 2bcc139443e6e9ae9d9caffb1d0e051e97feff32 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 15 Mar 2022 14:56:33 +0800 Subject: [PATCH 32/35] add stream msg routing --- include/common/tmsg.h | 5 ++- include/dnode/snode/snode.h | 3 ++ source/dnode/mgmt/impl/src/dndSnode.c | 45 ++++++++++++++++++++++----- source/dnode/snode/inc/sndInt.h | 19 +++++------ 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5a60761f11..221cf28f23 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -187,7 +187,10 @@ typedef struct SEp { typedef struct { int32_t contLen; - int32_t vgId; + union { + int32_t vgId; + int32_t streamTaskId; + }; } SMsgHead; // Submit message for one table diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 21a93532e0..9dcd58a05f 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -23,6 +23,9 @@ extern "C" { #endif +#define SND_UNIQUE_THREAD_NUM 2 +#define SND_SHARED_THREAD_NUM 2 + /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SSnode SSnode; diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index b27a25680a..5ea8a841d2 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -166,7 +166,7 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { static int32_t dndStartSnodeWorker(SDnode *pDnode) { SSnodeMgmt *pMgmt = &pDnode->smgmt; pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); - for (int32_t i = 0; i < 2; i++) { + for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); if (pUniqueWorker == NULL) { return -1; @@ -177,8 +177,8 @@ static int32_t dndStartSnodeWorker(SDnode *pDnode) { } taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker); } - if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", 4, 4, - dndProcessSnodeSharedQueue)) { + if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, + SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) { dError("failed to start snode shared worker since %s", terrstr()); return -1; } @@ -369,13 +369,39 @@ static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void dndWriteSnodeMsgToRandomWorker(SDnode *pDnode, SRpcMsg *pMsg) { +static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + pHead->streamTaskId = htonl(pHead->streamTaskId); + return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; +} + +static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) { int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; SSnode *pSnode = dndAcquireSnode(pDnode); if (pSnode != NULL) { - int32_t index = (pDnode->smgmt.uniqueWorkerInUse + 1) % taosArrayGetSize(pDnode->smgmt.uniqueWorkers); - SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, index); + int32_t index = dndGetSWIdFromMsg(pMsg); + SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index); + code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); + } + + dndReleaseSnode(pDnode, pSnode); + + if (code != 0) { + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + rpcFreeCont(pMsg->pCont); + } +} + +static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + SSnode *pSnode = dndAcquireSnode(pDnode); + if (pSnode != NULL) { + SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0); code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); } dndReleaseSnode(pDnode, pSnode); @@ -407,9 +433,12 @@ static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpc } } +void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg); +} + void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - // judge from msg to write to unique queue - dndWriteSnodeMsgToRandomWorker(pDnode, pMsg); + dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); } void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 5851e18478..5c792c840d 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -30,21 +30,26 @@ extern "C" { #endif enum { - STREAM_STATUS__READY = 1, + STREAM_STATUS__RUNNING = 1, STREAM_STATUS__STOPPED, STREAM_STATUS__CREATING, STREAM_STATUS__STOPING, - STREAM_STATUS__RESUMING, + STREAM_STATUS__RESTORING, STREAM_STATUS__DELETING, }; enum { - STREAM_RUNNER__RUNNING = 1, - STREAM_RUNNER__STOP, + STREAM_TASK_STATUS__RUNNING = 1, + STREAM_TASK_STATUS__STOP, }; +typedef struct { + SHashObj* pHash; // taskId -> streamTask +} SStreamMeta; + typedef struct SSnode { - SSnodeOpt cfg; + SStreamMeta* pMeta; + SSnodeOpt cfg; } SSnode; typedef struct { @@ -62,10 +67,6 @@ typedef struct { // storage handle } SStreamRunner; -typedef struct { - SHashObj* pHash; -} SStreamMeta; - int32_t sndCreateStream(); int32_t sndDropStream(); From b71cea74bc859208ff84c2d205fde4d0539f7a4e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 15 Mar 2022 15:14:01 +0800 Subject: [PATCH 33/35] [add insert and select case] --- tests/script/tsim/insert/basic0.sim | 334 ++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 tests/script/tsim/insert/basic0.sim diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim new file mode 100644 index 0000000000..eb4780caac --- /dev/null +++ b/tests/script/tsim/insert/basic0.sim @@ -0,0 +1,334 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 +sql show databases +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d0 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) + +sql show tables +if $rows != 2 then + return -1 +endi + + +print =============== insert data, mode1: one row one table in sql +print =============== insert data, mode1: mulit rows one table in sql +print =============== insert data, mode1: one rows mulit table in sql +print =============== insert data, mode1: mulit rows mulit table in sql +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) +sql insert into ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) +# after fix bug, modify sql_error to sql +sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) +sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) + +#=================================================================== +#=================================================================== +print =============== query data from child table +sql select * from ct1 +if $rows != 4 then # after fix bug, modify 4 to 7 + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + + +print =============== select count(*) from child table +sql select count(*) from ct1 +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 +if $data00 != 4 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $data00 != 4 then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data02 != 4 then + return -1 +endi +if $data03 != 4 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 13 then + return -1 +endi +if $data01 != 2.30000 then + return -1 +endi +if $data02 != 3.300000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 46 then + return -1 +endi +if $data01 != 8.599999905 then + return -1 +endi +if $data02 != 12.600000000 then + return -1 +endi + +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print $data00 $data01 $data02 +#if $rows != 4 then +# return -1 +#endi +#if $data00 != 10 then +# return -1 +#endi +#if $data01 != 2.00000 then +# return -1 +#endi +#if $data02 != 3.000000000 then +# return -1 +#endi +#if $data10 != 11 then +# return -1 +#endi +#if $data11 != 2.10000 then +# return -1 +#endi +#if $data12 != 3.100000000 then +# return -1 +#endi +#if $data30 != 13 then +# return -1 +#endi +#if $data31 != 2.30000 then +# return -1 +#endi +#if $data32 != 3.300000000 then +# return -1 +#endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#if $rows != 1 then +# return -1 +#endi +# +#print $data00 $data01 $data02 +#if $data00 != 8 then +# return -1 +#endi +# +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 8 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + + +#=================================================================== +#=================================================================== + +print =============== stop and restart taosd, then again do query above +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql select * from ct1 +if $rows != 4 then # after fix bug, modify 4 to 7 + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + + +print =============== select count(*) from child table +sql select count(*) from ct1 +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 +if $data00 != 4 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $data00 != 4 then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data02 != 4 then + return -1 +endi +if $data03 != 4 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 13 then + return -1 +endi +if $data01 != 2.30000 then + return -1 +endi +if $data02 != 3.300000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 46 then + return -1 +endi +if $data01 != 8.599999905 then + return -1 +endi +if $data02 != 12.600000000 then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From b710e0a7d5a48cbea76d6d6e0b06f8a5dc0eb4fd Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 15 Mar 2022 15:18:46 +0800 Subject: [PATCH 34/35] [add cases] --- tests/script/jenkins/basic.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b934272806..cafca76761 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -16,4 +16,6 @@ # ---- dnode ./test.sh -f tsim/dnode/basic1.sim +# ---- insert +./test.sh -f tsim/insert/basic0.sim #======================b1-end=============== From eced27c4f2297e8781c95c23a0e466c746a0e9e7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 15 Mar 2022 15:54:32 +0800 Subject: [PATCH 35/35] add uuid --- include/util/tuuid.h | 39 ++ source/dnode/snode/inc/sndInt.h | 14 +- source/dnode/snode/src/snode.c | 11 + source/libs/scheduler/src/scheduler.c | 681 +++++++++++++------------- source/util/src/tuuid.c | 59 +++ 5 files changed, 451 insertions(+), 353 deletions(-) create mode 100644 include/util/tuuid.h create mode 100644 source/util/src/tuuid.c diff --git a/include/util/tuuid.h b/include/util/tuuid.h new file mode 100644 index 0000000000..315c2ad497 --- /dev/null +++ b/include/util/tuuid.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include "taoserror.h" +#include "thash.h" + +/** + * Generate an non-negative signed 32bit id + *+------------+-----+-----------+---------------+ + *| uid|localIp| PId | timestamp | serial number | + *+------------+-----+-----------+---------------+ + *| 6bit |6bit | 12bit | 8bit | + *+------------+-----+-----------+---------------+ + * @return + */ +int32_t tGenIdPI32(void); + +/** + * Generate an non-negative signed 64bit id + *+------------+-----+-----------+---------------+ + *| uid|localIp| PId | timestamp | serial number | + *+------------+-----+-----------+---------------+ + *| 12bit |12bit|24bit |16bit | + *+------------+-----+-----------+---------------+ + * @return + */ +int64_t tGenIdPI64(void); diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 5c792c840d..ffe691aeb4 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -54,18 +54,18 @@ typedef struct SSnode { typedef struct { int64_t streamId; + int32_t taskId; int32_t IdxInLevel; int32_t level; -} SStreamInfo; +} SStreamTaskInfo; typedef struct { - SStreamInfo meta; - int8_t status; - void* executor; - STaosQueue* queue; - void* stateStore; + SStreamTaskInfo meta; + int8_t status; + void* executor; + void* stateStore; // storage handle -} SStreamRunner; +} SStreamTask; int32_t sndCreateStream(); int32_t sndDropStream(); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 91008dd03a..74e41d45c5 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -14,6 +14,7 @@ */ #include "sndInt.h" +#include "tuuid.h" SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *pSnode = calloc(1, sizeof(SSnode)); @@ -32,6 +33,16 @@ int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { void sndDestroy(const char *path) {} +static int32_t sndDeployTask(SSnode *pSnode, SRpcMsg *pMsg) { + SStreamTask *task = malloc(sizeof(SStreamTask)); + if (task == NULL) { + return -1; + } + task->meta.taskId = tGenIdPI32(); + taosHashPut(pSnode->pMeta->pHash, &task->meta.taskId, sizeof(int32_t), &task, sizeof(void *)); + return 0; +} + int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // stream deployment // stream stop/resume diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ebe70ca401..6b1ca25d93 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -13,26 +13,21 @@ * along with this program. If not, see . */ +#include "catalog.h" +#include "query.h" #include "schedulerInt.h" #include "tmsg.h" -#include "query.h" -#include "catalog.h" #include "tref.h" SSchedulerMgmt schMgmt = {0}; -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); -} +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { - return taosReleaseRef(schMgmt.jobRef, refId); -} +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -uint64_t schGenTaskId(void) { - return atomic_add_fetch_64(&schMgmt.taskId, 1); -} +uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } +#if 0 uint64_t schGenUUID(void) { static uint64_t hashId = 0; static int32_t requestSerialId = 0; @@ -54,11 +49,11 @@ uint64_t schGenUUID(void) { uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } +#endif - -int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { - pTask->plan = pPlan; - pTask->level = pLevel; +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { + pTask->plan = pPlan; + pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); @@ -70,7 +65,7 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel * return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask* pTask) { +void schFreeTask(SSchTask *pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -90,22 +85,20 @@ void schFreeTask(SSchTask* pTask) { } } - static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { int8_t status = SCH_GET_JOB_STATUS(pJob); if (pStatus) { *pStatus = status; } - return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED - || status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING - || status == JOB_TASK_STATUS_SUCCEED); + return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || + status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING || + status == JOB_TASK_STATUS_SUCCEED); } - int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask); - + switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: @@ -114,19 +107,22 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m case TDMT_VND_FETCH_RSP: case TDMT_VND_DROP_TASK: if (lastMsgType != (msgType - 1)) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%s", SCH_GET_TASK_STATUS(pTask), TMSG_INFO(msgType)); + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && + SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%s", SCH_GET_TASK_STATUS(pTask), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } break; default: SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%d", TMSG_INFO(msgType), SCH_GET_TASK_STATUS(pTask)); - + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -135,7 +131,6 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m return TSDB_CODE_SUCCESS; } - int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; @@ -147,37 +142,34 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_NOT_START) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_NOT_START: if (newStatus != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && + newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED && + newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_SUCCEED && + newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_SUCCEED: case JOB_TASK_STATUS_FAILED: @@ -185,13 +177,13 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); break; - + default: SCH_JOB_ELOG("invalid job status:%d", oriStatus); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); @@ -211,27 +203,26 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { _return: SCH_JOB_ELOG("invalid job status update, from %d to %d", oriStatus, newStatus); - + SCH_ERR_RET(code); } - int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t i = 0; i < pJob->levelNum; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSubplan *pPlan = pTask->plan; - int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; - int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; + int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; + int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; if (childNum > 0) { if (pJob->levelIdx == pLevel->level) { SCH_JOB_ELOG("invalid query plan, lowest level, childNum:%d", childNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); @@ -240,7 +231,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan *child = (SSubplan*)nodesListGetNode(pPlan->pChildren, n); + SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -258,7 +249,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("invalid task info, level:0, parentNum:%d", parentNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); @@ -272,7 +263,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan *parent = (SSubplan*)nodesListGetNode(pPlan->pParents, n); + SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -283,7 +274,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - } + } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); } @@ -298,11 +289,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } - int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -311,7 +302,6 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { if (NULL == taosArrayPush(pTask->execAddrs, addr)) { SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); @@ -321,23 +311,25 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad return TSDB_CODE_SUCCESS; } - int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; - + if (pDag->numOfSubplans <= 0) { SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); if (levelNum <= 0) { SCH_JOB_ELOG("invalid level num:%d", levelNum); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + SHashObj *planToTask = taosHashInit( + SCHEDULE_DEFAULT_MAX_TASK_NUM, + taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, + HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -354,10 +346,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pJob->subPlans = pDag->pSubplans; - SSchLevel level = {0}; + SSchLevel level = {0}; SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SSchLevel *pLevel = NULL; + int32_t taskNum = 0; + SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -369,8 +361,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel = taosArrayGet(pJob->levels, i); pLevel->level = i; - - plans = (SNodeListNode*)nodesListGetNode(pDag->pSubplans, i); + + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); if (NULL == plans) { SCH_JOB_ELOG("empty level plan, level:%d", i); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -383,15 +375,15 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pLevel->taskNum = taskNum; - + pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + for (int32_t n = 0; n < taskNum; ++n) { - SSubplan *plan = (SSubplan*)nodesListGetNode(plans->pNodeList, n); + SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); SCH_SET_JOB_TYPE(pJob, plan->subplanType); @@ -399,13 +391,13 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchTask *pTask = &task; SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); - + void *p = taosArrayPush(pLevel->subTasks, &task); if (NULL == p) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -452,10 +444,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { int32_t nodeNum = 0; if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -470,14 +462,14 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_QRY_INVALID_INPUT; } -/* - for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); - epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; - - ++epSet->numOfEps; - } -*/ + /* + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; + } + */ return TSDB_CODE_SUCCESS; } @@ -489,7 +481,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("task already in execTask list, code:%x", code); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -510,11 +502,11 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -522,13 +514,13 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks)); - + return TSDB_CODE_SUCCESS; } int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = false; - + if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); } @@ -537,11 +529,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -549,11 +541,10 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks)); - + return TSDB_CODE_SUCCESS; } - int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); @@ -563,11 +554,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -575,11 +566,10 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); - + return TSDB_CODE_SUCCESS; } - int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... @@ -587,20 +577,17 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; - //TODO CHECK epList/condidateList + // TODO CHECK epList/condidateList if (SCH_IS_DATA_SRC_TASK(pTask)) { - } else { int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); - + if ((pTask->candidateIdx + 1) >= candidateNum) { return TSDB_CODE_SUCCESS; } ++pTask->candidateIdx; } - - } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { @@ -623,9 +610,9 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { - int32_t code = 0; + int32_t code = 0; SSchHbTrans *hb = NULL; - + while (true) { hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { @@ -639,9 +626,11 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { SCH_ERR_RET(code); } - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, trans->trans.transHandle); - + qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 + ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", + trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, + trans->trans.transHandle); + return TSDB_CODE_SUCCESS; } @@ -649,11 +638,11 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - + if (hb->seqId >= trans->seqId) { - qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", - trans->seqId, hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); - + qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", trans->seqId, + hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_UNLOCK(SCH_WRITE, &hb->lock); return TSDB_CODE_SUCCESS; } @@ -663,16 +652,18 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, trans->trans.transHandle); - + qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 + ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", + trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, + trans->trans.transHandle); + return TSDB_CODE_SUCCESS; } int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { // if already FAILED, no more processing SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, status)); - + if (errCode) { atomic_store_32(&pJob->errCode, errCode); } @@ -684,11 +675,10 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod int32_t code = atomic_load_32(&pJob->errCode); SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); - + SCH_RET(code); } - // Note: no more task error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAILED, errCode)); @@ -699,18 +689,16 @@ int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROPPING, errCode)); } - - // Note: no more task error processing, handled in function internal int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; - + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); if (pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } - + if (atomic_load_8(&pJob->userFetch)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); } @@ -730,22 +718,22 @@ int32_t schProcessOnDataFetched(SSchJob *job) { // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { int8_t status = 0; - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("task failed not processed cause of job status, job status:%d", status); - + SCH_RET(atomic_load_32(&pJob->errCode)); } - bool needRetry = false; - bool moved = false; + bool needRetry = false; + bool moved = false; int32_t taskDone = 0; int32_t code = 0; SCH_TASK_DLOG("taskOnFailure, code:%s", tstrerror(errCode)); - + SCH_ERR_JRET(schTaskCheckSetRetry(pJob, pTask, errCode, &needRetry)); - + if (!needRetry) { SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); @@ -757,7 +745,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - + if (SCH_TASK_NEED_WAIT_ALL(pTask)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskFailed++; @@ -765,7 +753,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); atomic_store_32(&pJob->errCode, errCode); - + if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum); SCH_ERR_RET(errCode); @@ -773,7 +761,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } } else { SCH_ERR_JRET(schHandleTaskRetry(pJob, pTask)); - + return TSDB_CODE_SUCCESS; } @@ -784,7 +772,7 @@ _return: // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { - bool moved = false; + bool moved = false; int32_t code = 0; SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); @@ -796,17 +784,17 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask)); - + int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { int32_t taskDone = 0; - + if (SCH_TASK_NEED_WAIT_ALL(pTask)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); - + if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); return TSDB_CODE_SUCCESS; @@ -826,28 +814,31 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { pJob->fetchTask = pTask; SCH_ERR_JRET(schMoveTaskToExecList(pJob, pTask, &moved)); - + SCH_RET(schProcessOnJobPartialSuccess(pJob)); } -/* - if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { - strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); - job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; + /* + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { + strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; - ++job->dataSrcEps.numOfEps; - } -*/ + ++job->dataSrcEps.numOfEps; + } + */ for (int32_t i = 0; i < parentNum; ++i) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); - int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); + int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, .taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; + SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, + .taskId = pTask->taskId, + .schedId = schMgmt.sId, + .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - + if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { SCH_ERR_RET(schLaunchTaskImpl(pJob, par)); } @@ -860,11 +851,10 @@ _return: SCH_RET(schProcessOnJobFailure(pJob, code)); } - // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; - + if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); return TSDB_CODE_SUCCESS; @@ -881,7 +871,7 @@ int32_t schFetchFromRemote(SSchJob *pJob) { SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; - + _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); @@ -889,15 +879,15 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code)); } - // Note: no more task error processing, handled in function internal -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, + int32_t rspCode) { int32_t code = 0; - int8_t status = 0; - + int8_t status = 0; + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_ELOG("rsp not processed cause of job status, job status:%d", status); - + SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -905,13 +895,13 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: { - SCH_ERR_JRET(rspCode); - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + SCH_ERR_JRET(rspCode); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - break; - } + break; + } case TDMT_VND_SUBMIT_RSP: { - #if 0 //TODO OPEN THIS +#if 0 // TODO OPEN THIS SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { @@ -919,77 +909,77 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } pJob->resNumOfRows += rsp->affectedRows; - #else - SCH_ERR_JRET(rspCode); +#else + SCH_ERR_JRET(rspCode); - SSubmitRsp *rsp = (SSubmitRsp *)msg; - if (rsp) { - pJob->resNumOfRows += rsp->affectedRows; - } - #endif - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; + SSubmitRsp *rsp = (SSubmitRsp *)msg; + if (rsp) { + pJob->resNumOfRows += rsp->affectedRows; } +#endif + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; + } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - SCH_ERR_JRET(rsp->code); - - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); - - break; + SQueryTableRsp *rsp = (SQueryTableRsp *)msg; + + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + SCH_ERR_JRET(rsp->code); + + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + + break; + } case TDMT_VND_RES_READY_RSP: { - SResReadyRsp *rsp = (SResReadyRsp *)msg; - - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - SCH_ERR_JRET(rsp->code); - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + SCH_ERR_JRET(rsp->code); + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; + } case TDMT_VND_FETCH_RSP: { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - if (pJob->res) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); - tfree(rsp); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } - - atomic_store_ptr(&pJob->res, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); - - if (rsp->completed) { - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); - } - - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); - - schProcessOnDataFetched(pJob); - break; + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + + if (pJob->res) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); + tfree(rsp); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } + + atomic_store_ptr(&pJob->res, rsp); + atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + + if (rsp->completed) { + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + } + + SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + + schProcessOnDataFetched(pJob); + break; + } case TDMT_VND_DROP_TASK_RSP: { - // SHOULD NEVER REACH HERE - SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; - } + // SHOULD NEVER REACH HERE + SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + break; + } default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1002,15 +992,15 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } - -int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { - int32_t code = 0; +int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { + int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchTask *pTask = NULL; - + SSchTask *pTask = NULL; + SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, + pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1028,8 +1018,8 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in pTask = *task; SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); - - pTask->handle = pMsg->handle; + + pTask->handle = pMsg->handle; SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1042,42 +1032,41 @@ _return: SCH_RET(code); } -int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleSubmitCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code); } -int32_t schHandleCreateTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); } -int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } -int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleFetchCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code); } -int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); } -int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchCallbackParam *pParam = (SSchCallbackParam *)param; - qDebug("QID:%"PRIx64",TID:%"PRIx64" drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); + qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } - -int32_t schHandleHbCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); SCH_ERR_RET(code); } - + SSchedulerHbRsp rsp = {0}; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - + if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1088,21 +1077,22 @@ int32_t schHandleHbCallback(void* param, const SDataBuf* pMsg, int32_t code) { trans.seqId = rsp.seqId; trans.trans.transInst = pParam->transport; trans.trans.transHandle = pMsg->handle; - + SCH_RET(schUpdateHbConnection(&rsp.epId, &trans)); } int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); - + SSchJob *pJob = schAcquireJob(taskStatus->refId); if (NULL == pJob) { - qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, taskStatus->queryId, taskStatus->taskId); - //TODO DROP TASK FROM SERVER!!!! + qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, + taskStatus->queryId, taskStatus->taskId); + // TODO DROP TASK FROM SERVER!!!! continue; } - + // TODO schReleaseJob(taskStatus->refId); @@ -1115,22 +1105,21 @@ _return: SCH_RET(code); } - int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: *fp = schHandleCreateTableCallback; break; - case TDMT_VND_SUBMIT: + case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; - case TDMT_VND_QUERY: + case TDMT_VND_QUERY: *fp = schHandleQueryCallback; break; - case TDMT_VND_RES_READY: + case TDMT_VND_RES_READY: *fp = schHandleReadyCallback; break; - case TDMT_VND_FETCH: + case TDMT_VND_FETCH: *fp = schHandleFetchCallback; break; case TDMT_VND_DROP_TASK: @@ -1147,13 +1136,13 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } - -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, + uint32_t msgSize) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1173,15 +1162,14 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* param->taskId = SCH_TASK_ID(pTask); param->transport = trans->transInst; - pMsgSendInfo->param = param; pMsgSendInfo->msgInfo.pData = msg; pMsgSendInfo->msgInfo.len = msgSize; - pMsgSendInfo->msgInfo.handle = trans->transHandle; + pMsgSendInfo->msgInfo.handle = trans->transHandle; pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - - int64_t transporterId = 0; + + int64_t transporterId = 0; code = asyncSendMsgToServer(trans->transInst, epSet, &transporterId, pMsgSendInfo); if (code) { SCH_ERR_JRET(code); @@ -1191,7 +1179,7 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* return TSDB_CODE_SUCCESS; _return: - + tfree(param); tfree(pMsgSendInfo); SCH_RET(code); @@ -1199,9 +1187,9 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1235,13 +1223,13 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SSubQueryMsg *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->phyLen = htonl(pTask->msgLen); - pMsg->sqlLen = htonl(len); + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(pJob->queryId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->refId = htobe64(pJob->refId); + pMsg->taskType = TASK_TYPE_TEMP; + pMsg->phyLen = htonl(pTask->msgLen); + pMsg->sqlLen = htonl(len); memcpy(pMsg->msg, pJob->sql, len); memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); @@ -1257,12 +1245,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SResReadyReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } case TDMT_VND_FETCH: { @@ -1272,32 +1260,32 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SResFetchReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } - case TDMT_VND_DROP_TASK:{ + case TDMT_VND_DROP_TASK: { msgSize = sizeof(STaskDropReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + STaskDropReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->refId = htobe64(pJob->refId); break; } case TDMT_VND_QUERY_HEARTBEAT: { @@ -1337,24 +1325,24 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (isCandidateAddr) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); } - + return TSDB_CODE_SUCCESS; _return: SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); - + tfree(msg); SCH_RET(code); } int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); - + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { SCH_ERR_RET(schBuildAndSendMsg(pJob, NULL, addr, TDMT_VND_QUERY_HEARTBEAT)); @@ -1364,29 +1352,30 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { - int8_t status = 0; + int8_t status = 0; int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status, job status:%d", status); - + SCH_RET(atomic_load_32(&pJob->errCode)); } - + SSubplan *plan = pTask->plan; - if (NULL == pTask->msg) { // TODO add more detailed reason for failure + if (NULL == pTask->msg) { // TODO add more detailed reason for failure code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen); if (TSDB_CODE_SUCCESS != code) { - SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); + SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, + pTask->msgLen); SCH_ERR_RET(code); } else { SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } - + SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); // NOTE: race condition: the task should be put into the hash table before send msg to server @@ -1398,15 +1387,15 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_QUERY_JOB(pJob)) { SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } - + SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); - + return TSDB_CODE_SUCCESS; } // Note: no more error processing, handled in function internal int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { - bool enough = false; + bool enough = false; int32_t code = 0; if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1436,11 +1425,9 @@ int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { return TSDB_CODE_SUCCESS; } - - int32_t schLaunchJob(SSchJob *pJob) { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); - + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); SCH_ERR_RET(schCheckJobNeedFlowCtrl(pJob, level)); @@ -1457,7 +1444,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); - + if (size <= 0) { SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); return; @@ -1481,9 +1468,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { if (!SCH_TASK_NO_NEED_DROP(pTask)) { schDropTaskOnExecutedNode(pJob, pTask); } - + pIter = taosHashIterate(list, pIter); - } + } } void schDropJobAllTasks(SSchJob *pJob) { @@ -1493,10 +1480,9 @@ void schDropJobAllTasks(SSchJob *pJob) { } int32_t schCancelJob(SSchJob *pJob) { - //TODO - - //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + // TODO + // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } void schFreeJobImpl(void *job) { @@ -1506,7 +1492,7 @@ void schFreeJobImpl(void *job) { SSchJob *pJob = job; uint64_t queryId = pJob->queryId; - int64_t refId = pJob->refId; + int64_t refId = pJob->refId; if (pJob->status == JOB_TASK_STATUS_EXECUTING) { schCancelJob(pJob); @@ -1514,55 +1500,55 @@ void schFreeJobImpl(void *job) { schDropJobAllTasks(pJob); - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + int32_t numOfLevels = taosArrayGetSize(pJob->levels); - for(int32_t i = 0; i < numOfLevels; ++i) { + for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); schFreeFlowCtrl(pLevel); - + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); + for (int32_t j = 0; j < numOfTasks; ++j) { + SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); schFreeTask(pTask); } taosArrayDestroy(pLevel->subTasks); } - + taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->succTasks); - + taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); tfree(pJob->res); - + tfree(pJob); - qDebug("QID:0x%"PRIx64" job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); } - -static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pDag, int64_t *job, const char* sql, bool syncSchedule) { - qDebug("QID:0x%"PRIx64" job started", pDag->queryId); +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, + bool syncSchedule) { + qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { - qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId); + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pDag->queryId); } - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = calloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:%"PRIx64" calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; - pJob->sql = sql; + pJob->sql = sql; if (pNodeList != NULL) { pJob->nodeList = taosArrayDup(pNodeList); @@ -1570,19 +1556,22 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); - pJob->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->execTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->succTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->succTasks) { SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->failTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->failTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->failTasks) { SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1602,9 +1591,9 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_ERR_JRET(schLaunchJob(pJob)); schAcquireJob(pJob->refId); - + *job = pJob->refId; - + if (syncSchedule) { SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); tsem_wait(&pJob->rspSem); @@ -1613,7 +1602,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); schReleaseJob(pJob->refId); - + return TSDB_CODE_SUCCESS; _return: @@ -1622,7 +1611,6 @@ _return: SCH_RET(code); } - int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobRef) { qError("scheduler already initialized"); @@ -1631,7 +1619,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { if (cfg) { schMgmt.cfg = *cfg; - + if (schMgmt.cfg.maxJobNum == 0) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; } @@ -1642,7 +1630,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; } - + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); @@ -1660,12 +1648,13 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler %"PRIx64" initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); - + qInfo("scheduler %" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, + SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1676,20 +1665,21 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, in pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; schReleaseJob(*pJob); - + return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan* pDag, const char* sql, int64_t *pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pDag, const char *sql, int64_t *pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); - + return TSDB_CODE_SUCCESS; } +#if 0 int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { if (NULL == pDag || pDag->numOfSubplans <= 0 || LIST_LENGTH(pDag->pSubplans) == 0) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1810,14 +1800,14 @@ _return: SCH_RET(code); } +#endif - -int32_t schedulerFetchRows(int64_t job, void** pData) { +int32_t schedulerFetchRows(int64_t job, void **pData) { if (NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); @@ -1861,12 +1851,11 @@ int32_t schedulerFetchRows(int64_t job, void** pData) { SCH_JOB_ELOG("job failed or dropping, status:%d", status); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } - + if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } - while (true) { *pData = atomic_load_ptr(&pJob->res); if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { @@ -1891,7 +1880,7 @@ int32_t schedulerFetchRows(int64_t job, void** pData) { _return: atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - + schReleaseJob(job); SCH_RET(code); @@ -1944,17 +1933,17 @@ void schedulerFreeTaskList(SArray *taskList) { taosArrayDestroy(taskList); } - + void schedulerDestroy(void) { if (schMgmt.jobRef) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); - + while (pJob) { taosRemoveRef(schMgmt.jobRef, pJob->refId); - + pJob = taosIterateRef(schMgmt.jobRef, pJob->refId); } - + taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = 0; } diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c new file mode 100644 index 0000000000..0405403220 --- /dev/null +++ b/source/util/src/tuuid.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tuuid.h" + +static int64_t hashId = 0; +static int32_t SerialNo = 0; + +int32_t tGenIdPI32(void) { + if (hashId == 0) { + char uid[64]; + int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + if (code != TSDB_CODE_SUCCESS) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + hashId = MurmurHash3_32(uid, strlen(uid)); + } + } + + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&SerialNo, 1); + + int32_t id = ((hashId & 0x1F) << 26) | ((pid & 0x3F) << 20) | ((ts & 0xFFF) << 8) | (val & 0xFF); + return id; +} + +int64_t tGenIdPI64(void) { + if (hashId == 0) { + char uid[64]; + int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + if (code != TSDB_CODE_SUCCESS) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + hashId = MurmurHash3_32(uid, strlen(uid)); + } + } + + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&SerialNo, 1); + + int64_t id = ((hashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + return id; +}