From 264c30e5db9b348d06135bc9638d7e1cd9b66069 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 10:52:18 +0800 Subject: [PATCH 01/39] sync refactor --- source/libs/sync/inc/syncInt.h | 9 ++++--- source/libs/sync/inc/syncUtil.h | 1 + source/libs/sync/src/syncMain.c | 45 ++++++++++++++++++++++++++++++--- source/libs/sync/src/syncUtil.c | 4 ++- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index b8c7eb60e7..79412febd9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -99,8 +99,8 @@ typedef struct SRaftStore SRaftStore; struct SVotesGranted; typedef struct SVotesGranted SVotesGranted; -struct SVotesResponded; -typedef struct SVotesResponded SVotesResponded; +struct SVotesRespond; +typedef struct SVotesRespond SVotesRespond; typedef struct SRaftId { SyncNodeId addr; // typedef uint64_t SyncNodeId; @@ -112,6 +112,7 @@ typedef struct SSyncNode { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + char walPath[TSDB_FILENAME_LEN]; void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); void* queue; @@ -142,8 +143,8 @@ typedef struct SSyncNode { SRaftStore* pRaftStore; // tla+ candidate vars - SVotesGranted* pVotesGranted; - SVotesResponded* pVotesResponded; + SVotesGranted* pVotesGranted; + SVotesRespond* pVotesRespond; // tla+ leader vars SHashObj* pNextIndex; diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 9b481e82d9..eb1e888254 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -44,6 +44,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); // ---- misc ---- int32_t syncUtilRand(int32_t max); int32_t syncUtilElectRandomMS(); +int32_t syncUtilQuorum(int32_t replicaNum); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index db34d16690..836f1b0e83 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -20,10 +20,12 @@ #include "syncEnv.h" #include "syncInt.h" #include "syncRaft.h" +#include "syncRaftStore.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncTimeout.h" #include "syncUtil.h" +#include "syncVoteMgr.h" static int32_t tsNodeRefId = -1; @@ -35,6 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +static void UpdateTerm(SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); @@ -71,19 +74,22 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); + // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->syncCfg = pSyncInfo->syncCfg; memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); - pSyncNode->pFsm = pSyncInfo->pFsm; - + memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath)); pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; pSyncNode->queue = pSyncInfo->queue; pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg; + // init internal pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; - pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; + syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncInfo->vgId, &pSyncNode->raftId); + // init peersNum, peers, peersId + pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; int j = 0; for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { if (i != pSyncInfo->syncCfg.myIndex) { @@ -91,9 +97,37 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { j++; } } + for (int i = 0; i < pSyncNode->peersNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncInfo->vgId, &pSyncNode->peersId[i]); + } + // init replicaNum, replicasId + pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum; + for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]); + } + + // raft algorithm + pSyncNode->pFsm = pSyncInfo->pFsm; + pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum); + pSyncNode->leaderCache = EMPTY_RAFT_ID; + + // life cycle + + // init server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId); + pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath); + assert(pSyncNode->pRaftStore != NULL); + + // init candidate vars + pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pSyncNode->pVotesGranted != NULL); + pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); + assert(pSyncNode->pVotesRespond != NULL); + + // init leader vars + pSyncNode->pNextIndex = NULL; + pSyncNode->pMatchIndex = NULL; // init ping timer pSyncNode->pPingTimer = NULL; @@ -119,6 +153,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer; pSyncNode->heartbeatTimerCounter = 0; + // init callback pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; @@ -353,6 +388,8 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {} +static void UpdateTerm(SyncTerm term) {} + static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { pSyncNode->leaderCache = EMPTY_RAFT_ID; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index c70e490025..1e62301814 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -97,4 +97,6 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { int32_t syncUtilRand(int32_t max) { return rand() % max; } -int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } \ No newline at end of file +int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } + +int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } \ No newline at end of file From 4540739c45866641c3fd1a7a0f9f1db809ca9b72 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Mar 2022 11:34:38 +0800 Subject: [PATCH 02/39] minor changes --- source/libs/monitor/CMakeLists.txt | 1 - source/libs/monitor/inc/monInt.h | 1 - source/libs/transport/CMakeLists.txt | 1 + source/libs/transport/inc/thttp.h | 33 ++++++++++++++++++++++++++++ source/util/CMakeLists.txt | 2 +- 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 source/libs/transport/inc/thttp.h diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 050372fab3..58f1c08039 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -3,7 +3,6 @@ add_library(monitor STATIC ${MONITOR_SRC}) target_include_directories( monitor PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/monitor" - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index c3b6569555..6ef901410b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -19,7 +19,6 @@ #include "monitor.h" #include "tarray.h" -#include "tlockfree.h" #include "tjson.h" typedef struct { diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 465646ac95..5cc436cf32 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC zlib ) if (${BUILD_WITH_UV_TRANS}) if (${BUILD_WITH_UV}) diff --git a/source/libs/transport/inc/thttp.h b/source/libs/transport/inc/thttp.h new file mode 100644 index 0000000000..a9e3953f57 --- /dev/null +++ b/source/libs/transport/inc/thttp.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TRANSPORT_HTTP_H_ +#define _TD_TRANSPORT_HTTP_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag; + +int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_UTIL_H_*/ diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 4b28800c28..7a47639e75 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries( util PRIVATE os PUBLIC lz4_static - PUBLIC api cjson zlib + PUBLIC api cjson ) if(${BUILD_TEST}) From 18a22e6517d6d032ea8361583584a777ed8b587e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Mar 2022 13:41:03 +0800 Subject: [PATCH 03/39] minor changes --- source/libs/transport/inc/thttp.h | 33 ------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 source/libs/transport/inc/thttp.h diff --git a/source/libs/transport/inc/thttp.h b/source/libs/transport/inc/thttp.h deleted file mode 100644 index a9e3953f57..0000000000 --- a/source/libs/transport/inc/thttp.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TRANSPORT_HTTP_H_ -#define _TD_TRANSPORT_HTTP_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag; - -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_UTIL_H_*/ From c9c48e870d618573bd0d096727b2f4199187a43f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 13:43:54 +0800 Subject: [PATCH 04/39] sync refactor --- include/libs/sync/sync.h | 7 +-- source/libs/sync/inc/syncInt.h | 2 + source/libs/sync/inc/syncUtil.h | 9 ++-- source/libs/sync/src/syncMain.c | 65 ++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 44 ++++++++++++++- source/libs/sync/test/CMakeLists.txt | 14 +++++ source/libs/sync/test/syncInitTest.cpp | 74 ++++++++++++++++++++++++++ 7 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 source/libs/sync/test/syncInitTest.cpp diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index fddf18c571..ccbeb00bfd 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -31,9 +31,9 @@ typedef int64_t SyncIndex; typedef uint64_t SyncTerm; typedef enum { - TAOS_SYNC_STATE_FOLLOWER = 0, - TAOS_SYNC_STATE_CANDIDATE = 1, - TAOS_SYNC_STATE_LEADER = 2, + TAOS_SYNC_STATE_FOLLOWER = 100, + TAOS_SYNC_STATE_CANDIDATE = 101, + TAOS_SYNC_STATE_LEADER = 102, } ESyncState; typedef struct SSyncBuffer { @@ -134,6 +134,7 @@ typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; + char walPath[TSDB_FILENAME_LEN]; SSyncFSM* pFsm; void* rpcClient; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 79412febd9..fdacb87481 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "cJSON.h" #include "sync.h" #include "taosdef.h" #include "tglobal.h" @@ -189,6 +190,7 @@ typedef struct SSyncNode { SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); +cJSON* syncNode2Json(const SSyncNode* pSyncNode); int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index eb1e888254..6cb1105b2c 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -42,9 +42,12 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest); void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); // ---- misc ---- -int32_t syncUtilRand(int32_t max); -int32_t syncUtilElectRandomMS(); -int32_t syncUtilQuorum(int32_t replicaNum); +int32_t syncUtilRand(int32_t max); +int32_t syncUtilElectRandomMS(); +int32_t syncUtilQuorum(int32_t replicaNum); +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +cJSON* syncUtilRaftId2Json(const SRaftId* p); +const char* syncUtilState2String(ESyncState state); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 836f1b0e83..aebbd4d337 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -170,6 +170,71 @@ void syncNodeClose(SSyncNode* pSyncNode) { free(pSyncNode); } +cJSON* syncNode2Json(const SSyncNode* pSyncNode) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg); + cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg); + cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->me); + cJSON_AddItemToObject(pRoot, "me", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->raftId); + cJSON_AddItemToObject(pRoot, "raftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peers", pPeers); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peers[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } + + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // tla+ server vars + cJSON_AddStringToObject(pRoot, "state", syncUtilState2String(pSyncNode->state)); + + // tla+ candidate vars + + // tla+ leader vars + + // tla+ log vars + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); + return pJson; +} + int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1e62301814..04be3f33ac 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -99,4 +99,46 @@ int32_t syncUtilRand(int32_t max) { return rand() % max; } int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } -int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } \ No newline at end of file +int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +cJSON* syncUtilRaftId2Json(const SRaftId* p) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + snprintf(u64buf, sizeof(u64buf), "%lu", p->addr); + cJSON_AddStringToObject(pRoot, "addr", u64buf); + char host[128]; + uint16_t port; + syncUtilU642Addr(p->addr, host, sizeof(host), &port); + cJSON_AddStringToObject(pRoot, "host", host); + cJSON_AddNumberToObject(pRoot, "port", port); + cJSON_AddNumberToObject(pRoot, "vgId", p->vgId); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +const char* syncUtilState2String(ESyncState state) { + if (state == TAOS_SYNC_STATE_FOLLOWER) { + return "TAOS_SYNC_STATE_FOLLOWER"; + } else if (state == TAOS_SYNC_STATE_CANDIDATE) { + return "TAOS_SYNC_STATE_CANDIDATE"; + } else if (state == TAOS_SYNC_STATE_LEADER) { + return "TAOS_SYNC_STATE_LEADER"; + } else { + return "TAOS_SYNC_STATE_UNKNOWN"; + } +} \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 770d1d1bd8..3a8eea53d4 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -10,6 +10,7 @@ add_executable(syncIOSendMsgServerTest "") add_executable(syncRaftStoreTest "") add_executable(syncEnqTest "") add_executable(syncIndexTest "") +add_executable(syncInitTest "") target_sources(syncTest @@ -60,6 +61,10 @@ target_sources(syncIndexTest PRIVATE "syncIndexTest.cpp" ) +target_sources(syncInitTest + PRIVATE + "syncInitTest.cpp" +) target_include_directories(syncTest @@ -122,6 +127,11 @@ target_include_directories(syncIndexTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncInitTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -172,6 +182,10 @@ target_link_libraries(syncIndexTest sync gtest_main ) +target_link_libraries(syncInitTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp new file mode 100644 index 0000000000..602297b2a6 --- /dev/null +++ b/source/libs/sync/test/syncInitTest.cpp @@ -0,0 +1,74 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[3] = {7010, 7110, 7210}; + +SSyncNode* syncInitTest() { + SSyncFSM* pFsm; + + SSyncInfo syncInfo; + syncInfo.vgId = 1; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = 0; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = ports[0]; + snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = ports[1]; + snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = ports[2]; + snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[0]); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + printf("%s\n", serialized); + free(serialized); + cJSON_Delete(pJson); + + return 0; +} From f556d98159308abf0a15b4b522dae7a3b4fdd712 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 13:55:13 +0800 Subject: [PATCH 05/39] sync refactor --- source/libs/sync/inc/syncInt.h | 6 ++--- source/libs/sync/src/syncMain.c | 35 ++++++++++++++------------- source/libs/sync/test/syncEnqTest.cpp | 2 +- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index fdacb87481..2c8e36fd08 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -120,11 +120,11 @@ typedef struct SSyncNode { int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg); // init internal - SNodeInfo me; - SRaftId raftId; + SNodeInfo myNodeInfo; + SRaftId myRaftId; int32_t peersNum; - SNodeInfo peers[TSDB_MAX_REPLICA]; + SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; SRaftId peersId[TSDB_MAX_REPLICA]; int32_t replicaNum; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index aebbd4d337..44d2e4c160 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -85,20 +85,20 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg; // init internal - pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncInfo->vgId, &pSyncNode->raftId); + pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; + syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; int j = 0; for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { if (i != pSyncInfo->syncCfg.myIndex) { - pSyncNode->peers[j] = pSyncInfo->syncCfg.nodeInfo[i]; + pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i]; j++; } } for (int i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncInfo->vgId, &pSyncNode->peersId[i]); + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId @@ -190,16 +190,16 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf); // init internal - cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->me); - cJSON_AddItemToObject(pRoot, "me", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->raftId); - cJSON_AddItemToObject(pRoot, "raftId", pRaftId); + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); cJSON* pPeers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peers", pPeers); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); for (int i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peers[i])); + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); } cJSON* pPeersId = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "peersId", pPeersId); @@ -222,7 +222,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); // tla+ server vars - cJSON_AddStringToObject(pRoot, "state", syncUtilState2String(pSyncNode->state)); + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); // tla+ candidate vars @@ -283,7 +284,7 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { SRaftId destId; syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -294,8 +295,8 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId); + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); ret = syncNodePing(pSyncNode, &destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -304,7 +305,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t syncNodePingSelf(SSyncNode* pSyncNode) { int32_t ret; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &pSyncNode->raftId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); @@ -385,7 +386,7 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { cJSON_Delete(pJson); } - SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->raftId, &pMsg->srcId); + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsgReply, &rpcMsg); syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); @@ -485,7 +486,7 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { // static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_LEADER; - pSyncNode->leaderCache = pSyncNode->raftId; + pSyncNode->leaderCache = pSyncNode->myRaftId; // next Index +=1 // match Index = 0; diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 0bf43f933e..e2bc9a73ae 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -84,7 +84,7 @@ int main(int argc, char** argv) { gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->raftId, &pSyncNode->raftId); + SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); From ffe442301c532ca5d0139ac4e23d8097d2abf34e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 14:19:50 +0800 Subject: [PATCH 06/39] sync refactor --- source/libs/sync/inc/syncInt.h | 4 ++- source/libs/sync/inc/syncReplication.h | 1 + source/libs/sync/src/syncElection.c | 8 ++++-- source/libs/sync/src/syncMain.c | 35 +++++++++++++++++++++++--- source/libs/sync/src/syncReplication.c | 11 +++++++- source/libs/sync/src/syncTimeout.c | 2 +- source/libs/sync/src/syncUtil.c | 2 +- 7 files changed, 54 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 2c8e36fd08..874763cd45 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -155,7 +155,7 @@ typedef struct SSyncNode { SSyncLogStore* pLogStore; SyncIndex commitIndex; - // timer + // ping timer tmr_h pPingTimer; int32_t pingTimerMS; uint64_t pingTimerLogicClock; @@ -163,6 +163,7 @@ typedef struct SSyncNode { TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp uint64_t pingTimerCounter; + // elect timer tmr_h pElectTimer; int32_t electTimerMS; uint64_t electTimerLogicClock; @@ -170,6 +171,7 @@ typedef struct SSyncNode { TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp uint64_t electTimerCounter; + // heartbeat timer tmr_h pHeartbeatTimer; int32_t heartbeatTimerMS; uint64_t heartbeatTimerLogicClock; diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 467cfdde5c..aca6205b9d 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -53,6 +53,7 @@ extern "C" { // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); +int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); #ifdef __cplusplus diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 87017b718d..24f41a0e69 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -28,11 +28,15 @@ // mdest |-> j]) // /\ UNCHANGED <> // -int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {} +int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + return ret; +} int32_t syncNodeElect(SSyncNode* pSyncNode) { // start election - syncNodeRequestVotePeers(pSyncNode); + int32_t ret = syncNodeRequestVotePeers(pSyncNode); + return ret; } int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 44d2e4c160..f2939df60c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -37,7 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -static void UpdateTerm(SyncTerm term); +static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); @@ -452,9 +452,38 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { } } -static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {} +static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { + SSyncNode* pSyncNode = (SSyncNode*)param; + if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <= + atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { + SyncTimeout* pSyncMsg = + syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), + pSyncNode->heartbeatTimerMS, pSyncNode); -static void UpdateTerm(SyncTerm term) {} + SRpcMsg rpcMsg; + syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + syncTimeoutDestroy(pSyncMsg); + + // reset timer ms + // pSyncNode->heartbeatTimerMS += 100; + + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + } else { + sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", + pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + } +} + +static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { + if (term > pSyncNode->pRaftStore->currentTerm) { + pSyncNode->pRaftStore->currentTerm = term; + pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; + raftStorePersist(pSyncNode->pRaftStore); + syncNodeBecomeFollower(pSyncNode); + } +} static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 37e8959ff3..dfbe5db0ed 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -41,7 +41,16 @@ // mdest |-> j]) // /\ UNCHANGED <> // -int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {} +int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + return ret; +} + +int32_t syncNodeReplicate(SSyncNode* pSyncNode) { + // start replicate + int32_t ret = syncNodeAppendEntriesPeers(pSyncNode); + return ret; +} int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { sTrace("syncNodeAppendEntries pSyncNode:%p ", pSyncNode); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index df9b9d27b4..7cbfd6d40a 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -44,7 +44,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { } else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) { if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->heartbeatTimerCounter); - syncNodeAppendEntriesPeers(ths); + syncNodeReplicate(ths); } } else { sTrace("unknown timeoutType:%d", pMsg->timeoutType); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 04be3f33ac..fa6c245fd8 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -127,7 +127,7 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { cJSON_AddNumberToObject(pRoot, "vgId", p->vgId); cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + cJSON_AddItemToObject(pJson, "SRaftId", pRoot); return pJson; } From bf0e4b782824e5f398d1f53b42ffcc9d8d9be56e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 8 Mar 2022 14:31:02 +0800 Subject: [PATCH 07/39] Test/sangshuduo/td 13408 move tests in for3.0 (#10613) * restore .gitmodules * Revert "[TD-13408]: move tests out" This reverts commit f80a4ca49ff37431bc0bc0dafb5ccf5858b00beb. * revert f80a4ca49ff37431bc0bc0dafb5ccf5858b00beb * immigrate file change from stand-alone repo to TDengine for 3.0 * remove tests repository for Jenkinsfile2 * remove tests/test from .gitignore * add examples/rust back for master * fix typo Co-authored-by: tangfangzhi --- .gitmodules | 3 +++ examples/rust | 1 + 2 files changed, 4 insertions(+) create mode 160000 examples/rust diff --git a/.gitmodules b/.gitmodules index e9af594edc..07e4bb2b9c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "deps/TSZ"] path = deps/TSZ url = https://github.com/taosdata/TSZ.git +[submodule "examples/rust"] + path = examples/rust + url = https://github.com/songtianyi/tdengine-rust-bindings.git diff --git a/examples/rust b/examples/rust new file mode 160000 index 0000000000..1c8924dc66 --- /dev/null +++ b/examples/rust @@ -0,0 +1 @@ +Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df From c3119245fbe7c47ac0036d0eed21b6e7bf56e807 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 8 Mar 2022 14:37:37 +0800 Subject: [PATCH 08/39] [TD-13894]: test-all script for 3.0 (#10623) --- tests/test-all.sh | 129 ++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 61 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index c2bc305de0..aa7c4240bc 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -1,5 +1,7 @@ #!/bin/bash +# set -x + # Color setting RED='\033[0;31m' GREEN='\033[1;32m' @@ -12,19 +14,19 @@ tests_dir=`pwd` IN_TDINTERNAL="community" function stopTaosd { - echo "Stop taosd" + echo "Stop taosd" sudo systemctl stop taosd || echo 'no sudo or systemctl or stop fail' PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do + while [ -n "$PID" ] + do pkill -TERM -x taosd sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done } function dohavecore(){ - corefile=`find $corepath -mmin 1` + [ -d $corepath ] && corefile=`find $corepath -mmin 1` || return 1 if [ -n "$corefile" ];then core_file=`echo $corefile|cut -d " " -f2` proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` @@ -39,9 +41,9 @@ function dohavecore(){ cd community cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi - else + else cd ../../ - if [[ $1 == 1 ]];then + if [[ $1 == 1 ]];then #tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi @@ -56,20 +58,20 @@ function dohavecore(){ function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` - start_time=`date +%s` + case=`echo $line | grep sim$ |awk '{print $NF}'` + start_time=`date +%s` date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ./test.sh -f $case > /dev/null 2>&1 \ + && ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log ) \ + || ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log else echo -n $case ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log fi out_log=`tail -1 out.log ` @@ -85,35 +87,35 @@ function runSimCaseOneByOne { function runSimCaseOneByOnefq { start=`sed -n "/$1-start/=" jenkins/basic.txt` - end=`sed -n "/$1-end/=" jenkins/basic.txt` + end=`sed -n "/$1-end/=" jenkins/basic.txt` for ((i=$start;i<=$end;i++)) ; do line=`sed -n "$i"p jenkins/basic.txt` if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` + case=`echo $line | grep sim$ |awk '{print $NF}'` - start_time=`date +%s` + start_time=`date +%s` date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then echo -n $case - ./test.sh -f $case > case.log 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ./test.sh -f $case > case.log 2>&1 \ + && \ + ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else - pwd echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log ) fi - + out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` - else + else cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi dohavecore $2 1 @@ -125,7 +127,7 @@ function runSimCaseOneByOnefq { echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log dohavecore $2 1 fi - done + done rm -rf ../../../sim/case.log rm -rf ../../sim/case.log } @@ -134,7 +136,6 @@ function runPyCaseOneByOne { while read -r line; do if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - if [[ $line =~ '-r' ]];then case=`echo $line|awk '{print $4}'` else @@ -172,7 +173,6 @@ function runPyCaseOneByOnefq() { line=`sed -n "$i"p fulltest.sh` if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - if [[ $line =~ '-r' ]];then case=`echo $line|awk '{print $4}'` else @@ -186,7 +186,7 @@ function runPyCaseOneByOnefq() { fi $line > case.log 2>&1 && \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED} failed${NC}" | tee -a pytest-out.log + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then @@ -205,7 +205,7 @@ function runPyCaseOneByOnefq() { fi dohavecore $2 2 fi - done + done rm -rf ../../sim/case.log } @@ -239,7 +239,9 @@ totalExampleFailed=0 if [ "${OS}" == "Linux" ]; then corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` if [ -z "$corepath" ];then - echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied" + [ -d /coredump ] || mkdir /coredump || echo -e "failed to mdkir /coredump" + [ -d /coredump ] \ + && echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied" corepath="/coredump/" fi fi @@ -248,6 +250,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" ! echo "### run TSIM test case ###" cd $tests_dir/script + [ -d ../../sim ] || mkdir ../../sim || echo -e "failed to mkdir ../../sim" [ -f out.log ] && rm -f out.log if [ "$1" == "cron" ]; then echo "### run TSIM regression test ###" @@ -358,7 +361,7 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != " runPyCaseOneByOnefq p2 1 elif [ "$1" == "p3" ]; then echo "### run Python_3 test ###" - runPyCaseOneByOnefq p3 1 + runPyCaseOneByOnefq p3 1 elif [ "$1" == "p4" ]; then echo "### run Python_4 test ###" runPyCaseOneByOnefq p4 1 @@ -368,16 +371,20 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != " echo "### run Python smoke test ###" runPyCaseOneByOne smoketest.sh fi - totalPySuccess=`grep 'success' pytest-out.log | wc -l` - if [ "$totalPySuccess" -gt "0" ]; then - echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" - fi + if [ -f pytest-out.log ]; then + totalPySuccess=`grep 'success' pytest-out.log | wc -l` + if [ "$totalPySuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" + fi - totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` - if [ "$totalPyFailed" -ne "0" ]; then - echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" -# exit $totalPyFailed + totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` + if [ "$totalPyFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" + # exit $totalPyFailed + fi + else + echo -e "pytest-out.log not found!" fi fi @@ -395,14 +402,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != pwd cd debug/ - + stopTaosd rm -rf /var/lib/taos/* nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 & sleep 30 - - cd $tests_dir/../src/connector/jdbc - + + cd $tests_dir/../src/connector/jdbc + mvn test > jdbc-out.log 2>&1 tail -n 20 jdbc-out.log @@ -412,14 +419,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != JDBCFailed=`echo ${failed/%,}` error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'` JDBCError=`echo ${error/%,}` - + totalJDBCFailed=`expr $JDBCFailed + $JDBCError` totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed` if [ "$totalJDBCSuccess" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}" fi - + if [ "$totalJDBCFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}" fi @@ -427,7 +434,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != fi if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then - echo "### run Unit tests ###" + echo "### run Unit tests ###" stopTaosd cd $tests_dir @@ -443,19 +450,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != rm -rf /var/lib/taos/* nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & sleep 30 - + pwd ./queryTest > unittest-out.log 2>&1 tail -n 20 unittest-out.log - totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` + totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'` totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess` if [ "$totalUnitSuccess" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}" fi - + if [ "$totalUnitFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}" fi @@ -463,7 +470,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != fi if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then - echo "### run Example tests ###" + echo "### run Example tests ###" stopTaosd cd $tests_dir @@ -480,7 +487,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & echo "sleeping for 30 seconds" #sleep 30 - + cd $tests_dir echo "current dir: " pwd @@ -493,16 +500,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./apitest > /dev/null 2>&1 if [ $? != "0" ]; then echo "apitest failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "apitest pass" totalExamplePass=`expr $totalExamplePass + 1` - fi + fi ./prepare 127.0.0.1 > /dev/null 2>&1 if [ $? != "0" ]; then echo "prepare failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "prepare pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -511,7 +518,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./subscribe -test > /dev/null 2>&1 if [ $? != "0" ]; then echo "subscribe failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "subscribe pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -520,7 +527,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 if [ $? != "0" ]; then echo "asyncdemo failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "asyncdemo pass" totalExamplePass=`expr $totalExamplePass + 1` @@ -529,16 +536,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./demo 127.0.0.1 > /dev/null 2>&1 if [ $? != "0" ]; then echo "demo failed" - totalExampleFailed=`expr $totalExampleFailed + 1` + totalExampleFailed=`expr $totalExampleFailed + 1` else echo "demo pass" totalExamplePass=`expr $totalExamplePass + 1` fi - + if [ "$totalExamplePass" -gt "0" ]; then echo -e "\n${GREEN} ### Total $totalExamplePass examples succeed! ### ${NC}" fi - + if [ "$totalExampleFailed" -ne "0" ]; then echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}" fi From 54c260738fa01676431144acbfbc84e00b0c219a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 14:45:03 +0800 Subject: [PATCH 09/39] sync refactor --- source/libs/sync/src/syncMain.c | 55 ++++++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 2 +- source/libs/sync/test/CMakeLists.txt | 14 +++++++ source/libs/sync/test/syncUtilTest.cpp | 32 +++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 source/libs/sync/test/syncUtilTest.cpp diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f2939df60c..5338b5bd7f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -231,6 +231,61 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ log vars + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer); + cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer); + cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); return pJson; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index fa6c245fd8..d2b413bdaa 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -97,7 +97,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { int32_t syncUtilRand(int32_t max) { return rand() % max; } -int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } +int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 3a8eea53d4..f7f0a0795c 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable(syncRaftStoreTest "") add_executable(syncEnqTest "") add_executable(syncIndexTest "") add_executable(syncInitTest "") +add_executable(syncUtilTest "") target_sources(syncTest @@ -65,6 +66,10 @@ target_sources(syncInitTest PRIVATE "syncInitTest.cpp" ) +target_sources(syncUtilTest + PRIVATE + "syncUtilTest.cpp" +) target_include_directories(syncTest @@ -132,6 +137,11 @@ target_include_directories(syncInitTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncUtilTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -186,6 +196,10 @@ target_link_libraries(syncInitTest sync gtest_main ) +target_link_libraries(syncUtilTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp new file mode 100644 index 0000000000..9a1c113620 --- /dev/null +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -0,0 +1,32 @@ +#include "syncUtil.h" +//#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void electRandomMSTest() { + for (int i = 0; i < 10; ++i) { + int32_t ms = syncUtilElectRandomMS(); + printf("syncUtilElectRandomMS: %d \n", ms); + } +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + logTest(); + electRandomMSTest(); + + return 0; +} From 2ae6f747f918cecad4d6eeebbaaf35ad4c47ffea Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 17:07:29 +0800 Subject: [PATCH 10/39] sync refactor --- source/libs/sync/inc/syncVoteMgr.h | 10 ++- source/libs/sync/src/syncElection.c | 17 ++++ source/libs/sync/src/syncMain.c | 4 +- source/libs/sync/src/syncVoteMgr.c | 70 +++++++++++++++- source/libs/sync/test/CMakeLists.txt | 14 ++++ .../libs/sync/test/syncVotesGrantedTest.cpp | 80 +++++++++++++++++++ 6 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 source/libs/sync/test/syncVotesGrantedTest.cpp diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index e2307e9e66..a769bfbccd 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -28,10 +28,14 @@ extern "C" { #include "syncUtil.h" #include "taosdef.h" +// SVotesGranted ----------------------------- typedef struct SVotesGranted { + SRaftId (*replicas)[TSDB_MAX_REPLICA]; + int32_t replicaNum; + bool isGranted[TSDB_MAX_REPLICA]; + int32_t votes; SyncTerm term; int32_t quorum; - int32_t votes; bool toLeader; SSyncNode *pSyncNode; } SVotesGranted; @@ -41,7 +45,10 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted); bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); +cJSON * voteGranted2Json(SVotesGranted *pVotesGranted); +char * voteGranted2Str(SVotesGranted *pVotesGranted); +// SVotesRespond ----------------------------- typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; bool isRespond[TSDB_MAX_REPLICA]; @@ -51,6 +58,7 @@ typedef struct SVotesRespond { } SVotesRespond; SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode); +void votesRespondDestory(SVotesRespond *pVotesRespond); bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void Reset(SVotesRespond *pVotesRespond, SyncTerm term); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 24f41a0e69..223431336e 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -15,6 +15,7 @@ #include "syncElection.h" #include "syncMessage.h" +#include "syncRaftStore.h" // TLA+ Spec // RequestVote(i, j) == @@ -29,11 +30,27 @@ // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->peersId[i]; + pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); + pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); + + ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg); + assert(ret == 0); + syncRequestVoteDestroy(pMsg); + } return ret; } int32_t syncNodeElect(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + // start election int32_t ret = syncNodeRequestVotePeers(pSyncNode); return ret; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5338b5bd7f..ad424b9353 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -107,12 +107,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]); } - // raft algorithm + // init raft algorithm pSyncNode->pFsm = pSyncInfo->pFsm; pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum); pSyncNode->leaderCache = EMPTY_RAFT_ID; - // life cycle + // init life cycle // init server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index c9f0ceab57..407b7b1941 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -14,15 +14,25 @@ */ #include "syncVoteMgr.h" +#include "syncUtil.h" + +// SVotesGranted ----------------------------- +static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { + memset(pVotesGranted->isGranted, 0, sizeof(pVotesGranted->isGranted)); + pVotesGranted->votes = 0; +} SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted)); assert(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); - pVotesGranted->quorum = pSyncNode->quorum; + pVotesGranted->replicas = &(pSyncNode->replicasId); + pVotesGranted->replicaNum = pSyncNode->replicaNum; + voteGrantedClearVotes(pVotesGranted); + pVotesGranted->term = 0; - pVotesGranted->votes = 0; + pVotesGranted->quorum = pSyncNode->quorum; pVotesGranted->toLeader = false; pVotesGranted->pSyncNode = pSyncNode; @@ -43,15 +53,61 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) { void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { assert(pMsg->voteGranted == true); assert(pMsg->term == pVotesGranted->term); - pVotesGranted->votes++; + assert(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + + int j = -1; + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + if (syncUtilSameId(&((*(pVotesGranted->replicas))[i]), &(pMsg->srcId))) { + j = i; + break; + } + } + assert(j != -1); + + if (pVotesGranted->isGranted[j] != true) { + ++(pVotesGranted->votes); + pVotesGranted->isGranted[j] = true; + } + assert(pVotesGranted->votes <= pVotesGranted->replicaNum); } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->term = term; - pVotesGranted->votes = 0; + voteGrantedClearVotes(pVotesGranted); pVotesGranted->toLeader = false; } +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i]))); + } + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond)); assert(pVotesRespond != NULL); @@ -65,6 +121,12 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { return pVotesRespond; } +void votesRespondDestory(SVotesRespond *pVotesRespond) { + if (pVotesRespond != NULL) { + free(pVotesRespond); + } +} + bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { bool ret = false; for (int i = 0; i < pVotesRespond->replicaNum; ++i) { diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index f7f0a0795c..18b7748105 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(syncEnqTest "") add_executable(syncIndexTest "") add_executable(syncInitTest "") add_executable(syncUtilTest "") +add_executable(syncVotesGrantedTest "") target_sources(syncTest @@ -70,6 +71,10 @@ target_sources(syncUtilTest PRIVATE "syncUtilTest.cpp" ) +target_sources(syncVotesGrantedTest + PRIVATE + "syncVotesGrantedTest.cpp" +) target_include_directories(syncTest @@ -142,6 +147,11 @@ target_include_directories(syncUtilTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncVotesGrantedTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -200,6 +210,10 @@ target_link_libraries(syncUtilTest sync gtest_main ) +target_link_libraries(syncVotesGrantedTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp new file mode 100644 index 0000000000..97b807736b --- /dev/null +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -0,0 +1,80 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[3] = {7010, 7110, 7210}; + +SSyncNode* doSync(int myIndex) { + SSyncFSM* pFsm; + + SSyncInfo syncInfo; + syncInfo.vgId = 1; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = ports[0]; + snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = ports[1]; + snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = ports[2]; + snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + logTest(); + + int myIndex = 0; + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = doSync(myIndex); + SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pVotesGranted != NULL); + + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + + return 0; +} From 255d79544295df25703b18a8d03b436859cf7ec9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 8 Mar 2022 17:22:21 +0800 Subject: [PATCH 11/39] add tq push --- include/common/tcommon.h | 6 +- include/common/tmsg.h | 12 +- include/libs/qcom/query.h | 145 +++++++++++------- source/client/src/clientImpl.c | 4 +- source/client/src/tmq.c | 30 ++-- source/common/src/tmsg.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 6 +- source/dnode/mnode/impl/src/mndScheduler.c | 31 ++-- source/dnode/mnode/impl/src/mndSubscribe.c | 15 +- source/dnode/mnode/impl/src/mnode.c | 8 +- source/dnode/mnode/impl/test/db/db.cpp | 12 +- source/dnode/vnode/inc/tq.h | 14 +- source/dnode/vnode/src/inc/tqInt.h | 10 +- source/dnode/vnode/src/inc/tqPush.h | 77 ++++++++++ source/dnode/vnode/src/tq/tq.c | 63 ++++---- source/dnode/vnode/src/tq/tqPush.c | 84 ++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 12 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/test/catalogTests.cpp | 48 +++--- source/libs/executor/src/executorimpl.c | 4 +- source/libs/parser/src/dCDAstProcess.c | 4 +- .../libs/parser/test/mockCatalogService.cpp | 22 +-- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/planner/src/physicalPlanJson.c | 10 +- source/libs/scheduler/src/scheduler.c | 6 +- source/libs/scheduler/test/schedulerTests.cpp | 14 +- tests/test/c/tmqDemo.c | 6 +- 27 files changed, 438 insertions(+), 215 deletions(-) create mode 100644 source/dnode/vnode/src/inc/tqPush.h create mode 100644 source/dnode/vnode/src/tq/tqPush.c diff --git a/include/common/tcommon.h b/include/common/tcommon.h index d0ec5c9296..1d3ab4f340 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -135,7 +135,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) return (void*)buf; } -static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { +static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); @@ -156,7 +156,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp return tlen; } -static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { +static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); @@ -194,7 +194,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { // tfree(pBlock); } -static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { +static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { if (pRsp->schemas) { if (pRsp->schemas->nCols) { tfree(pRsp->schemas->pSchema); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d56db2046e..238753e5b7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -795,7 +795,7 @@ typedef struct SVgroupInfo { int32_t vgId; uint32_t hashBegin; uint32_t hashEnd; - SEpSet epset; + SEpSet epSet; } SVgroupInfo; typedef struct { @@ -1876,8 +1876,8 @@ typedef struct { } SVCreateTSmaReq; typedef struct { - int8_t type; // 0 status report, 1 update data - char indexName[TSDB_INDEX_NAME_LEN + 1]; // + int8_t type; // 0 status report, 1 update data + char indexName[TSDB_INDEX_NAME_LEN + 1]; // STimeWindow windows; } STSmaMsg; @@ -2073,7 +2073,7 @@ typedef struct { int32_t skipLogNum; int32_t numOfTopics; SArray* pBlockData; // SArray -} SMqConsumeRsp; +} SMqPollRsp; // one req for one vg+topic typedef struct { @@ -2086,7 +2086,7 @@ typedef struct { int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; -} SMqConsumeReq; +} SMqPollReq; typedef struct { int32_t vgId; @@ -2108,7 +2108,7 @@ typedef struct { struct tmq_message_t { SMqRspHead head; union { - SMqConsumeRsp consumeRsp; + SMqPollRsp consumeRsp; SMqCMGetSubEpRsp getEpRsp; }; void* extra; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 1f56254476..5d5ab74ba9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -44,10 +44,10 @@ enum { }; typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + int16_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema } STableComInfo; /* @@ -56,49 +56,45 @@ typedef struct STableComInfo { * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. */ typedef struct SCTableMeta { - int32_t vgId:24; + int32_t vgId : 24; int8_t tableType; uint64_t uid; uint64_t suid; } SCTableMeta; /* - * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. + * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a + * SCTableMeta. */ typedef struct STableMeta { - //BEGIN: KEEP THIS PART SAME WITH SCTableMeta - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; - //END: KEEP THIS PART SAME WITH SCTableMeta - - // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; + // BEGIN: KEEP THIS PART SAME WITH SCTableMeta + int32_t vgId : 24; + int8_t tableType; + uint64_t uid; + uint64_t suid; + // END: KEEP THIS PART SAME WITH SCTableMeta + + // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta + // info + int16_t sversion; + int16_t tversion; + STableComInfo tableInfo; + SSchema schema[]; } STableMeta; typedef struct SDBVgInfo { - int32_t vgVersion; + int32_t vgVersion; int8_t hashMethod; - SHashObj *vgHash; //key:vgId, value:SVgroupInfo + SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; typedef struct SUseDbOutput { - char db[TSDB_DB_FNAME_LEN]; - uint64_t dbId; - SDBVgInfo *dbVgroup; + char db[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgInfo* dbVgroup; } SUseDbOutput; -enum { - META_TYPE_NULL_TABLE = 1, - META_TYPE_CTABLE, - META_TYPE_TABLE, - META_TYPE_BOTH_TABLE -}; - +enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE }; typedef struct STableMetaOutput { int32_t metaType; @@ -107,30 +103,30 @@ typedef struct STableMetaOutput { char ctbName[TSDB_TABLE_NAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN]; SCTableMeta ctbMeta; - STableMeta *tbMeta; + STableMeta* tbMeta; } STableMetaOutput; typedef struct SDataBuf { - void *pData; - uint32_t len; - void *handle; + void* pData; + uint32_t len; + void* handle; } SDataBuf; typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; //async callback function - void *param; - uint64_t requestId; - uint64_t requestObjRefId; - int32_t msgType; - SDataBuf msgInfo; + __async_send_cb_fn_t fp; // async callback function + void* param; + uint64_t requestId; + uint64_t requestObjRefId; + int32_t msgType; + SDataBuf msgInfo; } SMsgSendInfo; typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId - SEpSet epset; + SEpSet epSet; } SQueryNodeAddr; int32_t initTaskQueue(); @@ -154,32 +150,67 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); * @param pInfo * @return */ -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); -int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp); +int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp); void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); -int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta); +int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); -extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); -extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); +extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); -#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE -#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE -#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE +#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE +#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE +#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) +#define qFatal(...) \ + do { \ + if (qDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qError(...) \ + do { \ + if (qDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qWarn(...) \ + do { \ + if (qDebugFlag & DEBUG_WARN) { \ + taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qInfo(...) \ + do { \ + if (qDebugFlag & DEBUG_INFO) { \ + taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 259d0e5799..e9febef7e2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -517,7 +517,7 @@ void* doFetchRow(SRequestObj* pRequest) { SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - epSet = pVgroupInfo->epset; + epSet = pVgroupInfo->epSet; } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -534,7 +534,7 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->body.requestMsg.pData = pShowReq; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - epSet = pVgroupInfo->epset; + epSet = pVgroupInfo->epSet; int64_t transporterId = 0; STscObj* pTscObj = pRequest->pTscObj; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 10dc378518..60103cc9c5 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE - #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -606,17 +604,17 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; return pRsp->skipLogNum; } void tmqShowMsg(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - static bool noPrintSchema; - char pBuf[128]; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; - int32_t colNum = pRsp->schemas->nCols; + static bool noPrintSchema; + char pBuf[128]; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; + int32_t colNum = pRsp->schemas->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -703,7 +701,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); + tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { /*printf("no data\n");*/ @@ -874,7 +872,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -886,7 +884,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien reqOffset = tmq->resetOffsetCfg; } - SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + SMqPollReq* pReq = malloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -900,7 +898,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + pReq->head.contLen = htonl(sizeof(SMqPollReq)); return pReq; } @@ -914,7 +912,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ /*continue;*/ /*}*/ - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem @@ -941,7 +939,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqConsumeReq), + .len = sizeof(SMqPollReq), .handle = NULL, }; sendInfo->requestId = generateRequestId(); @@ -982,7 +980,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { if (vgStatus != TMQ_VG_STATUS__IDLE) { continue; } - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); @@ -1011,7 +1009,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqConsumeReq), + .len = sizeof(SMqPollReq), .handle = NULL, }; sendInfo->requestId = generateRequestId(); @@ -1271,7 +1269,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->consumeRsp; tDeleteSMqConsumeRsp(pRsp); /*free(tmq_message);*/ taosFreeQitem(tmq_message); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e7309eacc8..97b19c1c79 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1481,7 +1481,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1; } return 0; @@ -1541,7 +1541,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1; taosArrayPush(pRsp->pVgroupInfos, &vgInfo); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 974f5fc982..9cde9201ed 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -900,10 +900,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { vgInfo.vgId = pVgroup->vgId; vgInfo.hashBegin = pVgroup->hashBegin; vgInfo.hashEnd = pVgroup->hashEnd; - vgInfo.epset.numOfEps = pVgroup->replica; + vgInfo.epSet.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp *pEp = &vgInfo.epset.eps[gid]; + SEp *pEp = &vgInfo.epSet.eps[gid]; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); @@ -911,7 +911,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - vgInfo.epset.inUse = gid; + vgInfo.epSet.inUse = gid; } } vindex++; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 5308c0c0f6..dfde6a9258 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -33,23 +33,29 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; SQueryDag* pDag = qStringToDag(pTopic->physicalPlan); - SArray* pAray = NULL; - SArray* unassignedVg = pSub->unassignedVg; + if (pDag == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } ASSERT(pSub->vgNum == 0); int32_t levelNum = taosArrayGetSize(pDag->pSubplans); if (levelNum != 1) { + qDestroyQueryDag(pDag); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; return -1; } - SArray* inner = taosArrayGet(pDag->pSubplans, 0); + SArray* plans = taosArrayGet(pDag->pSubplans, 0); - int32_t opNum = taosArrayGetSize(inner); + int32_t opNum = taosArrayGetSize(plans); if (opNum != 1) { + qDestroyQueryDag(pDag); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; return -1; } - SSubplan* plan = taosArrayGetP(inner, 0); + SSubplan* plan = taosArrayGetP(plans, 0); void* pIter = NULL; while (1) { @@ -62,17 +68,24 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib pSub->vgNum++; plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup); + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; consumerEp.consumerId = -1; - consumerEp.epSet = plan->execNode.epset; + consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; int32_t msgLen; - int32_t code = qSubPlanToString(plan, &consumerEp.qmsg, &msgLen); - taosArrayPush(unassignedVg, &consumerEp); + if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryDag(pDag); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + taosArrayPush(pSub->unassignedVg, &consumerEp); } + qDestroyQueryDag(pDag); + return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 9acd881897..1d964c4383 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -93,7 +93,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj strcpy(pSub->key, key); if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; tDeleteSMqSubscribeObj(pSub); free(pSub); return NULL; @@ -295,7 +294,11 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { for (int32_t k = 0; k < vgsz; k++) { char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + SMqSubVgEp vgEp = { + .epSet = pConsumerEp->epSet, + .vgId = pConsumerEp->vgId, + .offset = -1, + }; mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); if (pOffsetObj != NULL) { @@ -345,7 +348,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { - // TODO + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe)); @@ -412,7 +415,11 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { mInfo("mq rebalance will be triggered"); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg)}; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_DO_REBALANCE, + .pCont = pRebMsg, + .contLen = sizeof(SMqDoRebalanceMsg), + }; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64c7a66bf9..d3642f4204 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -96,7 +96,11 @@ static void mndCalMqRebalance(void *param, void *tmrId) { if (mndIsMaster(pMnode)) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_TIMER, + .pCont = pReq, + .contLen = contLen, + }; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } @@ -631,4 +635,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 17fda48cd7..9dbc1be4e9 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -292,9 +292,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); - EXPECT_EQ(pInfo->epset.inUse, 0); - EXPECT_EQ(pInfo->epset.numOfEps, 1); - SEp* pAddr = &pInfo->epset.eps[0]; + EXPECT_EQ(pInfo->epSet.inUse, 0); + EXPECT_EQ(pInfo->epSet.numOfEps, 1); + SEp* pAddr = &pInfo->epSet.eps[0]; EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } @@ -307,9 +307,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); - EXPECT_EQ(pInfo->epset.inUse, 0); - EXPECT_EQ(pInfo->epset.numOfEps, 1); - SEp* pAddr = &pInfo->epset.eps[0]; + EXPECT_EQ(pInfo->epSet.inUse, 0); + EXPECT_EQ(pInfo->epSet.numOfEps, 1); + SEp* pAddr = &pInfo->epSet.eps[0]; EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 36626514ec..b4d45e83fb 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -13,16 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_TQ_H_ -#define _TD_TQ_H_ +#ifndef _TQ_H_ +#define _TQ_H_ -#include "tcommon.h" #include "executor.h" -#include "tmallocator.h" #include "meta.h" -#include "scheduler.h" #include "taoserror.h" -#include "tlist.h" +#include "tcommon.h" +#include "tmallocator.h" #include "tmsg.h" #include "trpc.h" #include "ttimer.h" @@ -54,7 +52,7 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); int tqCommit(STQ*); -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); @@ -62,4 +60,4 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg); } #endif -#endif /*_TD_TQ_H_*/ +#endif /*_TQ_H_*/ diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index f416413859..fc6a3699d5 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -19,7 +19,7 @@ #include "meta.h" #include "tlog.h" #include "tq.h" -#include "trpc.h" +#include "tqPush.h" #ifdef __cplusplus extern "C" { @@ -31,30 +31,35 @@ extern "C" { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ } \ } + #define tqError(...) \ { \ if (tqDebugFlag & DEBUG_ERROR) { \ taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ } \ } + #define tqWarn(...) \ { \ if (tqDebugFlag & DEBUG_WARN) { \ taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ } \ } + #define tqInfo(...) \ { \ if (tqDebugFlag & DEBUG_INFO) { \ taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \ } \ } + #define tqDebug(...) \ { \ if (tqDebugFlag & DEBUG_DEBUG) { \ taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \ } \ } + #define tqTrace(...) \ { \ if (tqDebugFlag & DEBUG_TRACE) { \ @@ -138,9 +143,7 @@ typedef struct { // topics that are not connectted STqMetaList* unconnectTopic; - // TODO:temporaral use, to be replaced by unified tfile TdFilePtr pFile; - // TODO:temporaral use, to be replaced by unified tfile TdFilePtr pIdxFile; char* dirPath; @@ -157,6 +160,7 @@ struct STQ { STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; + STqPushMgr* tqPushMgr; SWal* pWal; SMeta* pVnodeMeta; }; diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h new file mode 100644 index 0000000000..32fd7c3ddf --- /dev/null +++ b/source/dnode/vnode/src/inc/tqPush.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TQ_PUSH_H_ +#define _TQ_PUSH_H_ + +#include "thash.h" +#include "trpc.h" +#include "ttimer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + TQ_PUSHER_TYPE__CLIENT = 1, + TQ_PUSHER_TYPE__STREAM, +}; + +typedef struct { + int8_t type; + int8_t reserved[3]; + int32_t ttl; + int64_t consumerId; + SRpcMsg* pMsg; + // SMqPollRsp* rsp; +} STqClientPusher; + +typedef struct { + int8_t type; + int8_t nodeType; + int8_t reserved[6]; + int64_t streamId; + SEpSet epSet; +} STqStreamPusher; + +typedef struct { + int8_t type; // mq or stream +} STqPusher; + +typedef struct { + SHashObj* pHash; // +} STqPushMgr; + +typedef struct { + int8_t inited; + tmr_h timer; +} STqPushMgmt; + +static STqPushMgmt tqPushMgmt; + +int32_t tqPushMgrInit(); +void tqPushMgrCleanUp(); + +STqPushMgr* tqPushMgrOpen(); +void tqPushMgrClose(STqPushMgr* pushMgr); + +STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl); +STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TQ_PUSH_H_*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d540b589b7..16809f1527 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,28 +12,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -int tqInit() { - int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); - if (old == 1) return 0; +int32_t tqInit() { return tqPushMgrInit(); } - tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); - return 0; -} +void tqCleanUp() { tqPushMgrCleanUp(); } -void tqCleanUp() { - int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0); - if (old == 0) return; - taosTmrStop(tqMgmt.timer); - taosTmrCleanUp(tqMgmt.timer); -} - -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -42,7 +30,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->pWal = pWal; - pTq->pVnodeMeta = pMeta; + pTq->pVnodeMeta = pVnodeMeta; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -60,6 +48,13 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl return NULL; } + pTq->tqPushMgr = tqPushMgrOpen(); + if (pTq->tqPushMgr == NULL) { + // free store + free(pTq); + return NULL; + } + return pTq; } @@ -72,6 +67,8 @@ void tqClose(STQ* pTq) { } int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + // iterate hash + // process all msg // if waiting // memcpy and send msg to fetch thread // TODO: add reference @@ -199,7 +196,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu for (int j = 0; j < TQ_BUFFER_SIZE; j++) { pTopic->buffer.output[j].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); - SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta}; + SReadHandle handle = { + .reader = pReadHandle, + .meta = pTq->pVnodeMeta, + }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); } @@ -208,11 +208,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { - SMqConsumeReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset; - int64_t blockingTime = pReq->blockingTime; +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t fetchOffset; + int64_t blockingTime = pReq->blockingTime; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = 0; @@ -222,7 +222,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset = pReq->currentOffset + 1; } - SMqConsumeRsp rsp = { + SMqPollRsp rsp = { .consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL, @@ -236,6 +236,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { rpcSendResponse(pMsg); return 0; } + int sz = taosArrayGetSize(pConsumer->topics); ASSERT(sz == 1); STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); @@ -247,13 +248,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SWalHead* pHead; while (1) { - int8_t pos = fetchOffset % TQ_BUFFER_SIZE; + /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user break; } + int8_t pos = fetchOffset % TQ_BUFFER_SIZE; pHead = pTopic->pReadhandle->pHead; if (pHead->head.msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; @@ -280,7 +282,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { rsp.numOfTopics = 1; rsp.pBlockData = pRes; - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; @@ -290,7 +292,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ((SMqRspHead*)buf)->epoch = pReq->epoch; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqConsumeRsp(&abuf, &rsp); + tEncodeSMqPollRsp(&abuf, &rsp); taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); pMsg->pCont = buf; pMsg->contLen = tlen; @@ -304,7 +306,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } } - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); + /*if (blockingTime != 0) {*/ + /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ + /*} else {*/ + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; @@ -314,12 +319,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ((SMqRspHead*)buf)->epoch = pReq->epoch; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqConsumeRsp(&abuf, &rsp); + tEncodeSMqPollRsp(&abuf, &rsp); rsp.pBlockData = NULL; pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; rpcSendResponse(pMsg); + /*}*/ + return 0; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c new file mode 100644 index 0000000000..fea65846be --- /dev/null +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tqPush.h" + +int32_t tqPushMgrInit() { + // + int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1); + if (old == 1) return 0; + + tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); + return 0; +} + +void tqPushMgrCleanUp() { + int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0); + if (old == 0) return; + taosTmrStop(tqPushMgmt.timer); + taosTmrCleanUp(tqPushMgmt.timer); +} + +STqPushMgr* tqPushMgrOpen() { + STqPushMgr* mgr = malloc(sizeof(STqPushMgr)); + if (mgr == NULL) { + return NULL; + } + mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return mgr; +} + +void tqPushMgrClose(STqPushMgr* pushMgr) { + taosHashCleanup(pushMgr->pHash); + free(pushMgr); +} + +STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { + STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher)); + if (clientPusher == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + clientPusher->type = TQ_PUSHER_TYPE__CLIENT; + clientPusher->pMsg = pMsg; + clientPusher->consumerId = consumerId; + clientPusher->ttl = ttl; + if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(clientPusher); + // TODO send rsp back + return NULL; + } + return clientPusher; +} + +STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { + STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher)); + if (streamPusher == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + streamPusher->type = TQ_PUSHER_TYPE__STREAM; + streamPusher->nodeType = 0; + streamPusher->streamId = streamId; + memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet)); + + if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(streamPusher); + return NULL; + } + return streamPusher; +} diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4b47413715..ccb59c26de 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -29,7 +29,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta}; switch (pMsg->msgType) { - case TDMT_VND_QUERY:{ + case TDMT_VND_QUERY: { return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); } case TDMT_VND_QUERY_CONTINUE: @@ -63,7 +63,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: - return tqProcessConsumeReq(pVnode->pTq, pMsg); + return tqProcessPollReq(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -71,8 +71,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { - STbCfg * pTbCfg = NULL; - STbCfg * pStbCfg = NULL; + STbCfg *pTbCfg = NULL; + STbCfg *pStbCfg = NULL; tb_uid_t uid; int32_t nCols; int32_t nTagCols; @@ -204,9 +204,9 @@ static void freeItemHelper(void *pItem) { */ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta); - SArray * pArray = taosArrayInit(10, POINTER_BYTES); + SArray *pArray = taosArrayInit(10, POINTER_BYTES); - char * name = NULL; + char *name = NULL; int32_t totalLen = 0; int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 56c0ec1130..77d25fa164 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -845,7 +845,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index aeac6b3eee..c7867c4da5 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -228,10 +228,10 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { vgInfo.vgId = i + 1; vgInfo.hashBegin = i * hashUnit; vgInfo.hashEnd = hashUnit * (i + 1) - 1; - vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vgInfo.epset.inUse = i % vgInfo.epset.numOfEps; - for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) { - SEp *addr = &vgInfo.epset.eps[n]; + vgInfo.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1; + vgInfo.epSet.inUse = i % vgInfo.epSet.numOfEps; + for (int32_t n = 0; n < vgInfo.epSet.numOfEps; ++n) { + SEp *addr = &vgInfo.epSet.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = n + 22; } @@ -301,10 +301,10 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * vg.hashEnd = htonl(UINT32_MAX); } - vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vg.epset.inUse = i % vg.epset.numOfEps; - for (int32_t n = 0; n < vg.epset.numOfEps; ++n) { - SEp *addr = &vg.epset.eps[n]; + vg.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg.epSet.inUse = i % vg.epSet.numOfEps; + for (int32_t n = 0; n < vg.epSet.numOfEps; ++n) { + SEp *addr = &vg.epSet.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = n + 22; } @@ -877,7 +877,7 @@ TEST(tableMeta, normalTable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { usleep(50000); @@ -1384,7 +1384,7 @@ TEST(refreshGetMeta, normal2normal) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1463,7 +1463,7 @@ TEST(refreshGetMeta, normal2notexist) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1537,7 +1537,7 @@ TEST(refreshGetMeta, normal2child) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1621,7 +1621,7 @@ TEST(refreshGetMeta, stable2child) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1706,7 +1706,7 @@ TEST(refreshGetMeta, stable2stable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1794,7 +1794,7 @@ TEST(refreshGetMeta, child2stable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (true) { uint64_t n = 0; @@ -1879,7 +1879,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 8); - ASSERT_EQ(vgInfo->epset.numOfEps, 3); + ASSERT_EQ(vgInfo->epSet.numOfEps, 3); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1921,7 +1921,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 9); - ASSERT_EQ(vgInfo->epset.numOfEps, 4); + ASSERT_EQ(vgInfo->epSet.numOfEps, 4); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1964,13 +1964,13 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 1); - ASSERT_EQ(vgInfo->epset.numOfEps, 1); + ASSERT_EQ(vgInfo->epSet.numOfEps, 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1); ASSERT_EQ(vgInfo->vgId, 2); - ASSERT_EQ(vgInfo->epset.numOfEps, 2); + ASSERT_EQ(vgInfo->epSet.numOfEps, 2); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2); ASSERT_EQ(vgInfo->vgId, 3); - ASSERT_EQ(vgInfo->epset.numOfEps, 3); + ASSERT_EQ(vgInfo->epSet.numOfEps, 3); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2025,14 +2025,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.epset.numOfEps, 3); + ASSERT_EQ(vgInfo.epSet.numOfEps, 3); code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->epset.numOfEps, 3); + ASSERT_EQ(pvgInfo->epSet.numOfEps, 3); taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); @@ -2053,14 +2053,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); - ASSERT_EQ(vgInfo.epset.numOfEps, 2); + ASSERT_EQ(vgInfo.epSet.numOfEps, 2); code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->epset.numOfEps, 3); + ASSERT_EQ(pvgInfo->epSet.numOfEps, 3); taosArrayDestroy(vgList); catalogDestroy(); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3ebad151fd..eeb48a1f3d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4977,7 +4977,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf SSourceDataInfo *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); @@ -5000,7 +5000,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 713847807d..7ec2d05d44 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -58,7 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - *pEpSet = info->epset; + *pEpSet = info->epSet; *outputLen = sizeof(SVShowTablesReq); *output = pShowReq; @@ -902,4 +902,4 @@ SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseCon } return pModifSqlStmt; -} \ No newline at end of file +} diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index cdb547ee1b..cd3ffcdce9 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -43,11 +43,11 @@ public: SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; - vgroup.epset.eps[0] = (SEp){"dnode_1", 6030}; - vgroup.epset.eps[1] = (SEp){"dnode_2", 6030}; - vgroup.epset.eps[2] = (SEp){"dnode_3", 6030}; - vgroup.epset.inUse = 0; - vgroup.epset.numOfEps = 3; + vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030}; + vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030}; + vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030}; + vgroup.epSet.inUse = 0; + vgroup.epSet.numOfEps = 3; meta_->vgs.emplace_back(vgroup); return *this; @@ -122,7 +122,7 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo vgInfo->vgId = 1; - addEpIntoEpSet(&vgInfo->epset, "node1", 6030); + addEpIntoEpSet(&vgInfo->epSet, "node1", 6030); return 0; } @@ -143,10 +143,10 @@ public: meta_[db][tbname]->schema->uid = id_++; SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; - addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030); - addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030); - addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030); - vgroup.epset.inUse = 0; + addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); + vgroup.epSet.inUse = 0; meta_[db][tbname]->vgs.emplace_back(vgroup); // super table @@ -313,4 +313,4 @@ int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableM int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { return impl_->catalogGetTableHashVgroup(pTableName, vgInfo); -} \ No newline at end of file +} diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 748be8fcd8..dbc0340b34 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -254,7 +254,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { pNodeAddr->nodeId = vg->vgId; - pNodeAddr->epset = vg->epset; + pNodeAddr->epSet = vg->epSet; } static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) { @@ -363,7 +363,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); - subplan->execNode.epset = blocks->vg.epset; + subplan->execNode.epSet = blocks->vg.epSet; subplan->pDataSink = createDataInserter(pCxt, blocks, NULL); subplan->pNode = NULL; subplan->type = QUERY_TYPE_MODIFY; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index b2109c0a4f..f61102a66d 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -762,11 +762,11 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) { bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); if (res) { - res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse); + res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epSet.inUse); } if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps); + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epSet.eps, sizeof(SEp), pAddr->epSet.numOfEps); } return res; } @@ -775,11 +775,11 @@ static bool queryNodeAddrFromJson(const cJSON* json, void* obj) { SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj; pAddr->nodeId = getNumber(json, jkNodeAddrId); - pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse); + pAddr->epSet.inUse = getNumber(json, jkNodeAddrInUse); int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps); - pAddr->epset.numOfEps = numOfEps; + bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps); + pAddr->epSet.numOfEps = numOfEps; return res; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f1ed0cef7d..8ed39eb0b7 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -423,13 +423,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pTask->plan->execNode.epset.numOfEps > 0) { + if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps); + SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -1061,7 +1061,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, isCandidateAddr = true; } - SEpSet epSet = addr->epset; + SEpSet epSet = addr->epSet; switch (msgType) { case TDMT_VND_CREATE_TABLE: diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index daf65cf251..8ed963d875 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -104,8 +104,8 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->type = QUERY_TYPE_SCAN; scanPlan->execNode.nodeId = 1; - scanPlan->execNode.epset.inUse = 0; - addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030); + scanPlan->execNode.epSet.inUse = 0; + addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030); scanPlan->pChildren = NULL; scanPlan->level = 1; @@ -118,7 +118,7 @@ void schtBuildQueryDag(SQueryDag *dag) { mergePlan->id.subplanId = 0x5555555555; mergePlan->type = QUERY_TYPE_MERGE; mergePlan->level = 0; - mergePlan->execNode.epset.numOfEps = 0; + mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pParents = NULL; @@ -157,8 +157,8 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[0].level = 0; insertPlan[0].execNode.nodeId = 1; - insertPlan[0].execNode.epset.inUse = 0; - addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030); + insertPlan[0].execNode.epSet.inUse = 0; + addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030); insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; @@ -173,8 +173,8 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[1].level = 0; insertPlan[1].execNode.nodeId = 1; - insertPlan[1].execNode.epset.inUse = 0; - addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030); + insertPlan[1].execNode.epSet.inUse = 0; + addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030); insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 3fe0425c91..3eb8e60d56 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -352,7 +352,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { int32_t cnt = 0; /*clock_t startTime = clock();*/ while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { cnt++; msg_process(tmqmessage); @@ -383,7 +383,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { msg_process(tmqmessage); tmq_message_destroy(tmqmessage); @@ -411,7 +411,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { batchCnt++; skipLogNum += tmqGetSkipLogNum(tmqmessage); From 8447cea613ed90a6f63016ff96ba864b641da2b6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 8 Mar 2022 17:33:27 +0800 Subject: [PATCH 12/39] disable destory --- source/dnode/mnode/impl/src/mndScheduler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index dfde6a9258..6b81852545 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -85,7 +85,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib taosArrayPush(pSub->unassignedVg, &consumerEp); } - qDestroyQueryDag(pDag); + /*qDestroyQueryDag(pDag);*/ return 0; } From cc24ae769ca6ddd4a55c9db5aeb3084be07cfe38 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 8 Mar 2022 17:38:10 +0800 Subject: [PATCH 13/39] fix crash --- source/dnode/mnode/impl/src/mndScheduler.c | 4 +- source/libs/planner/src/physicalPlanJson.c | 209 +++++++++++---------- 2 files changed, 115 insertions(+), 98 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 6b81852545..ce288d33f7 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -47,7 +47,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } - SArray* plans = taosArrayGet(pDag->pSubplans, 0); + SArray* plans = taosArrayGetP(pDag->pSubplans, 0); int32_t opNum = taosArrayGetSize(plans); if (opNum != 1) { @@ -85,7 +85,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib taosArrayPush(pSub->unassignedVg, &consumerEp); } - /*qDestroyQueryDag(pDag);*/ + qDestroyQueryDag(pDag); return 0; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index f61102a66d..4018741091 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "plannerInt.h" -#include "parser.h" #include "cJSON.h" +#include "parser.h" +#include "plannerInt.h" -typedef bool (*FToJson)(const void* obj, cJSON* json); -typedef bool (*FFromJson)(const cJSON* json, void* obj); +typedef bool (*FToJson)(const void* obj, cJSON* json); +typedef bool (*FFromJson)(const cJSON* json, void* obj); static char* getString(const cJSON* json, const char* name) { char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); @@ -30,7 +30,7 @@ static void copyString(const cJSON* json, const char* name, char* dst) { } static uint64_t getBigintFromString(const cJSON* json, const char* name) { - char* val = getString(json, name); + char* val = getString(json, name); uint64_t intVal = strtoul(val, NULL, 10); tfree(val); @@ -39,7 +39,7 @@ static uint64_t getBigintFromString(const cJSON* json, const char* name) { static int64_t getNumber(const cJSON* json, const char* name) { double d = cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); - return (int64_t) d; + return (int64_t)d; } static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { @@ -72,7 +72,8 @@ static bool fromObject(const cJSON* json, const char* name, FFromJson func, void return func(jObj, obj); } -static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) { +static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, + bool required) { cJSON* jObj = cJSON_GetObjectItem(json, name); if (NULL == jObj) { return !required; @@ -85,7 +86,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f } static const char* jkPnodeType = "Type"; -static int32_t getPnodeTypeSize(cJSON* json) { +static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { case OP_StreamScan: case OP_TableScan: @@ -119,7 +120,7 @@ static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void* static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func, SArray** array) { const cJSON* jArray = cJSON_GetObjectItem(json, name); - int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); if (size > 0) { *array = taosArrayInit(size, POINTER_BYTES); if (NULL == *array) { @@ -128,7 +129,7 @@ static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func, } for (int32_t i = 0; i < size; ++i) { cJSON* jItem = cJSON_GetArrayItem(jArray, i); - void* item = calloc(1, getPnodeTypeSize(jItem)); + void* item = calloc(1, getPnodeTypeSize(jItem)); if (NULL == item || !func(jItem, item)) { return false; } @@ -161,9 +162,10 @@ static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* return addTarray(json, name, func, array, true); } -static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize, bool isPoint) { +static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize, + bool isPoint) { const cJSON* jArray = cJSON_GetObjectItem(json, name); - int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); if (size > 0) { *array = taosArrayInit(size, isPoint ? POINTER_BYTES : itemSize); if (NULL == *array) { @@ -188,7 +190,8 @@ static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArra return fromTarray(json, name, func, array, itemSize, true); } -static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) { +static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, + int32_t size) { if (size > 0) { cJSON* jArray = cJSON_AddArrayToObject(json, name); if (NULL == jArray) { @@ -218,7 +221,8 @@ static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t i return true; } -static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { +static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, + int32_t* size) { const cJSON* jArray = getArray(json, name, size); if (*size > 0) { *array = calloc(1, itemSize * (*size)); @@ -229,7 +233,8 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, + int32_t* size) { const cJSON* jArray = getArray(json, name, size); return fromItem(jArray, func, array, itemSize, *size); } @@ -240,7 +245,7 @@ static const char* jkSchemaBytes = "Bytes"; // The 'name' field do not need to be serialized. static bool schemaToJson(const void* obj, cJSON* jSchema) { const SSlotSchema* schema = (const SSlotSchema*)obj; - bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type); + bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type); if (res) { res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId); } @@ -264,7 +269,8 @@ static const char* jkDataBlockSchemaPrecision = "Precision"; static bool dataBlockSchemaToJson(const void* obj, cJSON* json) { const SDataBlockSchema* schema = (const SDataBlockSchema*)obj; - bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema), schema->numOfCols); + bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema), + schema->numOfCols); if (res) { res = cJSON_AddNumberToObject(json, jkDataBlockSchemaResultRowSize, schema->resultRowSize); } @@ -279,7 +285,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); + return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), + sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -290,7 +297,7 @@ static const char* jkColumnFilterInfoUpperBnd = "UpperBnd"; static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) { const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj; - bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr); + bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr); if (res) { res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr); } @@ -323,7 +330,7 @@ static const char* jkColumnInfoFilterList = "FilterList"; static bool columnInfoToJson(const void* obj, cJSON* jCol) { const SColumnInfo* col = (const SColumnInfo*)obj; - bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId); + bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId); if (res) { res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type); } @@ -331,8 +338,9 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) { res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes); } - if (res) { // TODO: temporarily disable it -// res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters); + if (res) { // TODO: temporarily disable it + // res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, + // sizeof(SColumnFilterInfo), col->flist.numOfFilters); } return res; @@ -341,10 +349,11 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) { static bool columnInfoFromJson(const cJSON* json, void* obj) { SColumnInfo* col = (SColumnInfo*)obj; col->colId = getNumber(json, jkColumnInfoColId); - col->type = getNumber(json, jkColumnInfoType); + col->type = getNumber(json, jkColumnInfoType); col->bytes = getNumber(json, jkColumnInfoBytes); int32_t size = 0; - bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); + bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, + (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); col->flist.numOfFilters = size; return res; } @@ -355,7 +364,7 @@ static const char* jkColumnInfo = "Info"; static bool columnToJson(const void* obj, cJSON* jCol) { const SColumn* col = (const SColumn*)obj; - bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid); + bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid); if (res) { res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag); } @@ -381,7 +390,7 @@ static const char* jkExprNodeRight = "Right"; static bool operatorToJson(const void* obj, cJSON* jOper) { const tExprNode* exprInfo = (const tExprNode*)obj; - bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr); + bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr); if (res) { res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft); } @@ -406,9 +415,10 @@ static const char* jkFunctionChild = "Child"; static bool functionToJson(const void* obj, cJSON* jFunc) { const tExprNode* exprInfo = (const tExprNode*)obj; - bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); + bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); if (res && NULL != exprInfo->_function.pChild) { - res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); + res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), + exprInfo->_function.num); } return res; } @@ -420,7 +430,8 @@ static bool functionFromJson(const cJSON* json, void* obj) { if (NULL == exprInfo->_function.pChild) { return false; } - return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num); + return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, + sizeof(tExprNode*), &exprInfo->_function.num); } static const char* jkVariantType = "Type"; @@ -430,12 +441,12 @@ static const char* jkVariantValue = "Value"; static bool variantToJson(const void* obj, cJSON* jVar) { const SVariant* var = (const SVariant*)obj; - bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType); + bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType); if (res) { res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen); } if (res) { - if (0/* in */) { + if (0 /* in */) { res = addArray(jVar, jkVariantvalues, variantToJson, var->arr); } else if (IS_NUMERIC_TYPE(var->nType)) { res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d); @@ -450,7 +461,7 @@ static bool variantFromJson(const cJSON* json, void* obj) { SVariant* var = (SVariant*)obj; var->nType = getNumber(json, jkVariantType); var->nLen = getNumber(json, jkVariantLen); - if (0/* in */) { + if (0 /* in */) { return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant)); } else if (IS_NUMERIC_TYPE(var->nType)) { var->d = getNumber(json, jkVariantValue); @@ -468,7 +479,7 @@ static const char* jkExprNodeValue = "Value"; static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) { const tExprNode* exprInfo = *(const tExprNode**)obj; - bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); + bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); if (res) { switch (exprInfo->nodeType) { case TEXPR_BINARYEXPR_NODE: @@ -502,7 +513,8 @@ static bool exprNodeFromJson(const cJSON* json, void* obj) { case TEXPR_FUNCTION_NODE: return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false); case TEXPR_COL_NODE: - return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema), false); + return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema), + false); case TEXPR_VALUE_NODE: return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false); default: @@ -518,7 +530,7 @@ static const char* jkSqlExprParams = "Params"; // token does not need to be serialized. static bool sqlExprToJson(const void* obj, cJSON* jExpr) { const SSqlExpr* expr = (const SSqlExpr*)obj; - bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema); + bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema); if (res) { res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols); } @@ -533,9 +545,10 @@ static bool sqlExprToJson(const void* obj, cJSON* jExpr) { static bool sqlExprFromJson(const cJSON* json, void* obj) { SSqlExpr* expr = (SSqlExpr*)obj; - bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false); + bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false); if (res) { - res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols); + res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), + &expr->numOfCols); } if (res) { expr->interBytes = getNumber(json, jkSqlExprInterBytes); @@ -553,7 +566,7 @@ static const char* jkExprInfoExpr = "Expr"; static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { const SExprInfo* exprInfo = (const SExprInfo*)obj; - bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); + bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); if (res) { res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, &exprInfo->pExpr); } @@ -562,9 +575,10 @@ static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { static bool exprInfoFromJson(const cJSON* json, void* obj) { SExprInfo* exprInfo = (SExprInfo*)obj; - bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true); + bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true); if (res) { - res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true); + res = + fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true); } return res; } @@ -576,12 +590,12 @@ static bool timeWindowToJson(const void* obj, cJSON* json) { const STimeWindow* win = (const STimeWindow*)obj; char tmp[40] = {0}; - snprintf(tmp, tListLen(tmp),"%"PRId64, win->skey); + snprintf(tmp, tListLen(tmp), "%" PRId64, win->skey); bool res = cJSON_AddStringToObject(json, jkTimeWindowStartKey, tmp); if (res) { memset(tmp, 0, tListLen(tmp)); - snprintf(tmp, tListLen(tmp),"%"PRId64, win->ekey); + snprintf(tmp, tListLen(tmp), "%" PRId64, win->ekey); res = cJSON_AddStringToObject(json, jkTimeWindowEndKey, tmp); } return res; @@ -604,7 +618,7 @@ static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* pNode = (const SScanPhyNode*)obj; char uid[40] = {0}; - snprintf(uid, tListLen(uid), "%"PRIu64, pNode->uid); + snprintf(uid, tListLen(uid), "%" PRIu64, pNode->uid); bool res = cJSON_AddStringToObject(json, jkScanNodeTableId, uid); if (res) { @@ -629,11 +643,11 @@ static bool scanNodeToJson(const void* obj, cJSON* json) { static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* pNode = (SScanPhyNode*)obj; - pNode->uid = getBigintFromString(json, jkScanNodeTableId); + pNode->uid = getBigintFromString(json, jkScanNodeTableId); pNode->tableType = getNumber(json, jkScanNodeTableType); - pNode->count = getNumber(json, jkScanNodeTableCount); - pNode->order = getNumber(json, jkScanNodeTableOrder); - pNode->reverse = getNumber(json, jkScanNodeTableRevCount); + pNode->count = getNumber(json, jkScanNodeTableCount); + pNode->order = getNumber(json, jkScanNodeTableOrder); + pNode->reverse = getNumber(json, jkScanNodeTableRevCount); return true; } @@ -644,7 +658,7 @@ static const char* jkColIndexName = "Name"; static bool colIndexToJson(const void* obj, cJSON* json) { const SColIndex* col = (const SColIndex*)obj; - bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId); + bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId); if (res) { res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex); } @@ -673,7 +687,7 @@ static const char* jkAggNodeGroupByList = "GroupByList"; static bool aggNodeToJson(const void* obj, cJSON* json) { const SAggPhyNode* agg = (const SAggPhyNode*)obj; - bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo); + bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo); if (res) { res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit); } @@ -703,7 +717,7 @@ static const char* jkTableScanNodeTagsConditions = "TagsConditions"; static bool tableScanNodeToJson(const void* obj, cJSON* json) { const STableScanPhyNode* scan = (const STableScanPhyNode*)obj; - bool res = scanNodeToJson(obj, json); + bool res = scanNodeToJson(obj, json); if (res) { res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag); } @@ -718,7 +732,7 @@ static bool tableScanNodeToJson(const void* obj, cJSON* json) { static bool tableScanNodeFromJson(const cJSON* json, void* obj) { STableScanPhyNode* scan = (STableScanPhyNode*)obj; - bool res = scanNodeFromJson(json, obj); + bool res = scanNodeFromJson(json, obj); if (res) { scan->scanFlag = getNumber(json, jkTableScanNodeFlag); } @@ -736,7 +750,7 @@ static const char* jkEpAddrPort = "Port"; static bool epAddrToJson(const void* obj, cJSON* json) { const SEp* ep = (const SEp*)obj; - bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); + bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); if (res) { res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); } @@ -750,16 +764,16 @@ static bool epAddrFromJson(const cJSON* json, void* obj) { return true; } -static const char* jkNodeAddrId = "NodeId"; -static const char* jkNodeAddrInUse = "InUse"; +static const char* jkNodeAddrId = "NodeId"; +static const char* jkNodeAddrInUse = "InUse"; static const char* jkNodeAddrEpAddrs = "Ep"; -static const char* jkNodeAddr = "NodeAddr"; -static const char* jkNodeTaskId = "TaskId"; +static const char* jkNodeAddr = "NodeAddr"; +static const char* jkNodeTaskId = "TaskId"; static const char* jkNodeTaskSchedId = "SchedId"; static bool queryNodeAddrToJson(const void* obj, cJSON* json) { - const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*) obj; - bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); + const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*)obj; + bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); if (res) { res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epSet.inUse); @@ -772,24 +786,24 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) { } static bool queryNodeAddrFromJson(const cJSON* json, void* obj) { - SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj; + SQueryNodeAddr* pAddr = (SQueryNodeAddr*)obj; pAddr->nodeId = getNumber(json, jkNodeAddrId); pAddr->epSet.inUse = getNumber(json, jkNodeAddrInUse); int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps); + bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps); pAddr->epSet.numOfEps = numOfEps; return res; } static bool nodeAddrToJson(const void* obj, cJSON* json) { - const SDownstreamSource* pSource = (const SDownstreamSource*) obj; - bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId); + const SDownstreamSource* pSource = (const SDownstreamSource*)obj; + bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId); if (res) { char t[30] = {0}; - snprintf(t, tListLen(t), "%"PRIu64, pSource->schedId); + snprintf(t, tListLen(t), "%" PRIu64, pSource->schedId); res = cJSON_AddStringToObject(json, jkNodeTaskSchedId, t); } @@ -813,9 +827,10 @@ static const char* jkExchangeNodeSrcEndPoints = "SrcAddrs"; static bool exchangeNodeToJson(const void* obj, cJSON* json) { const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj; - bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); + bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); if (res) { - res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData, sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints)); + res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData, + sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints)); } return res; } @@ -823,7 +838,8 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) { static bool exchangeNodeFromJson(const cJSON* json, void* obj) { SExchangePhyNode* exchange = (SExchangePhyNode*)obj; exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId); - return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SDownstreamSource)); + return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, + sizeof(SDownstreamSource)); } static bool specificPhyNodeToJson(const void* obj, cJSON* json) { @@ -855,7 +871,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { case OP_AllTimeWindow: case OP_AllMultiTableTimeInterval: case OP_Order: - break; // todo + break; // todo case OP_Exchange: return exchangeNodeToJson(obj, json); default: @@ -893,7 +909,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { case OP_AllTimeWindow: case OP_AllMultiTableTimeInterval: case OP_Order: - break; // todo + break; // todo case OP_Exchange: return exchangeNodeFromJson(json, obj); default: @@ -910,7 +926,7 @@ static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { const SPhyNode* phyNode = (const SPhyNode*)obj; - bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type); + bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type); if (res) { res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); } @@ -933,7 +949,7 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { } static bool phyNodeFromJson(const cJSON* json, void* obj) { - SPhyNode* node = (SPhyNode*) obj; + SPhyNode* node = (SPhyNode*)obj; node->info.type = getNumber(json, jkPnodeType); node->info.name = opTypeToOpName(node->info.type); @@ -959,7 +975,7 @@ static const char* jkInserterDataSize = "DataSize"; static bool inserterToJson(const void* obj, cJSON* json) { const SDataInserter* inserter = (const SDataInserter*)obj; - bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables); + bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables); if (res) { res = cJSON_AddNumberToObject(json, jkInserterDataSize, inserter->size); } @@ -1005,7 +1021,7 @@ static const char* jkDataSinkSchema = "Schema"; static bool dataSinkToJson(const void* obj, cJSON* json) { const SDataSink* dsink = (const SDataSink*)obj; - bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name); + bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name); if (res) { res = addObject(json, dsink->info.name, specificDataSinkToJson, dsink); } @@ -1034,7 +1050,7 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) { const SSubplanId* id = (const SSubplanId*)obj; char ids[40] = {0}; - snprintf(ids, tListLen(ids), "%"PRIu64, id->queryId); + snprintf(ids, tListLen(ids), "%" PRIu64, id->queryId); bool res = cJSON_AddStringToObject(jId, jkIdQueryId, ids); if (res) { @@ -1049,9 +1065,9 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) { static bool subplanIdFromJson(const cJSON* json, void* obj) { SSubplanId* id = (SSubplanId*)obj; - id->queryId = getBigintFromString(json, jkIdQueryId); + id->queryId = getBigintFromString(json, jkIdQueryId); id->templateId = getNumber(json, jkIdTemplateId); - id->subplanId = getNumber(json, jkIdSubplanId); + id->subplanId = getNumber(json, jkIdSubplanId); return true; } @@ -1094,9 +1110,10 @@ static SSubplan* subplanFromJson(const cJSON* json) { } if (res) { - res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); + res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), + false); } - + if (!res) { qDestroySubplan(subplan); return NULL; @@ -1137,15 +1154,15 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) { cJSON* qDagToJson(const SQueryDag* pDag) { cJSON* pRoot = cJSON_CreateObject(); - if(pRoot == NULL) { + if (pRoot == NULL) { return NULL; } cJSON_AddNumberToObject(pRoot, "Number", pDag->numOfSubplans); cJSON_AddNumberToObject(pRoot, "QueryId", pDag->queryId); - cJSON *pLevels = cJSON_CreateArray(); - if(pLevels == NULL) { + cJSON* pLevels = cJSON_CreateArray(); + if (pLevels == NULL) { cJSON_Delete(pRoot); return NULL; } @@ -1153,19 +1170,19 @@ cJSON* qDagToJson(const SQueryDag* pDag) { cJSON_AddItemToObject(pRoot, "Subplans", pLevels); size_t level = taosArrayGetSize(pDag->pSubplans); - for(size_t i = 0; i < level; i++) { + for (size_t i = 0; i < level; i++) { const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i); - size_t num = taosArrayGetSize(pSubplans); - cJSON* plansOneLevel = cJSON_CreateArray(); - if(plansOneLevel == NULL) { + size_t num = taosArrayGetSize(pSubplans); + cJSON* plansOneLevel = cJSON_CreateArray(); + if (plansOneLevel == NULL) { cJSON_Delete(pRoot); return NULL; } cJSON_AddItemToArray(pLevels, plansOneLevel); - for(size_t j = 0; j < num; j++) { + for (size_t j = 0; j < num; j++) { cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j)); - if(pSubplan == NULL) { + if (pSubplan == NULL) { cJSON_Delete(pRoot); return NULL; } @@ -1183,22 +1200,22 @@ char* qDagToString(const SQueryDag* pDag) { SQueryDag* qJsonToDag(const cJSON* pRoot) { SQueryDag* pDag = malloc(sizeof(SQueryDag)); - if(pDag == NULL) { + if (pDag == NULL) { return NULL; } pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number")); pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId")); - pDag->pSubplans = taosArrayInit(0, sizeof(SArray)); + pDag->pSubplans = taosArrayInit(0, sizeof(void*)); if (pDag->pSubplans == NULL) { free(pDag); return NULL; } cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans"); - int level = cJSON_GetArraySize(pLevels); - for(int i = 0; i < level; i++) { + int level = cJSON_GetArraySize(pLevels); + for (int i = 0; i < level; i++) { SArray* plansOneLevel = taosArrayInit(0, sizeof(void*)); - if(plansOneLevel == NULL) { - for(int j = 0; j < i; j++) { + if (plansOneLevel == NULL) { + for (int j = 0; j < i; j++) { taosArrayDestroy(taosArrayGetP(pDag->pSubplans, j)); } taosArrayDestroy(pDag->pSubplans); @@ -1206,13 +1223,13 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) { return NULL; } cJSON* pItem = cJSON_GetArrayItem(pLevels, i); - int sz = cJSON_GetArraySize(pItem); - for(int j = 0; j < sz; j++) { - cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j); + int sz = cJSON_GetArraySize(pItem); + for (int j = 0; j < sz; j++) { + cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j); SSubplan* pSubplan = subplanFromJson(pSubplanJson); taosArrayPush(plansOneLevel, &pSubplan); } - taosArrayPush(pDag->pSubplans, plansOneLevel); + taosArrayPush(pDag->pSubplans, &plansOneLevel); } return pDag; } From d958655dd3221c14356b26cae1c3b8e3e6826186 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 20:22:31 +0800 Subject: [PATCH 14/39] sync refactor --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncUtil.h | 1 + source/libs/sync/src/syncMain.c | 7 + source/libs/sync/src/syncUtil.c | 7 + source/libs/sync/src/syncVoteMgr.c | 14 +- source/libs/sync/test/syncInitTest.cpp | 71 ++++++---- source/libs/sync/test/syncPingTest.cpp | 3 +- .../libs/sync/test/syncVotesGrantedTest.cpp | 130 ++++++++++++++---- 8 files changed, 182 insertions(+), 52 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 874763cd45..447b75a5e8 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -193,6 +193,7 @@ typedef struct SSyncNode { SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 6cb1105b2c..f2c979da99 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -47,6 +47,7 @@ int32_t syncUtilElectRandomMS(); int32_t syncUtilQuorum(int32_t replicaNum); cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); +char* syncUtilRaftId2Str(const SRaftId* p); const char* syncUtilState2String(ESyncState state); #ifdef __cplusplus diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ad424b9353..3b8d716dbe 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -291,6 +291,13 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { return pJson; } +char* syncNode2Str(const SSyncNode* pSyncNode) { + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index d2b413bdaa..3fb430a714 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -131,6 +131,13 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { return pJson; } +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + const char* syncUtilState2String(ESyncState state) { if (state == TAOS_SYNC_STATE_FOLLOWER) { return "TAOS_SYNC_STATE_FOLLOWER"; diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 407b7b1941..3b6105ea0a 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -63,6 +63,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { } } assert(j != -1); + assert(j >= 0 && j < pVotesGranted->replicaNum); if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); @@ -87,6 +88,14 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { for (int i = 0; i < pVotesGranted->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i]))); } + int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); cJSON_AddStringToObject(pRoot, "term", u64buf); @@ -95,6 +104,9 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); return pJson; @@ -102,7 +114,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 602297b2a6..669c4e68a5 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -1,8 +1,10 @@ #include #include +#include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,13 +15,16 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[3] = {7010, 7110, 7210}; +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; -SSyncNode* syncInitTest() { - SSyncFSM* pFsm; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; - SSyncInfo syncInfo; - syncInfo.vgId = 1; +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; @@ -29,46 +34,66 @@ SSyncNode* syncInitTest() { snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = 0; - pCfg->replicaNum = 3; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; - pCfg->nodeInfo[0].nodePort = ports[0]; - snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = ports[1]; - snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = ports[2]; - snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -int main() { +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[0]); + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - cJSON* pJson = syncNode2Json(pSyncNode); - char* serialized = cJSON_Print(pJson); + char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); free(serialized); - cJSON_Delete(pJson); + + initRaftId(pSyncNode); return 0; } diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 4e5f7b78b5..450e097cc8 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -25,7 +25,8 @@ SSyncNode* doSync(int myIndex) { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 97b807736b..3edde509f8 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -4,6 +4,7 @@ #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "syncUtil.h" #include "syncVoteMgr.h" void logTest() { @@ -15,66 +16,141 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[3] = {7010, 7110, 7210}; +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; -SSyncNode* doSync(int myIndex) { - SSyncFSM* pFsm; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; - SSyncInfo syncInfo; - syncInfo.vgId = 1; +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; - pCfg->replicaNum = 3; + pCfg->replicaNum = replicaNum; - pCfg->nodeInfo[0].nodePort = ports[0]; - snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } - pCfg->nodeInfo[1].nodePort = ports[1]; - snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = ports[2]; - snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); - - SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -int main() { +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - int myIndex = 0; + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(myIndex); - SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); - assert(pVotesGranted != NULL); + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); - char* serialized = voteGranted2Str(pVotesGranted); - assert(serialized != NULL); + char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); free(serialized); + initRaftId(pSyncNode); + + SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pVotesGranted != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + SyncTerm term = 1234; + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, term); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + for (int i = 0; i < replicaNum; ++i) { + SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); + reply->destId = pSyncNode->myRaftId; + reply->srcId = ids[i]; + reply->term = term; + reply->voteGranted = true; + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + } + + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, 123456789); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedDestroy(pVotesGranted); return 0; } From 3fb4b86248f284308d438004c368feec49c2de93 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 8 Mar 2022 20:43:12 +0800 Subject: [PATCH 15/39] sync refactor --- source/libs/sync/src/syncVoteMgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 3b6105ea0a..893a4e3a55 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -86,7 +86,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { cJSON *pReplicas = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i]))); + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); } int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); for (int i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -114,7 +114,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } From 43e4cdc7132c940c3bad738fe136209cb31988e3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 8 Mar 2022 22:29:52 +0800 Subject: [PATCH 16/39] [TD-13760]: redefine socket api. --- include/os/osSocket.h | 85 +-- source/libs/transport/src/rpcTcp.c | 120 ++-- source/libs/transport/src/rpcUdp.c | 45 +- source/libs/transport/src/thttp.c | 17 +- source/os/src/osSocket.c | 904 +++++++++++++++++------------ 5 files changed, 654 insertions(+), 517 deletions(-) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index da1a9651e9..6544c89548 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -21,7 +21,10 @@ #define socket SOCKET_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID #define listen LISTEN_FUNC_TAOS_FORBID - // #define accept ACCEPT_FUNC_TAOS_FORBID + #define accept ACCEPT_FUNC_TAOS_FORBID + #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID + #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID + #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID #endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -38,31 +41,6 @@ extern "C" { #endif -#define TAOS_EPOLL_WAIT_TIME 500 -typedef int32_t SOCKET; -typedef SOCKET EpollFd; -#define EpollClose(pollFd) taosCloseSocket(pollFd) - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef SOCKET SocketFd; -#else -typedef int32_t SocketFd; -#endif - -int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); -int32_t taosWriteSocket(SocketFd fd, void *msg, int len); -int32_t taosReadSocket(SocketFd fd, void *msg, int len); -int32_t taosCloseSocketNoCheck(SocketFd fd); -int32_t taosCloseSocket(SocketFd fd); -void taosShutDownSocketRD(SOCKET fd); -void taosShutDownSocketWR(SOCKET fd); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); - -uint32_t taosInetAddr(const char *ipAddr); -const char *taosInetNtoa(struct in_addr ipInt); - #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #define htobe64 htonll #if defined(_TD_GO_DLL_) @@ -74,25 +52,54 @@ const char *taosInetNtoa(struct in_addr ipInt); #define htobe64 htonll #endif -int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); -int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); +#define TAOS_EPOLL_WAIT_TIME 500 -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(SOCKET sockFd); +typedef struct TdSocketServer *TdSocketServerPtr; +typedef struct TdSocket *TdSocketPtr; +typedef struct TdEpoll *TdEpollPtr; -void taosBlockSIGPIPE(); +int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); +int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); +int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosCloseSocket(TdSocketPtr *ppSocket); +int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); +int32_t taosShutDownSocketRD(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer); +int32_t taosShutDownSocketWR(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer); +int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket); +int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer); +int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on); +int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen); +int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); +int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); +int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); + +TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(TdSocketPtr pSocket); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen); + +int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen); + +void taosBlockSIGPIPE(); uint32_t taosGetIpv4FromFqdn(const char *); int32_t taosGetFqdn(char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); -void taosIgnSIGPIPE(); -void taosSetMaskSIGPIPE(); +void taosIgnSIGPIPE(); +void taosSetMaskSIGPIPE(); +uint32_t taosInetAddr(const char *ipAddr); +const char *taosInetNtoa(struct in_addr ipInt); + +TdEpollPtr taosCreateEpoll(int32_t size); +int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event); +int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout); +int32_t taosCloseEpoll(TdEpollPtr *ppEpoll); #ifdef __cplusplus } diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d95ac3d36d..aac38b21e8 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -24,7 +24,7 @@ #ifndef USE_UV typedef struct SFdObj { void * signature; - SOCKET fd; // TCP socket FD + TdSocketPtr pSocket; // TCP socket FD void * thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; @@ -40,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - EpollFd pollFd; + TdEpollPtr pEpoll; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -56,20 +56,20 @@ typedef struct { } SClientObj; typedef struct { - SOCKET fd; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; - SThreadObj **pThreadObj; - pthread_t thread; + TdSocketServerPtr pSocketServer; + uint32_t ip; + uint16_t port; + int8_t stop; + int8_t reserve; + char label[TSDB_LABEL_LEN]; + int numOfThreads; + void * shandle; + SThreadObj **pThreadObj; + pthread_t thread; } SServerObj; static void * taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void * taosAcceptTcpConnection(void *arg); @@ -85,7 +85,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } - pServerObj->fd = -1; + pServerObj->pSocketServer = NULL; taosResetPthread(&pServerObj->thread); pServerObj->ip = ip; pServerObj->port = port; @@ -118,7 +118,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pServerObj->pThreadObj[i] = pThreadObj; - pThreadObj->pollFd = -1; + pThreadObj->pEpoll = NULL; taosResetPthread(&pThreadObj->thread); pThreadObj->processData = fp; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); @@ -135,8 +135,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { + pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter + if (pThreadObj->pEpoll == NULL) { tError("%s failed to create TCP epoll", label); code = -1; break; @@ -151,8 +151,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pThreadObj->threadId = i; } - pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->fd < 0) code = -1; + pServerObj->pSocketServer = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + if (pServerObj->pSocketServer == NULL) code = -1; if (code == 0) { code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); @@ -196,8 +196,8 @@ void taosStopTcpServer(void *handle) { if (pServerObj == NULL) return; pServerObj->stop = 1; - if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + if (pServerObj->pSocketServer != NULL) { + taosShutDownSocketServerRD(pServerObj->pSocketServer); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) { } static void *taosAcceptTcpConnection(void *arg) { - SOCKET connFd = -1; + TdSocketPtr pSocket = NULL; struct sockaddr_in caddr; int threadId = 0; SThreadObj * pThreadObj; @@ -239,13 +239,13 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); - connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + pSocket = taosAcceptTcpConnectSocket(pServerObj->pSocketServer, (struct sockaddr *)&caddr, &addrlen); if (pServerObj->stop) { tDebug("%s TCP server stop accepting new connections", pServerObj->label); break; } - if (connFd == -1) { + if (pSocket == NULL) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); break; @@ -255,11 +255,11 @@ static void *taosAcceptTcpConnection(void *arg) { continue; } - taosKeepTcpAlive(connFd); + taosKeepTcpAlive(pSocket); struct timeval to = {5, 0}; - int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + int32_t ret = taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); continue; @@ -268,14 +268,14 @@ static void *taosAcceptTcpConnection(void *arg) { // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + tDebug("%s new TCP connection from %s:%hu, FD:%p numOfFds:%d", pServerObj->label, + taosInetNtoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); } else { - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); } @@ -285,7 +285,7 @@ static void *taosAcceptTcpConnection(void *arg) { threadId = threadId % pServerObj->numOfThreads; } - taosCloseSocket(pServerObj->fd); + taosCloseSocketServer(&pServerObj->pSocketServer); return NULL; } @@ -339,8 +339,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { + pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter + if (pThreadObj->pEpoll == NULL) { tError("%s failed to create TCP epoll", label); code = -1; break; @@ -388,21 +388,17 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; - SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd == (SOCKET)-1) return NULL; -#else - if (fd <= 0) return NULL; -#endif + TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); + if (pSocket == NULL) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + if (taosGetSocketName(pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { localPort = (uint16_t)ntohs(sin.sin_port); } - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); if (pFdObj) { pFdObj->thandle = thandle; @@ -415,7 +411,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin ipport, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); - taosCloseSocket(fd); + taosCloseSocket(&pSocket); } return pFdObj; @@ -430,7 +426,7 @@ void taosCloseTcpConnection(void *chandle) { // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; - taosShutDownSocketWR(pFdObj->fd); + taosShutDownSocketWR(pFdObj->pSocket); } int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { @@ -438,8 +434,8 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; SThreadObj *pThreadObj = pFdObj->pThreadObj; - int ret = taosWriteMsg(pFdObj->fd, data, len); - tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + int ret = taosWriteMsg(pFdObj->pSocket, data, len); + tTrace("%s %p TCP data is sent, FD:%p bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, ret); return ret; } @@ -449,7 +445,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { // notify the upper layer, so it will clean the associated context if (pFdObj->closedByApp == 0) { - taosShutDownSocketWR(pFdObj->fd); + taosShutDownSocketWR(pFdObj->pSocket); SRecvInfo recvInfo; recvInfo.msg = NULL; @@ -473,7 +469,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { SThreadObj *pThreadObj = pFdObj->pThreadObj; - headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); + headLen = taosReadMsg(pFdObj->pSocket, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); return -1; @@ -486,13 +482,12 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, - buffer); + tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); } msg = buffer + tsRpcOverhead; leftLen = msgLen - headLen; - retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); + retLen = taosReadMsg(pFdObj->pSocket, msg + headLen, leftLen); if (leftLen != retLen) { tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); @@ -532,7 +527,7 @@ static void *taosProcessTcpData(void *param) { setThreadName(name); while (1) { - int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); + int fdNum = taosWaitEpoll(pThreadObj->pEpoll, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label); break; @@ -561,7 +556,7 @@ static void *taosProcessTcpData(void *param) { } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - shutdown(pFdObj->fd, SHUT_WR); + taosShutDownSocketWR(pFdObj->pSocket); continue; } @@ -572,9 +567,9 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >= 0) { - EpollClose(pThreadObj->pollFd); - pThreadObj->pollFd = -1; + if (pThreadObj->pEpoll != NULL) { + taosCloseEpoll(&pThreadObj->pEpoll); + pThreadObj->pEpoll = NULL; } while (pThreadObj->pHead) { @@ -590,7 +585,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); @@ -599,13 +594,13 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { } pFdObj->closedByApp = 0; - pFdObj->fd = fd; + pFdObj->pSocket = pSocket; pFdObj->pThreadObj = pThreadObj; pFdObj->signature = pFdObj; event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -635,8 +630,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { } pFdObj->signature = NULL; - epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - taosCloseSocket(pFdObj->fd); + taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_DEL, pFdObj->pSocket, NULL); + taosCloseSocket(&pFdObj->pSocket); pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) @@ -655,8 +650,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, - pFdObj->fd, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); tfree(pFdObj); } diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 3640414a4c..81c8d0af76 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -30,17 +30,17 @@ #define RPC_MAX_UDP_SIZE 65480 typedef struct { - int index; - SOCKET fd; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; - void * hash; - void * shandle; // handle passed by upper layer during server initialization - void * pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data + int index; + TdSocketPtr pSocket; + uint16_t port; // peer port + uint16_t localPort; // local port + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; + pthread_t thread; + void *hash; + void *shandle; // handle passed by upper layer during server initialization + void *pSet; + void *(*processData)(SRecvInfo *pRecv); + char *buffer; // buffer to receive data } SUdpConn; typedef struct { @@ -86,8 +86,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads for (i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; ownPort = (port ? port + i : 0); - pConn->fd = taosOpenUdpSocket(ip, ownPort); - if (pConn->fd < 0) { + pConn->pSocket = taosOpenUdpSocket(ip, ownPort); + if (pConn->pSocket == NULL) { tError("%s failed to open UDP socket %x:%hu", label, ip, port); break; } @@ -100,7 +100,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads struct sockaddr_in sin; unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && + if (taosGetSocketName(pConn->pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); } @@ -138,9 +138,9 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR); - if (pConn->fd >= 0) taosCloseSocket(pConn->fd); - pConn->fd = -1; + if (pConn->pSocket != NULL) taosShutDownSocketRDWR(pConn->pSocket); + if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); + pConn->pSocket = NULL; } for (int i = 0; i < pSet->threads; ++i) { @@ -163,7 +163,7 @@ void taosCleanUpUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >= 0) taosCloseSocket(pConn->fd); + if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); } tDebug("%s UDP is cleaned up", pSet->label); @@ -199,13 +199,12 @@ static void *taosRecvUdpData(void *param) { setThreadName("recvUdpData"); while (1) { - dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); + dataLen = taosReadFromSocket(pConn->pSocket, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); if (dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, - pConn->fd); + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d", pConn->label, strerror(errno), (int32_t)dataLen); // for windows usage, remote shutdown also returns - 1 in windows client - if (pConn->fd == -1) { + if (pConn->pSocket == NULL) { break; } else { continue; @@ -255,7 +254,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c destAdd.sin_addr.s_addr = ip; destAdd.sin_port = htons(port); - int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); + int ret = taosSendto(pConn->pSocket, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); return ret; } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 95e67290ac..6d1c691b9b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -173,7 +173,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 #else int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { int32_t code = -1; - SOCKET fd = 0; + TdSocketPtr pSocket = NULL; uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { @@ -182,8 +182,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 goto SEND_OVER; } - fd = taosOpenTcpClientSocket(ip, port, 0); - if (fd < 0) { + pSocket = taosOpenTcpClientSocket(ip, port, 0); + if (pSocket == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create http socket to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; @@ -200,21 +200,20 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 char header[1024] = {0}; int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); - - if (taosWriteSocket(fd, header, headLen) < 0) { + if (taosWriteMsg(pSocket, header, headLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to send http header to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } - if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + if (taosWriteMsg(pSocket, (void*)pCont, contLen) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to send http content to %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } // read something to avoid nginx error 499 - if (taosReadSocket(fd, header, 10) < 0) { + if (taosWriteMsg(pSocket, header, 10) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to receive response from %s:%u since %s", server, port, terrstr()); goto SEND_OVER; @@ -223,8 +222,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 code = 0; SEND_OVER: - if (fd != 0) { - taosCloseSocket(fd); + if (pSocket != NULL) { + taosCloseSocket(&pSocket); } return code; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 096516a98d..9330cb7b32 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -18,176 +18,206 @@ #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #include "winsock2.h" - #include - #include - #include -#else - #include - #include - #include - #include - #include - #include - #include - #include - #include -#endif - -// typedef struct TdSocketServer { -// #if SOCKET_WITH_LOCK -// pthread_rwlock_t rwlock; -// #endif -// int refId; -// SocketFd fd; -// } * TdSocketServerPtr, TdSocketServer; - -// typedef struct TdSocketConnector { -// #if SOCKET_WITH_LOCK -// pthread_rwlock_t rwlock; -// #endif -// int refId; -// SocketFd fd; -// } * TdSocketConnectorPtr, TdSocketConnector; - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -#define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags) -int32_t taosSendto(SocketFd fd, void *buf, int len, unsigned int flags, const struct sockaddr *to, int tolen) { - return sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen); -} -int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { return send((SOCKET)fd, buf, len, 0); } -int32_t taosReadSocket(SocketFd fd, void *buf, int len) { return recv((SOCKET)fd, buf, len, 0)(); } -int32_t taosCloseSocketNoCheck(SocketFd fd) { return closesocket((SOCKET)fd); } -int32_t taosCloseSocket(SocketFd fd) { closesocket((SOCKET)fd) } - -#else - - #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) - int32_t taosSendto(SocketFd fd, void * buf, int len, unsigned int flags, const struct sockaddr * dest_addr, int addrlen) { - return sendto(fd, buf, len, flags, dest_addr, addrlen); - } - - int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { - return write(fd, buf, len); - } - - int32_t taosReadSocket(SocketFd fd, void *buf, int len) { - return read(fd, buf, len); - } - - int32_t taosCloseSocketNoCheck(SocketFd fd) { - return close(fd); - } - - int32_t taosCloseSocket(SocketFd fd) { - if (fd > -1) { - close(fd); - } - } -#endif - -void taosShutDownSocketRD(SOCKET fd) { -#ifdef WINDOWS - closesocket(fd); -#elif __APPLE__ - close(fd); -#else - shutdown(fd, SHUT_RD); -#endif -} - -void taosShutDownSocketWR(SOCKET fd) { -#ifdef WINDOWS - closesocket(fd); -#elif __APPLE__ - close(fd); -#else - shutdown(fd, SHUT_WR); -#endif -} - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { - int32_t flags = 0; - if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { - //printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - if (on) - flags |= O_NONBLOCK; - else - flags &= ~O_NONBLOCK; - - if ((flags = fcntl(sock, F_SETFL, flags)) < 0) { - //printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - return 0; -} - - - - -#endif - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_32)) - -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} - -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen) { - return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen); -} - -#endif - -#if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) - -uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } - -const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } - -#endif - -#if defined(_TD_DARWIN_64) - -/* - * darwin implementation - */ - -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { - if (level == SOL_SOCKET && optname == SO_SNDBUF) { - return 0; - } - - if (level == SOL_SOCKET && optname == SO_RCVBUF) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -/* - * windows implementation - */ - #include #include +#include #include #include #include +#include #include #include +#include "winsock2.h" +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif -void taosWinSocketInit() { +typedef int32_t SocketFd; +typedef SocketFd EpollFd; + +typedef struct TdSocketServer { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketServerPtr, TdSocketServer; + +typedef struct TdSocket { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketPtr, TdSocket; + +typedef struct TdEpoll { +#if SOCKET_WITH_LOCK + pthread_rwlock_t rwlock; +#endif + int refId; + EpollFd fd; +} *TdEpollPtr, TdEpoll; + +int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr, + int addrlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); +#else + return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); +#endif +} +int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return send(pSocket->fd, buf, len, 0); + ; +#else + return write(pSocket->fd, buf, len); +#endif +} +int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return recv(pSocket->fd, buf, len, 0); + ; +#else + return read(pSocket->fd, buf, len); +#endif +} + +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); +} +int32_t taosCloseSocketNoCheck1(SocketFd fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return closesocket(fd); +#else + return close(fd); +#endif +} +int32_t taosCloseSocket(TdSocketPtr *ppSocket) { + int32_t code; + if (ppSocket == NULL || *ppSocket == NULL || (*ppSocket)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppSocket)->fd); + (*ppSocket)->fd = -1; + free(*ppSocket); + return code; +} +int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { + int32_t code; + if (ppSocketServer == NULL || *ppSocketServer == NULL || (*ppSocketServer)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); + (*ppSocketServer)->fd = -1; + free(*ppSocketServer); + return code; +} + +int32_t taosShutDownSocketRD(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_RD); +#endif +} +int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_RD); +#endif +} + +int32_t taosShutDownSocketWR(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_WR); +#endif +} +int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_WR); +#endif +} +int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocket->fd); +#elif __APPLE__ + return close(pSocket->fd); +#else + return shutdown(pSocket->fd, SHUT_RDWR); +#endif +} +int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { + if (pSocketServer == NULL || pSocketServer->fd < 0) { + return -1; + } +#ifdef WINDOWS + return closesocket(pSocketServer->fd); +#elif __APPLE__ + return close(pSocketServer->fd); +#else + return shutdown(pSocketServer->fd, SHUT_RDWR); +#endif +} + +#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) + #if defined(_TD_GO_DLL_) + uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } + #endif +#endif + +void taosWinSocketInit1() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) static char flag = 0; if (flag == 0) { WORD wVersionRequested; @@ -197,21 +227,46 @@ void taosWinSocketInit() { flag = 1; } } +#else +#endif } - -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { +int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) u_long mode; if (on) { mode = 1; - ioctlsocket(sock, FIONBIO, &mode); + ioctlsocket(pSocket->fd, FIONBIO, &mode); } else { mode = 0; - ioctlsocket(sock, FIONBIO, &mode); + ioctlsocket(pSocket->fd, FIONBIO, &mode); } +#else + int32_t flags = 0; + if ((flags = fcntl(pSocket->fd, F_GETFL, 0)) < 0) { + // printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); + return 1; + } + + if (on) + flags |= O_NONBLOCK; + else + flags &= ~O_NONBLOCK; + + if ((flags = fcntl(pSocket->fd, F_SETFL, flags)) < 0) { + // printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); + return 1; + } +#endif return 0; } - -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { +int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -228,13 +283,23 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op return 0; } - return setsockopt(socketfd, level, optname, optval, optlen); + return setsockopt(pSocket->fd, level, optname, optval, optlen); +#else + return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen); +#endif +} +int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen); +#endif } - -#ifdef _MSC_VER -//#if _MSC_VER >= 1900 - uint32_t taosInetAddr(const char *ipAddr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { @@ -242,39 +307,37 @@ uint32_t taosInetAddr(const char *ipAddr) { } else { return value; } +#else + return inet_addr(ipAddr); +#endif } - const char *taosInetNtoa(struct in_addr ipInt) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); +#else + return inet_ntoa(ipInt); +#endif } -//#endif -#endif - -#if defined(_TD_GO_DLL_) - -uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } - -#endif - -#endif - #ifndef SIGPIPE - #define SIGPIPE EPIPE +#define SIGPIPE EPIPE #endif #define TCP_CONN_TIMEOUT 3000 // conn timeout -int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t nleft, nwritten; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; while (nleft > 0) { - nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); + nwritten = taosWriteSocket(pSocket, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) continue; @@ -293,20 +356,21 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { +int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t nleft, nread; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; - if (fd < 0) return -1; - while (nleft > 0) { - nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); + nread = taosReadSocket(pSocket, ptr, (size_t)nleft); if (nread == 0) { break; } else if (nread < 0) { - if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { + if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK*/) { continue; } else { return -1; @@ -324,11 +388,14 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { return (nbytes - nleft); } -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { - taosSetNonblocking(fd, 1); +int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + taosSetNonblocking(pSocket, 1); - int32_t nleft, nwritten, nready; - fd_set fset; + int32_t nleft, nwritten, nready; + fd_set fset; struct timeval tv; nleft = nbytes; @@ -336,24 +403,24 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { tv.tv_sec = 30; tv.tv_usec = 0; FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + FD_SET(pSocket->fd, &fset); + if ((nready = select((SocketFd)(pSocket->fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; - //printf("fd %d timeout, no enough space to write", fd); + // printf("fd %d timeout, no enough space to write", fd); break; } else if (nready < 0) { if (errno == EINTR) continue; - //printf("select error, %d (%s)", errno, strerror(errno)); + // printf("select error, %d (%s)", errno, strerror(errno)); return -1; } - nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); + nwritten = (int32_t)send(pSocket->fd, ptr, (size_t)nleft, MSG_NOSIGNAL); if (nwritten <= 0) { if (errno == EAGAIN || errno == EINTR) continue; - //printf("write error, %d (%s)", errno, strerror(errno)); + // printf("write error, %d (%s)", errno, strerror(errno)); return -1; } @@ -361,121 +428,99 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { ptr += nwritten; } - taosSetNonblocking(fd, 0); + taosSetNonblocking(pSocket, 0); return (nbytes - nleft); } -int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { - int32_t nread, nready, nleft = nbytes; - - fd_set fset; - struct timeval tv; - - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - //printf("fd %d timeout\n", fd); - break; - } else if (nready < 0) { - if (errno == EINTR) continue; - //printf("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { - if (errno == EINTR) continue; - //printf("read error, %d (%s)", errno, strerror(errno)); - return -1; - - } else if (nread == 0) { - //printf("fd %d EOF", fd); - break; // EOF - } - - nleft -= nread; - ptr += nread; - } - - return (nbytes - nleft); -} - -SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { +TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; - int32_t bufSize = 1024000; + SocketFd fd; + int32_t bufSize = 1024000; - //printf("open udp socket:0x%x:%hu", ip, port); + // printf("open udp socket:0x%x:%hu", ip, port); memset((char *)&localAddr, 0, sizeof(localAddr)); localAddr.sin_family = AF_INET; localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { - //printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + // printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } /* bind socket to local address */ - if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { - //printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { + // printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); + taosCloseSocket(&pSocket); + return NULL; } - return sockFd; + return pSocket; } -SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; - int32_t ret; +TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + SocketFd fd = -1; + int32_t ret; struct sockaddr_in serverAddr, clientAddr; - int32_t bufSize = 1024 * 1024; + int32_t bufSize = 1024 * 1024; - sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - if (sockFd <= 2) { - //printf("failed to open the socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if (fd <= 2) { + // printf("failed to open the socket: %d (%s)", errno, strerror(errno)); + if (fd >= 0) taosCloseSocketNoCheck1(fd); + return NULL; } + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + /* set REUSEADDR option, so the portnumber can be re-used */ int32_t reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the send buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the send buffer size for TCP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - //printf("failed to set the receive buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + // printf("failed to set the receive buffer size for TCP socket\n"); + taosCloseSocket(&pSocket); + return NULL; } if (clientIp != 0) { @@ -485,11 +530,11 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie clientAddr.sin_port = 0; /* bind socket to client address */ - if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - //printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, - // strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { + // printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, + // strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } } @@ -498,159 +543,193 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie serverAddr.sin_addr.s_addr = destIp; serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); -#ifdef _TD_LINUX - taosSetNonblocking(sockFd, 1); - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#ifdef _TD_LINUX + taosSetNonblocking(pSocket, 1); + ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); if (ret == -1) { if (errno == EHOSTUNREACH) { - //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(&pSocket); + return -1; } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { - struct pollfd wfd[1]; + struct pollfd wfd[1]; - wfd[0].fd = sockFd; + wfd[0].fd = pSocket->fd; wfd[0].events = POLLOUT; - + int res = poll(wfd, 1, TCP_CONN_TIMEOUT); if (res == -1 || res == 0) { - //printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); - taosCloseSocket(sockFd); // + // printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); + taosCloseSocket(&pSocket); // return -1; } - int optVal = -1, optLen = sizeof(int); - if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { - //printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); - taosCloseSocket(sockFd); // + int optVal = -1, optLen = sizeof(int); + if ((0 != taosGetSockOpt(pSocket, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { + // printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); + taosCloseSocket(&pSocket); // return -1; } ret = 0; - } else { // Other error - //printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); - taosCloseSocket(sockFd); // - return -1; - } + } else { // Other error + // printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); + taosCloseSocket(&pSocket); // + return -1; + } } - taosSetNonblocking(sockFd, 0); + taosSetNonblocking(pSocket, 0); #else - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); + ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); #endif if (ret != 0) { - //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - sockFd = -1; + // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } else { - taosKeepTcpAlive(sockFd); + taosKeepTcpAlive(pSocket); } - return sockFd; + return pSocket; } -int32_t taosKeepTcpAlive(SOCKET sockFd) { +int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } int32_t alive = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { - //printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { + // printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } #ifndef __APPLE__ // all fails on macosx int32_t probes = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { - //printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { + // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } int32_t alivetime = 10; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { - //printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { + // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } int32_t interval = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { - //printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { + // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } -#endif // __APPLE__ +#endif // __APPLE__ int32_t nodelay = 1; - if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { - //printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { + // printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } struct linger linger = {0}; linger.l_onoff = 1; linger.l_linger = 3; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { - //printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { + // printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } return 0; } -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { +TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; + SocketFd fd; int32_t reuse; - //printf("open tcp server socket:0x%x:%hu", ip, port); + // printf("open tcp server socket:0x%x:%hu", ip, port); bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - //printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return NULL; } + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->refId = 0; + pSocket->fd = fd; + /* set REUSEADDR option, so the portnumber can be re-used */ reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } /* bind socket to server address */ - if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - //printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (taosKeepTcpAlive(sockFd) < 0) { - //printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (taosKeepTcpAlive(pSocket) < 0) { + // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - if (listen(sockFd, 1024) < 0) { - //printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; + if (listen(pSocket->fd, 1024) < 0) { + // printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; } - return sockFd; + return (TdSocketServerPtr)pSocket; } +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, + socklen_t *addrLen) { + if (pServerSocket == NULL || pServerSocket->fd < 0) { + return NULL; + } + SocketFd fd = accept(pServerSocket->fd, destAddr, addrLen); + if (fd == -1) { + // tError("TCP accept failure(%s)", strerror(errno)); + return NULL; + } + + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pSocket->fd = fd; + pSocket->refId = 0; + return pSocket; +} #define COPY_SIZE 32768 // sendfile shall be used -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { +int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len) { + if (pSrcSocket == NULL || pSrcSocket->fd < 0 || pDestSocket == NULL || pDestSocket->fd < 0) { + return -1; + } int64_t leftLen; int64_t readLen, writeLen; char temp[COPY_SIZE]; @@ -663,18 +742,18 @@ int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { else readLen = COPY_SIZE; // 4K - int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); + int64_t retLen = taosReadMsg(pSrcSocket, temp, (int32_t)readLen); if (readLen != retLen) { - //printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, retLen, len, leftLen, strerror(errno)); + // printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + // readLen, retLen, len, leftLen, strerror(errno)); return -1; } - writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen); + writeLen = taosWriteMsg(pDestSocket, temp, (int32_t)readLen); if (readLen != writeLen) { - //printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, writeLen, len, leftLen, strerror(errno)); + // printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + // readLen, writeLen, len, leftLen, strerror(errno)); return -1; } @@ -692,7 +771,7 @@ void taosBlockSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { - //printf("failed to block SIGPIPE"); + // printf("failed to block SIGPIPE"); } #endif } @@ -706,7 +785,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr * sa = result->ai_addr; + struct sockaddr *sa = result->ai_addr; struct sockaddr_in *si = (struct sockaddr_in *)sa; struct in_addr ia = si->sin_addr; uint32_t ip = ia.s_addr; @@ -715,12 +794,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); } else { - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); } #else - //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); #endif return 0xFFFFFFFF; } @@ -730,7 +809,7 @@ int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { - //printf("failed to get hostname, reason:%s", strerror(errno)); + // printf("failed to get hostname, reason:%s", strerror(errno)); return -1; } @@ -742,21 +821,21 @@ int32_t taosGetFqdn(char *fqdn) { // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return // immediately hints.ai_family = AF_INET; -#else // __APPLE__ +#else // __APPLE__ hints.ai_flags = AI_CANONNAME; -#endif // __APPLE__ +#endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - //printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + // printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } #ifdef __APPLE__ // refer to comments above strcpy(fqdn, hostname); -#else // __APPLE__ +#else // __APPLE__ strcpy(fqdn, result->ai_canonname); -#endif // __APPLE__ +#endif // __APPLE__ freeaddrinfo(result); return 0; } @@ -793,7 +872,6 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } - void taosIgnSIGPIPE() { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #else @@ -809,7 +887,67 @@ void taosSetMaskSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { - //printf("failed to setmask SIGPIPE"); + // printf("failed to setmask SIGPIPE"); } #endif -} \ No newline at end of file +} + +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) { + if (pSocket == NULL || pSocket->fd < 0) { + return -1; + } + return getsockname(pSocket->fd, destAddr, addrLen); +} + + +TdEpollPtr taosCreateEpoll(int32_t size) { + EpollFd fd = -1; +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + fd = epoll_create(size); +#endif + if (fd < 0) { + return NULL; + } + + TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll)); + if (pEpoll == NULL) { + taosCloseSocketNoCheck1(fd); + return NULL; + } + pEpoll->fd = fd; + pEpoll->refId = 0; + return pEpoll; +} +int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event) { + int32_t code = -1; + if (pEpoll == NULL || pEpoll->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); +#endif + return code; +} +int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout) { + int32_t code = -1; + if (pEpoll == NULL || pEpoll->fd < 0) { + return -1; + } +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else + code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); +#endif + return code; +} +int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { + int32_t code; + if (ppEpoll == NULL || *ppEpoll == NULL || (*ppEpoll)->fd < 0) { + return -1; + } + code = taosCloseSocketNoCheck1((*ppEpoll)->fd); + (*ppEpoll)->fd = -1; + free(*ppEpoll); + return code; +} From 3b4149ba3268a9460f3c8fcfd8f52cacf929bd6a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Mar 2022 10:15:40 +0800 Subject: [PATCH 17/39] sync refactor --- source/libs/sync/inc/syncVoteMgr.h | 6 +- source/libs/sync/src/syncVoteMgr.c | 51 +++++- source/libs/sync/test/CMakeLists.txt | 14 ++ .../libs/sync/test/syncVotesRespondTest.cpp | 156 ++++++++++++++++++ 4 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 source/libs/sync/test/syncVotesRespondTest.cpp diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index a769bfbccd..ae9cfe8d01 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -31,8 +31,8 @@ extern "C" { // SVotesGranted ----------------------------- typedef struct SVotesGranted { SRaftId (*replicas)[TSDB_MAX_REPLICA]; - int32_t replicaNum; bool isGranted[TSDB_MAX_REPLICA]; + int32_t replicaNum; int32_t votes; SyncTerm term; int32_t quorum; @@ -61,7 +61,9 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode); void votesRespondDestory(SVotesRespond *pVotesRespond); bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); -void Reset(SVotesRespond *pVotesRespond, SyncTerm term); +void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); +cJSON * votesRespond2Json(SVotesRespond *pVotesRespond); +char * votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 893a4e3a55..a2f10ce339 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -153,8 +153,8 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) { assert(pVotesRespond->term == pMsg->term); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - if (syncUtilSameId(&(*pVotesRespond->replicas)[i], &pMsg->srcId)) { - assert(pVotesRespond->isRespond[i] == false); + if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { + // assert(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } @@ -162,9 +162,52 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p assert(0); } -void Reset(SVotesRespond *pVotesRespond, SyncTerm term) { +void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; + memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); + /* + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + pVotesRespond->isRespond[i] = false; + } + */ +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; } \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 18b7748105..8172a42a80 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(syncIndexTest "") add_executable(syncInitTest "") add_executable(syncUtilTest "") add_executable(syncVotesGrantedTest "") +add_executable(syncVotesRespondTest "") target_sources(syncTest @@ -75,6 +76,10 @@ target_sources(syncVotesGrantedTest PRIVATE "syncVotesGrantedTest.cpp" ) +target_sources(syncVotesRespondTest + PRIVATE + "syncVotesRespondTest.cpp" +) target_include_directories(syncTest @@ -152,6 +157,11 @@ target_include_directories(syncVotesGrantedTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncVotesRespondTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -214,6 +224,10 @@ target_link_libraries(syncVotesGrantedTest sync gtest_main ) +target_link_libraries(syncVotesRespondTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp new file mode 100644 index 0000000000..74d42cd531 --- /dev/null +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + SVotesRespond* pVotesRespond = votesRespondCreate(pSyncNode); + assert(pVotesRespond != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + SyncTerm term = 1234; + printf("---------------------------------------\n"); + votesRespondReset(pVotesRespond, term); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + for (int i = 0; i < replicaNum; ++i) { + SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); + reply->destId = pSyncNode->myRaftId; + reply->srcId = ids[i]; + reply->term = term; + reply->voteGranted = true; + + votesRespondAdd(pVotesRespond, reply); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + votesRespondAdd(pVotesRespond, reply); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + } + + printf("---------------------------------------\n"); + votesRespondReset(pVotesRespond, 123456789); + { + char* serialized = votesRespond2Str(pVotesRespond); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + votesRespondDestory(pVotesRespond); + return 0; +} From 6667446db3979a5c7930f0deaf40936a6b992f93 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 9 Mar 2022 10:46:31 +0800 Subject: [PATCH 18/39] [TD-13876]: forbid big-end machine. --- include/os/osSysinfo.h | 1 + source/dnode/mgmt/daemon/src/dmnMain.c | 5 +++++ source/os/src/osSysinfo.c | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 1ebad370b5..54a3cfef7b 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -33,6 +33,7 @@ typedef struct { SDiskSize size; } SDiskSpace; +bool taosCheckSystemIsSmallEnd(); void taosGetSystemInfo(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c index df705898ca..7ba272453c 100644 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -97,6 +97,11 @@ int32_t dmnRunDnode() { } int main(int argc, char const *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + uError("TDengine does not run on non-small-end machines."); + return -1; + } + if (dmnParseOption(argc, argv) != 0) { return -1; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ff9d5fb71b..02d7e6c0e9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,6 +16,15 @@ #define _DEFAULT_SOURCE #include "os.h" +bool taosCheckSystemIsSmallEnd() { + union check{ + int16_t i; + char ch[2]; + }c; + c.i=1; + return c.ch[0]==1; +} + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* From d3b1d5d8f1063928d958cff873bd639748eca668 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Mar 2022 10:58:22 +0800 Subject: [PATCH 19/39] sync refactor --- source/libs/sync/inc/syncIndexMgr.h | 49 +++++++ source/libs/sync/src/syncIndexMgr.c | 99 +++++++++++++ source/libs/sync/test/CMakeLists.txt | 14 ++ source/libs/sync/test/syncIndexMgrTest.cpp | 156 +++++++++++++++++++++ 4 files changed, 318 insertions(+) create mode 100644 source/libs/sync/inc/syncIndexMgr.h create mode 100644 source/libs/sync/src/syncIndexMgr.c create mode 100644 source/libs/sync/test/syncIndexMgrTest.cpp diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h new file mode 100644 index 0000000000..7116ae9d46 --- /dev/null +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_INDEX_MGR_H +#define _TD_LIBS_SYNC_INDEX_MGR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" + +// SIndexMgr ----------------------------- +typedef struct SSyncIndexMgr { + SRaftId (*replicas)[TSDB_MAX_REPLICA]; + SyncIndex index[TSDB_MAX_REPLICA]; + int32_t replicaNum; + SSyncNode *pSyncNode; +} SSyncIndexMgr; + +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode); +void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); +void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); +void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); +cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_INDEX_MGR_H*/ diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c new file mode 100644 index 0000000000..86b719ba1e --- /dev/null +++ b/source/libs/sync/src/syncIndexMgr.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncIndexMgr.h" +#include "syncUtil.h" + +// SMatchIndex ----------------------------- + +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { + SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr)); + assert(pSyncIndexMgr != NULL); + memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); + + pSyncIndexMgr->replicas = &(pSyncNode->replicasId); + pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; + pSyncIndexMgr->pSyncNode = pSyncNode; + syncIndexMgrClear(pSyncIndexMgr); + + return pSyncIndexMgr; +} + +void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { + if (pSyncIndexMgr != NULL) { + free(pSyncIndexMgr); + } +} + +void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { + memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); + /* + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + pSyncIndexMgr->index[i] = 0; + } + */ +} + +void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->index)[i] = index; + return; + } + } + assert(0); +} + +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + SyncIndex idx = (pSyncIndexMgr->index)[i]; + return idx; + } + } + assert(0); +} + +cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); + } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->index[i]; + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + free(arr); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "pSyncIndexMgr", pRoot); + return pJson; +} + +char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { + cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 8172a42a80..5a5186c7e2 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -14,6 +14,7 @@ add_executable(syncInitTest "") add_executable(syncUtilTest "") add_executable(syncVotesGrantedTest "") add_executable(syncVotesRespondTest "") +add_executable(syncIndexMgrTest "") target_sources(syncTest @@ -80,6 +81,10 @@ target_sources(syncVotesRespondTest PRIVATE "syncVotesRespondTest.cpp" ) +target_sources(syncIndexMgrTest + PRIVATE + "syncIndexMgrTest.cpp" +) target_include_directories(syncTest @@ -162,6 +167,11 @@ target_include_directories(syncVotesRespondTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncIndexMgrTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -228,6 +238,10 @@ target_link_libraries(syncVotesRespondTest sync gtest_main ) +target_link_libraries(syncIndexMgrTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp new file mode 100644 index 0000000000..3edde509f8 --- /dev/null +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); + snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + char* serialized = syncNode2Str(pSyncNode); + printf("%s\n", serialized); + free(serialized); + + initRaftId(pSyncNode); + + SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); + assert(pVotesGranted != NULL); + + printf("---------------------------------------\n"); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + SyncTerm term = 1234; + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, term); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + for (int i = 0; i < replicaNum; ++i) { + SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); + reply->destId = pSyncNode->myRaftId; + reply->srcId = ids[i]; + reply->term = term; + reply->voteGranted = true; + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedVote(pVotesGranted, reply); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + } + + printf("---------------------------------------\n"); + voteGrantedReset(pVotesGranted, 123456789); + { + char* serialized = voteGranted2Str(pVotesGranted); + assert(serialized != NULL); + printf("%s\n", serialized); + free(serialized); + } + + voteGrantedDestroy(pVotesGranted); + return 0; +} From 2fcc970d79727ba9ac3d06e4ff1fc67e771d960e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Mar 2022 11:27:22 +0800 Subject: [PATCH 20/39] sync refactor --- source/libs/sync/inc/syncInt.h | 7 +++- source/libs/sync/src/syncIndexMgr.c | 5 ++- source/libs/sync/test/syncIndexMgrTest.cpp | 49 ++++++++-------------- 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 447b75a5e8..8b77e292c4 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -103,6 +103,9 @@ typedef struct SVotesGranted SVotesGranted; struct SVotesRespond; typedef struct SVotesRespond SVotesRespond; +struct SSyncIndexMgr; +typedef struct SSyncIndexMgr SSyncIndexMgr; + typedef struct SRaftId { SyncNodeId addr; // typedef uint64_t SyncNodeId; SyncGroupId vgId; // typedef int32_t SyncGroupId; @@ -148,8 +151,8 @@ typedef struct SSyncNode { SVotesRespond* pVotesRespond; // tla+ leader vars - SHashObj* pNextIndex; - SHashObj* pMatchIndex; + SSyncIndexMgr* pNextIndex; + SSyncIndexMgr* pMatchIndex; // tla+ log vars SSyncLogStore* pLogStore; diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 86b719ba1e..fff54638e2 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -81,8 +81,9 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { arr[i] = pSyncIndexMgr->index[i]; } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); free(arr); + cJSON_AddItemToObject(pRoot, "index", pIndex); snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); @@ -93,7 +94,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } \ No newline at end of file diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 3edde509f8..4e4cd9222b 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -1,4 +1,5 @@ -#include +#include "syncIndexMgr.h" +//#include #include #include "syncEnv.h" #include "syncIO.h" @@ -97,60 +98,44 @@ int main(int argc, char** argv) { initRaftId(pSyncNode); - SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode); - assert(pVotesGranted != NULL); + SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode); + assert(pSyncIndexMgr != NULL); printf("---------------------------------------\n"); { - char* serialized = voteGranted2Str(pVotesGranted); + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); free(serialized); } - SyncTerm term = 1234; + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); + syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); + printf("---------------------------------------\n"); - voteGrantedReset(pVotesGranted, term); { - char* serialized = voteGranted2Str(pVotesGranted); + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); free(serialized); } - for (int i = 0; i < replicaNum; ++i) { - SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild(); - reply->destId = pSyncNode->myRaftId; - reply->srcId = ids[i]; - reply->term = term; - reply->voteGranted = true; - - voteGrantedVote(pVotesGranted, reply); - { - char* serialized = voteGranted2Str(pVotesGranted); - assert(serialized != NULL); - printf("%s\n", serialized); - free(serialized); - } - - voteGrantedVote(pVotesGranted, reply); - { - char* serialized = voteGranted2Str(pVotesGranted); - assert(serialized != NULL); - printf("%s\n", serialized); - free(serialized); - } + printf("---------------------------------------\n"); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); + printf("index %d : %lu \n", i, idx); } + syncIndexMgrClear(pSyncIndexMgr); printf("---------------------------------------\n"); - voteGrantedReset(pVotesGranted, 123456789); { - char* serialized = voteGranted2Str(pVotesGranted); + char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); free(serialized); } - voteGrantedDestroy(pVotesGranted); + syncIndexMgrDestroy(pSyncIndexMgr); return 0; } From 5afb3e2c00804cbdbdb7fb352ba384f8e2b0c3f5 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 9 Mar 2022 15:47:29 +0800 Subject: [PATCH 21/39] [TD-13761]: redefine dir api. --- include/os/osDir.h | 22 +++++++++ source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- source/libs/index/src/index_tfile.c | 12 ++--- source/libs/tfs/inc/tfsInt.h | 2 +- source/libs/tfs/src/tfs.c | 67 +++++++++++++------------- source/libs/wal/src/walMeta.c | 24 ++++----- source/os/src/osDir.c | 48 ++++++++++++++++++ source/os/src/osSemaphore.c | 56 ++++++++++----------- source/os/src/osTimer.c | 8 +-- tests/test/c/tmqDemo.c | 17 ++++--- 10 files changed, 165 insertions(+), 93 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index 223c603352..6cf28fb878 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -16,10 +16,24 @@ #ifndef _TD_OS_DIR_H_ #define _TD_OS_DIR_H_ +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define opendir OPENDIR_FUNC_TAOS_FORBID + #define readdir READDIR_FUNC_TAOS_FORBID + #define closedir CLOSEDIR_FUNC_TAOS_FORBID + #define dirname DIRNAME_FUNC_TAOS_FORBID + #undef basename + #define basename BASENAME_FUNC_TAOS_FORBID +#endif + #ifdef __cplusplus extern "C" { #endif +typedef struct TdDir *TdDirPtr; +typedef struct TdDirEntry *TdDirEntryPtr; + + void taosRemoveDir(const char *dirname); bool taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); @@ -27,6 +41,14 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); bool taosIsDir(const char *dirname); +char* taosDirName(char *dirname); +char* taosDirEntryBaseName(char *dirname); + +TdDirPtr taosOpenDir(const char *dirname); +TdDirEntryPtr taosReadDir(TdDirPtr pDir); +bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry); +char* taosGetDirEntryName(TdDirEntryPtr pDirEntry); +int32_t taosCloseDir(TdDirPtr pDir); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 74fb8c1c1f..00e97c7b61 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); - if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) { + if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { tfree(s); return -1; } diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fd267fbf03..89f3f8ba8a 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -722,13 +722,13 @@ static SArray* tfileGetFileList(const char* path) { uint32_t version; SArray* files = taosArrayInit(4, sizeof(void*)); - DIR* dir = opendir(path); - if (NULL == dir) { + TdDirPtr pDir = taosOpenDir(path); + if (NULL == pDir) { return NULL; } - struct dirent* entry; - while ((entry = readdir(dir)) != NULL) { - char* file = entry->d_name; + TdDirEntryPtr pDirEntry; + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* file = taosGetDirEntryName(pDirEntry); if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } @@ -738,7 +738,7 @@ static SArray* tfileGetFileList(const char* path) { sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } - closedir(dir); + taosCloseDir(pDir); taosArraySort(files, tfileCompare); tfileRmExpireFile(files); diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 913f34d6c2..f16d0445c6 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -59,7 +59,7 @@ typedef struct STfsDir { SDiskID did; char dirname[TSDB_FILENAME_LEN]; STfsFile tfile; - DIR *dir; + TdDirPtr pDir; STfs *pTfs; } STfsDir; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 2579490791..c46989dc5d 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -192,14 +192,14 @@ void tfsBasename(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); - tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); + tstrncpy(dest, taosDirEntryBaseName(tname), TSDB_FILENAME_LEN); } void tfsDirname(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); - tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); + tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN); } int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } @@ -233,7 +233,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { // the pointer directly in this recursion. // See // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html - char *dir = strdup(dirname(s)); + char *dir = strdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { free(s); @@ -324,45 +324,46 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { return pDir; } -const STfsFile *tfsReaddir(STfsDir *pDir) { - if (pDir == NULL || pDir->dir == NULL) return NULL; +const STfsFile *tfsReaddir(STfsDir *pTfsDir) { + if (pTfsDir == NULL || pTfsDir->pDir == NULL) return NULL; char bname[TMPNAME_LEN * 2] = "\0"; while (true) { - struct dirent *dp = NULL; - dp = readdir(pDir->dir); - if (dp != NULL) { + TdDirEntryPtr pDirEntry = NULL; + pDirEntry = taosReadDir(pTfsDir->pDir); + if (pDirEntry != NULL) { // Skip . and .. - if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; + char *name = taosGetDirEntryName(pDirEntry); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - if (pDir->dirname == NULL || pDir->dirname[0] == 0) { - snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name); + if (pTfsDir->dirname == NULL || pTfsDir->dirname[0] == 0) { + snprintf(bname, TMPNAME_LEN * 2, "%s", name); } else { - snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pTfsDir->dirname, TD_DIRSEP, name); } - tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); - return &pDir->tfile; + tfsInitFile(pTfsDir->pTfs, &pTfsDir->tfile, pTfsDir->did, bname); + return &pTfsDir->tfile; } - if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) { + if (tfsOpendirImpl(pTfsDir->pTfs, pTfsDir) < 0) { return NULL; } - if (pDir->dir == NULL) { + if (pTfsDir->pDir == NULL) { terrno = TSDB_CODE_SUCCESS; return NULL; } } } -void tfsClosedir(STfsDir *pDir) { - if (pDir) { - if (pDir->dir != NULL) { - closedir(pDir->dir); - pDir->dir = NULL; +void tfsClosedir(STfsDir *pTfsDir) { + if (pTfsDir) { + if (pTfsDir->pDir != NULL) { + taosCloseDir(pTfsDir->pDir); + pTfsDir->pDir = NULL; } - free(pDir); + free(pTfsDir); } } @@ -487,29 +488,29 @@ static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) { return pDisk; } -static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { +static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { STfsDisk *pDisk = NULL; char adir[TMPNAME_LEN * 2] = "\0"; - if (pDir->dir != NULL) { - closedir(pDir->dir); - pDir->dir = NULL; + if (pTfsDir->pDir != NULL) { + taosCloseDir(pTfsDir->pDir); + pTfsDir->pDir = NULL; } while (true) { - pDisk = tfsNextDisk(pTfs, &pDir->iter); + pDisk = tfsNextDisk(pTfs, &pTfsDir->iter); if (pDisk == NULL) return 0; - pDir->did.level = pDisk->level; - pDir->did.id = pDisk->id; + pTfsDir->did.level = pDisk->level; + pTfsDir->did.id = pDisk->id; if (pDisk->path == NULL || pDisk->path[0] == 0) { - snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname); + snprintf(adir, TMPNAME_LEN * 2, "%s", pTfsDir->dirname); } else { - snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pTfsDir->dirname); } - pDir->dir = opendir(adir); - if (pDir->dir != NULL) break; + pTfsDir->pDir = taosOpenDir(adir); + if (pTfsDir->pDir != NULL) break; } return 0; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2a4f3497e4..248f758787 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -130,16 +130,16 @@ int walCheckAndRepairMeta(SWal* pWal) { regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); - DIR* dir = opendir(pWal->path); - if (dir == NULL) { + TdDirPtr pDir = taosOpenDir(pWal->path); + if (pDir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } // scan log files and build new meta - struct dirent* ent; - while ((ent = readdir(dir)) != NULL) { - char* name = basename(ent->d_name); + TdDirEntryPtr pDirEntry; + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry)); int code = regexec(&logRegPattern, name, 0, NULL, 0); if (code == 0) { SWalFileInfo fileInfo; @@ -149,7 +149,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - closedir(dir); + taosCloseDir(pDir); regfree(&logRegPattern); regfree(&idxRegPattern); @@ -337,25 +337,25 @@ static int walFindCurMetaVer(SWal* pWal) { regex_t walMetaRegexPattern; regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); - DIR* dir = opendir(pWal->path); - if (dir == NULL) { + TdDirPtr pDir = taosOpenDir(pWal->path); + if (pDir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } - struct dirent* ent; + TdDirEntryPtr pDirEntry; // find existing meta-ver[x].json int metaVer = -1; - while ((ent = readdir(dir)) != NULL) { - char* name = basename(ent->d_name); + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry)); int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); if (code == 0) { sscanf(name, "meta-ver%d", &metaVer); break; } } - closedir(dir); + taosCloseDir(pDir); regfree(&walMetaRegexPattern); return metaVer; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 7d7382d83f..b4058b3c0e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #include "osString.h" @@ -36,6 +37,10 @@ #include #include +typedef struct dirent dirent; +typedef struct DIR TdDir; +typedef struct dirent TdDirent; + void taosRemoveDir(const char *dirname) { DIR *dir = opendir(dirname); if (dir == NULL) return; @@ -149,4 +154,47 @@ bool taosIsDir(const char *dirname) { return false; } +char* taosDirName(char *name) { + return dirname(name); +} + +char* taosDirEntryBaseName(char *name) { + return basename(name); +} + +TdDirPtr taosOpenDir(const char *dirname) { + if (dirname == NULL) { + return NULL; + } + return (TdDirPtr)opendir(dirname); +} + +TdDirEntryPtr taosReadDir(TdDirPtr pDir) { + if (pDir == NULL) { + return NULL; + } + return (TdDirEntryPtr)readdir((DIR*)pDir); +} + +bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { + if (pDirEntry == NULL) { + return false; + } + return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0; +} + +char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) { + if (pDirEntry == NULL) { + return NULL; + } + return ((dirent*)pDirEntry)->d_name; +} + +int32_t taosCloseDir(TdDirPtr pDir) { + if (pDir == NULL) { + return -1; + } + return closedir((DIR*)pDir); +} + #endif diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 0d7066b5c8..e5b506e8d8 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -99,7 +99,7 @@ static void *sem_thread_routine(void *arg) { sem_port = mach_task_self(); kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); if (ret != KERN_SUCCESS) { - fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); sem_inited = -1; return NULL; } @@ -112,7 +112,7 @@ static void once_init(void) { int r = 0; r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); return; } while (sem_inited == 0) { @@ -139,14 +139,14 @@ struct tsem_s { }; int tsem_init(tsem_t *sem, int pshared, unsigned int value) { - // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); if (*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); if (!p) { - fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -162,7 +162,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { p->val = value; } while (0); if (r) { - fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #elif defined(SEM_USE_POSIX) @@ -181,27 +181,27 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { int e = errno; if (e == EEXIST) continue; if (e == EINTR) continue; - fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, strerror(e)); abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) pthread_once(&sem_once, once_init); if (sem_inited != 1) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; return -1; } kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); if (ret != KERN_SUCCESS) { - fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // we fail-fast here, because we have less-doc about semaphore_create for the moment abort(); } #else // SEM_USE_PTHREAD p->sem = dispatch_semaphore_create(value); if (p->sem == NULL) { - fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #endif // SEM_USE_PTHREAD @@ -215,28 +215,28 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { int tsem_wait(tsem_t *sem) { if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val -= 1; if (p->val < 0) { if (pthread_cond_wait(&p->cond, &p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } return 0; @@ -251,28 +251,28 @@ int tsem_wait(tsem_t *sem) { int tsem_post(tsem_t *sem) { if (!*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } struct tsem_s *p = *sem; if (!p->valid) { - fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val += 1; if (p->val <= 0) { if (pthread_cond_signal(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } return 0; @@ -286,34 +286,34 @@ int tsem_post(tsem_t *sem) { } int tsem_destroy(tsem_t *sem) { - // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); if (!*sem) { - // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // abort(); return 0; } struct tsem_s *p = *sem; if (!p->valid) { - // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // abort(); return 0; } #ifdef SEM_USE_PTHREAD if (pthread_mutex_lock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->valid = 0; if (pthread_cond_destroy(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } if (pthread_mutex_unlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } if (pthread_mutex_destroy(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #elif defined(SEM_USE_POSIX) @@ -322,7 +322,7 @@ int tsem_destroy(tsem_t *sem) { int r = sem_unlink(name); if (r) { int e = errno; - fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, strerror(e)); abort(); } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index bb526e0ba0..6b60923189 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -79,7 +79,7 @@ static void* timer_routine(void* arg) { struct kevent64_s kev[10] = {0}; r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to); if (r != 0) { - fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); abort(); } timer_callback(SIGALRM); // just mock @@ -97,14 +97,14 @@ int taosInitTimer(void (*callback)(int), int ms) { timer_kq = kqueue(); if (timer_kq == -1) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } r = pthread_create(&timer_thread, NULL, timer_routine, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } @@ -116,7 +116,7 @@ void taosUninitTimer() { timer_stop = 1; r = pthread_join(timer_thread, NULL); if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment abort(); } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 3eb8e60d56..cb7d7c67ce 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -192,11 +192,11 @@ static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } // calc dir size (not include itself 4096Byte) int64_t getDirectorySize(char *dir) { - DIR *dp; - struct dirent *entry; + TdDirPtr pDir; + TdDirEntryPtr pDirEntry; int64_t totalSize=0; - if ((dp = opendir(dir)) == NULL) { + if ((pDir = taosOpenDir(dir)) == NULL) { fprintf(stderr, "Cannot open dir: %s\n", dir); return -1; } @@ -204,26 +204,27 @@ int64_t getDirectorySize(char *dir) //lstat(dir, &statbuf); //totalSize+=statbuf.st_size; - while ((entry = readdir(dp)) != NULL) { + while ((pDirEntry = taosReadDir(pDir)) != NULL) { char subdir[1024]; - sprintf(subdir, "%s/%s", dir, entry->d_name); + char* fileName = taosGetDirEntryName(pDirEntry); + sprintf(subdir, "%s/%s", dir, fileName); //printf("===d_name: %s\n", entry->d_name); if (taosIsDir(subdir)) { - if (strcmp(".", entry->d_name) == 0 || strcmp("..", entry->d_name) == 0) { + if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { continue; } int64_t subDirSize = getDirectorySize(subdir); totalSize+=subDirSize; - } else if (0 == strcmp(strchr(entry->d_name, '.'), ".log")) { // only calc .log file size, and not include .idx file + } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file int64_t file_size = 0; taosStatFile(subdir, &file_size, NULL); totalSize+=file_size; } } - closedir(dp); + taosCloseDir(pDir); return totalSize; } From aad01e0155a02ceb461a575f7ef628359ff443bc Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 9 Mar 2022 16:38:26 +0800 Subject: [PATCH 22/39] add SSmaStat in tsdb to demonstrate window valid status --- include/common/taosdef.h | 7 +- source/dnode/vnode/src/inc/tsdbDef.h | 1 + source/dnode/vnode/src/inc/tsdbFS.h | 5 +- source/dnode/vnode/src/inc/tsdbSma.h | 11 +- source/dnode/vnode/src/meta/metaBDBImpl.c | 8 +- source/dnode/vnode/src/tsdb/tsdbBDBImpl.c | 14 +++ source/dnode/vnode/src/tsdb/tsdbMain.c | 1 + source/dnode/vnode/src/tsdb/tsdbSma.c | 128 ++++++++++++++++++++++ 8 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbBDBImpl.c diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 69c2618ac8..7788dfd871 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -50,12 +50,17 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; -typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; +typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2} TDUpdateConfig; typedef enum { TSDB_STATIS_OK = 0, // statis part exist and load successfully TSDB_STATIS_NONE = 1, // statis part not exist } ETsdbStatisStatus; +typedef enum { + TSDB_SMA_STAT_OK = 0, // ready to provide service + TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired +} ETsdbSmaStat; + extern char *qtypeStr[]; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 96a76ea7d4..1451ac9685 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -52,6 +52,7 @@ struct STsdb { STsdbFS * fs; SMeta * pMeta; STfs * pTfs; + SSmaStat * pSmaStat; }; #define REPO_ID(r) ((r)->vgId) diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 641255a294..173e991631 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,7 +42,10 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - SArray * smaf; // sma data file array + + // SArray * v2f100.tsma.index_name + + SArray * smaf; // sma data file array v2f1900.tsma.index_name } SFSStatus; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index e4de7a6685..6e4ad909ae 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_SMA_H_ #define _TD_TSDB_SMA_H_ +typedef struct SSmaStat SSmaStat; + // insert/update interface int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData); int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData); @@ -26,13 +28,14 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData); int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult); // management interface -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result); +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg); +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); - - - +int32_t tsdbFreeSmaState(SSmaStat *pSmaStat); // internal func + + static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { int32_t len = 0; len += taosEncodeFixedU64(pData, tableUid); diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index f49515412b..efdb3e0fe4 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -923,6 +923,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { SMetaDB *pDB = pMeta->pDB; DBC * pCur = NULL; DBT pkey = {0}, pval = {0}; + uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP; int ret; pUids = taosArrayInit(16, sizeof(tb_uid_t)); @@ -941,13 +942,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { void *pBuf = NULL; // TODO: lock? - while (true) { - ret = pCur->get(pCur, &pkey, &pval, isDup ? DB_NEXT_DUP : DB_NEXT_NODUP); - if(ret == 0) { + while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) { taosArrayPush(pUids, pkey.data); - continue; - } - break; } if (pCur) { diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c new file mode 100644 index 0000000000..f2f48bbc8a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 2d8c470113..1b3e00f090 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -89,6 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { tsdbFreeFS(pTsdb->fs); + tsdbFreeSmaState(pTsdb->pSmaStat); tfree(pTsdb->path); free(pTsdb); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index b465dc3a88..5cfa597eb2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -21,6 +21,10 @@ #define SMA_STORE_SINGLE_BLOCKS // store SMA data by single block or multiple blocks +#define SMA_STATE_HASH_SLOT 4 +#define SMA_STATE_ITEM_HASH_SLOT 32 + +#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test typedef enum { SMA_STORAGE_LEVEL_TSDB = 0, // store TSma in dir e.g. vnode${N}/tsdb/.tsma SMA_STORAGE_LEVEL_DFILESET = 1 // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name} @@ -48,6 +52,22 @@ typedef struct { // TODO } STSmaReadH; +typedef struct { + /** + * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from + * Streaming Module or TSDB local persistence. + */ + int8_t state; // ETsdbSmaStat + SHashObj *expiredWindows; // key: skey of time window, value: N/A +} SSmaStatItem; + +struct SSmaStat { + SHashObj *smaStatItems; // key: indexName, value: SSmaStatItem +}; + // declaration of static functions static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData); @@ -64,6 +84,114 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin); +static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { + ASSERT(pSmaStat != NULL); + // TODO: lock and create when each put, or create during tsdbNew. + if (*pSmaStat == NULL) { + *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); + if (*pSmaStat == NULL) { + // TODO: unlock + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + (*pSmaStat)->smaStatItems = + taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + if ((*pSmaStat)->smaStatItems == NULL) { + tfree(*pSmaStat); + // TODO: unlock + return TSDB_CODE_FAILED; + } + } + // TODO: unlock + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem)); + if (pItem) { + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + tfree(pItem); + } + } + return pItem; +} + +int32_t tsdbFreeSmaState(SSmaStat *pSmaStat) { + if (pSmaStat) { + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); + while (item != NULL) { + taosHashCleanup(item->expiredWindows); + item = taosHashIterate(pSmaStat->smaStatItems, item); + } + + taosHashCleanup(pSmaStat->smaStatItems); + free(pSmaStat); + } +} + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { + if (msg == NULL) { + return TSDB_CODE_FAILED; + } + + // TODO: decode the msg => start + const char * indexName = SMA_TEST_INDEX_NAME; + const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; + TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; + int64_t now = taosGetTimestampMs(); + for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { + expiredWindows[i] = now + i; + } + // TODO: decode the msg <= end + + SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems; + + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName)); + if (!pItem) { + pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexName not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + free(pItem); + return TSDB_CODE_FAILED; + } + } + + int8_t state = TSDB_SMA_STAT_EXPIRED; + for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { + if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) { + // If error occurs during put expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would tell + // query module to query raw TS data. + taosHashCleanup(pItem->expiredWindows); + taosHashRemove(pItemsHash, indexName, sizeof(indexName)); + return TSDB_CODE_FAILED; + } + } + + return TSDB_CODE_SUCCESS; +} + /** * @brief Judge the tSma storage level * From b91899e7574bb43710003bf82539571dbb07cecc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Mar 2022 04:32:12 -0500 Subject: [PATCH 23/39] TD-13704 super table plan split --- include/libs/nodes/nodes.h | 2 + include/libs/nodes/plannodes.h | 39 ++-- include/libs/planner/planner.h | 4 +- source/libs/executor/src/executorimpl.c | 6 +- source/libs/function/src/builtins.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 12 ++ source/libs/nodes/src/nodesCodeFuncs.c | 41 +++- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/planner/inc/plannerInt.h | 1 + source/libs/planner/src/physicalPlan.c | 200 +++++++++++++++--- source/libs/planner/src/planner.c | 4 +- source/libs/planner/src/splitPlan.c | 165 +++++++++++++++ source/libs/planner/test/plannerTest.cpp | 7 + source/libs/scheduler/src/scheduler.c | 4 +- source/libs/scheduler/test/schedulerTests.cpp | 14 +- 15 files changed, 440 insertions(+), 63 deletions(-) create mode 100644 source/libs/planner/src/splitPlan.c diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index b27150c9fb..a108cc6937 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -66,6 +66,7 @@ typedef enum ENodeType { QUERY_NODE_DATABLOCK_DESC, QUERY_NODE_SLOT_DESC, QUERY_NODE_COLUMN_DEF, + QUERY_NODE_DOWNSTREAM_SOURCE, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, @@ -98,6 +99,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_AGG, QUERY_NODE_LOGIC_PLAN_PROJECT, QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, + QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_SUBPLAN, QUERY_NODE_LOGIC_PLAN, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index c805eba320..084d642292 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -67,12 +67,17 @@ typedef struct SProjectLogicNode { } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - ENodeType type;; + SLogicNode node;; int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; } SVnodeModifLogicNode; +typedef struct SExchangeLogicNode { + SLogicNode node; + int32_t srcGroupId; +} SExchangeLogicNode; + typedef enum ESubplanType { SUBPLAN_TYPE_MERGE = 1, SUBPLAN_TYPE_PARTIAL, @@ -80,13 +85,22 @@ typedef enum ESubplanType { SUBPLAN_TYPE_MODIFY } ESubplanType; +typedef struct SSubplanId { + uint64_t queryId; + int32_t groupId; + int32_t subplanId; +} SSubplanId; + typedef struct SSubLogicPlan { ENodeType type; + SSubplanId id; SNodeList* pChildren; SNodeList* pParents; SLogicNode* pNode; ESubplanType subplanType; + SVgroupsInfo* pVgroupList; int32_t level; + int32_t splitFlag; } SSubLogicPlan; typedef struct SQueryLogicPlan { @@ -161,20 +175,21 @@ typedef struct SAggPhysiNode { SNodeList* pAggFuncs; } SAggPhysiNode; -typedef struct SDownstreamSource { +typedef struct SDownstreamSourceNode { + ENodeType type; SQueryNodeAddr addr; - uint64_t taskId; - uint64_t schedId; -} SDownstreamSource; + uint64_t taskId; + uint64_t schedId; +} SDownstreamSourceNode; typedef struct SExchangePhysiNode { - SPhysiNode node; - uint64_t srcTemplateId; // template id of datasource suplans - SArray* pSrcEndPoints; // SArray, scheduler fill by calling qSetSuplanExecutionNode + SPhysiNode node; + int32_t srcGroupId; // group id of datasource suplans + SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; typedef struct SDataSinkNode { - ENodeType type;; + ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; } SDataSinkNode; @@ -189,12 +204,6 @@ typedef struct SDataInserterNode { char *pData; } SDataInserterNode; -typedef struct SSubplanId { - uint64_t queryId; - int32_t templateId; - int32_t subplanId; -} SSubplanId; - typedef struct SSubplan { ENodeType type; SSubplanId id; // unique id of the subplan diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 0d62b7f0df..5d6ec46d85 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -32,9 +32,9 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule -// @templateId templateId of a group of datasource subplans of this @subplan +// @groupId id of a group of datasource subplans of this @subplan // @ep one execution location of this group of datasource subplans -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); +void qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource); // Convert to subplan to string for the scheduler to send to the executor int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ea97f10dd5..f5ef9ccc16 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4984,7 +4984,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf return pTaskInfo->code; } - SDownstreamSource *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SDownstreamSourceNode *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, @@ -5082,7 +5082,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SEx } SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, i); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); SSDataBlock* pRes = pExchangeInfo->pResult; @@ -5179,7 +5179,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { tsem_wait(&pExchangeInfo->ready); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); SRetrieveTableRsp* pRsp = pDataInfo->pRsp; if (pRsp->numOfRows == 0) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index d9eeb6eeeb..e3dd2b499c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -51,7 +51,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { } }; -const int funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); +const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 27d71ff532..e11a7a6833 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -60,6 +60,9 @@ #define CLONE_OBJECT_FIELD(fldname, cloneFunc) \ do { \ + if (NULL == (pSrc)->fldname) { \ + break; \ + } \ (pDst)->fldname = cloneFunc((pSrc)->fldname); \ if (NULL == (pDst)->fldname) { \ nodesDestroyNode((SNode*)(pDst)); \ @@ -234,10 +237,17 @@ static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* } static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(msgType); return (SNode*)pDst; } +static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(srcGroupId); + return (SNode*)pDst; +} + static SNode* logicSubplanCopy(const SSubLogicPlan* pSrc, SSubLogicPlan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); @@ -304,6 +314,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicProjectCopy((const SProjectLogicNode*)pNode, (SProjectLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SSubLogicPlan*)pNode, (SSubLogicPlan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b28c45e554..2e004aa4b2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -496,6 +496,37 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkExchangePhysiPlanSrcGroupId = "SrcGroupId"; +static const char* jkExchangePhysiPlanSrcEndPoints = "SrcEndPoints"; + +static int32_t physiExchangeNodeToJson(const void* pObj, SJson* pJson) { + const SExchangePhysiNode* pNode = (const SExchangePhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangePhysiPlanSrcGroupId, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkExchangePhysiPlanSrcEndPoints, pNode->pSrcEndPoints); + } + + return code; +} + +static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { + SExchangePhysiNode* pNode = (SExchangePhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkExchangePhysiPlanSrcGroupId, &pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkExchangePhysiPlanSrcEndPoints, &pNode->pSrcEndPoints); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -517,7 +548,7 @@ static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { } static const char* jkSubplanIdQueryId = "QueryId"; -static const char* jkSubplanIdTemplateId = "TemplateId"; +static const char* jkSubplanIdGroupId = "GroupId"; static const char* jkSubplanIdSubplanId = "SubplanId"; static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { @@ -525,7 +556,7 @@ static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdTemplateId, pNode->templateId); + code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); @@ -539,7 +570,7 @@ static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdTemplateId, &pNode->templateId); + code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); @@ -1387,7 +1418,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_AGG: return physiAggNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + return physiExchangeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: + break; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -1459,6 +1492,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiJoinNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_AGG: return jsonToPhysiAggNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + return jsonToPhysiExchangeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39e64ba107..c2b96b1c99 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -129,6 +129,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SProjectLogicNode)); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return makeNode(type, sizeof(SVnodeModifLogicNode)); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SSubLogicPlan)); case QUERY_NODE_LOGIC_PLAN: diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 991ad72a31..d57f40ca35 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -50,6 +50,7 @@ extern "C" { int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode); +int32_t applySplitRule(SSubLogicPlan* pSubplan); int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList); #ifdef __cplusplus diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 61c5f517a7..b0c087b36a 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -28,6 +28,8 @@ typedef struct SPhysiPlanContext { int16_t nextDataBlockId; SArray* pLocationHelper; SArray* pExecNodeList; + int32_t groupId; + int32_t subplanId; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, char* pKey) { @@ -81,12 +83,7 @@ static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SData SNode* pNode = NULL; int16_t slotId = taosHashGetSize(pHash); FOREACH(pNode, pList) { - SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); - CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(pDataBlockDesc->pSlots, (SNode*)pSlot)) { - nodesDestroyNode(pSlot); - return TSDB_CODE_OUT_OF_MEMORY; - } + CHECK_CODE_EXT(nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId))); SSlotIndex index = { .dataBlockId = pDataBlockDesc->dataBlockId, .slotId = slotId }; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; @@ -97,7 +94,7 @@ static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SData CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); REPLACE_NODE(pTarget); - pDataBlockDesc->resultRowSize += ((SSlotDescNode*)pSlot)->dataType.bytes; + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; ++slotId; } return TSDB_CODE_SUCCESS; @@ -467,6 +464,14 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return (SPhysiNode*)pProject; } +static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + CHECK_ALLOC(pExchange, NULL); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; + return (SPhysiNode*)pExchange; +} + static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SLogicNode* pLogicPlan) { SNodeList* pChildren = nodesMakeList(); CHECK_ALLOC(pChildren, NULL); @@ -495,6 +500,9 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, case QUERY_NODE_LOGIC_PLAN_PROJECT: pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); break; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + pPhyNode = createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicPlan); + break; default: break; } @@ -525,8 +533,15 @@ static SDataSinkNode* createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysi return (SDataSinkNode*)pDispatcher; } +static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { + SSubplan* pSubplan = nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + CHECK_ALLOC(pSubplan, NULL); + pSubplan->id = pLogicSubplan->id; + return pSubplan; +} + static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { - SSubplan* pSubplan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); CHECK_ALLOC(pSubplan, NULL); if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; @@ -540,9 +555,22 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog pSubplan->msgType = TDMT_VND_QUERY; } pSubplan->subplanType = pLogicSubplan->subplanType; + pSubplan->level = pLogicSubplan->level; return pSubplan; } +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + doSetLogicNodeParent((SLogicNode*)pChild, pNode); + } +} + +static void setLogicNodeParent(SLogicNode* pNode) { + doSetLogicNodeParent(pNode, NULL); +} + static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubLogicPlan** pSubLogicPlan) { *pSubLogicPlan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); CHECK_ALLOC(*pSubLogicPlan, TSDB_CODE_OUT_OF_MEMORY); @@ -553,8 +581,9 @@ static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, S } else { (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_MERGE; } - // todo split - return TSDB_CODE_SUCCESS; + (*pSubLogicPlan)->id.queryId = pCxt->pPlanCxt->queryId; + setLogicNodeParent((*pSubLogicPlan)->pNode); + return applySplitRule(*pSubLogicPlan); } static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t level, SNodeList* pSubplans) { @@ -571,6 +600,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l CHECK_ALLOC(pGroup->pNodeList, TSDB_CODE_OUT_OF_MEMORY); } CHECK_CODE(nodesListStrictAppend(pGroup->pNodeList, pSubplan), TSDB_CODE_OUT_OF_MEMORY); + return TSDB_CODE_SUCCESS; } SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { @@ -583,29 +613,105 @@ SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* p } pDst->subplanType = pSrc->subplanType; pDst->level = level; + pDst->id.queryId = pCxt->pPlanCxt->queryId; + pDst->id.groupId = pCxt->groupId; + pDst->id.subplanId = pCxt->subplanId++; return pDst; } -static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan) { - if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; - CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); - } - } else { +static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { + SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); + SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; + // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); + CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); } + return TSDB_CODE_SUCCESS; +} +static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { + // SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + // CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); + // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); + // return TSDB_CODE_SUCCESS; + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t doSetScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + CHECK_ALLOC(pScan->pVgroupList, TSDB_CODE_OUT_OF_MEMORY); + memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); + *pFound = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + int32_t code = doSetScanVgroup(pCxt, (SLogicNode*)pChild, pVgroup, pFound); + if (TSDB_CODE_SUCCESS != code || *pFound) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup) { + bool found = false; + return doSetScanVgroup(pCxt, pNode, pVgroup, &found); +} + +static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { + if (pSubplan->pVgroupList) { + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE_EXT(setScanVgroup(pCxt, pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i)); + // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); + CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); + } + return TSDB_CODE_SUCCESS; + } else { + return scaleOutForMerge(pCxt, pSubplan, level, pLogicPlan, pGroup); + } +} + +// static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup, int32_t level, SQueryLogicPlan* pLogicPlan) { +// FOREACH() { + +// } +// } + +static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SHashObj* pHash, SNodeList* pParentsGroup) { + SNodeList* pCurrentGroup = nodesMakeList(); + CHECK_ALLOC(pCurrentGroup, TSDB_CODE_OUT_OF_MEMORY); + int32_t code = TSDB_CODE_SUCCESS; + switch (pSubplan->subplanType) { + case SUBPLAN_TYPE_MERGE: + code = scaleOutForMerge(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + break; + case SUBPLAN_TYPE_SCAN: + code = scaleOutForScan(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + break; + case SUBPLAN_TYPE_MODIFY: + code = scaleOutForModify(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + break; + default: + break; + } + if (TSDB_CODE_SUCCESS != code) { + return code; + } + // pushHierarchicalPlan(pParentsGroup, pCurrentGroup, level, pLogicPlan); + CHECK_CODE(taosHashPut(pHash, &pCxt->groupId, sizeof(pCxt->groupId), &pSubplan->id.groupId, sizeof(pSubplan->id.groupId)), TSDB_CODE_OUT_OF_MEMORY); + ++(pCxt->groupId); SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, level + 1, pLogicPlan)); + CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, level + 1, pLogicPlan, pHash, pCurrentGroup)); } return TSDB_CODE_SUCCESS; @@ -622,10 +728,49 @@ static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { return pLogicPlan; } +static int32_t doMappingLogicPlan(SLogicNode* pNode, SHashObj* pHash) { + if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pNode)) { + SExchangeLogicNode* pExchange = (SExchangeLogicNode*)pNode; + int32_t* pGroupId = taosHashGet(pHash, &pExchange->srcGroupId, sizeof(pExchange->srcGroupId)); + if (NULL == pGroupId) { + return TSDB_CODE_FAILED; + } + pExchange->srcGroupId = *pGroupId; + return TSDB_CODE_SUCCESS; + } + + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + doMappingLogicPlan((SLogicNode*)pChild, pHash); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t mappingLogicPlan(SQueryLogicPlan* pLogicPlan, SHashObj* pHash) { + SNode* pNode = NULL; + FOREACH(pNode, pLogicPlan->pSubplans) { + SNode* pSubplan = NULL; + FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { + int32_t code = doMappingLogicPlan(((SSubLogicPlan*)pSubplan)->pNode, pHash); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) { *pLogicPlan = makeQueryLogicPlan(pCxt); CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - return doScaleOut(pCxt, pRootSubLogicPlan, 0, *pLogicPlan); + SHashObj* pHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); + int32_t code = doScaleOut(pCxt, pRootSubLogicPlan, 0, *pLogicPlan, pHash, NULL); + if (TSDB_CODE_SUCCESS == code) { + code = mappingLogicPlan(*pLogicPlan, pHash); + } + taosHashCleanup(pHash); + return code; } typedef struct SBuildPhysiSubplanCxt { @@ -681,13 +826,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - SQueryLogicPlan* pLogicPlan; - SSubLogicPlan* pSubLogicPlan; + SQueryLogicPlan* pLogicPlan = NULL; + SSubLogicPlan* pSubLogicPlan = NULL; int32_t code = splitLogicPlan(&cxt, pLogicNode, &pSubLogicPlan); if (TSDB_CODE_SUCCESS == code) { code = scaleOutLogicPlan(&cxt, pSubLogicPlan, &pLogicPlan); } - // todo maping if (TSDB_CODE_SUCCESS == code) { code = buildPhysiPlan(&cxt, pLogicPlan, pPlan); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index d7d9f1d129..4fa9b6ba10 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -34,7 +34,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo return code; } -void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { +void qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { } @@ -62,5 +62,5 @@ SQueryPlan* qStringToQueryPlan(const char* pStr) { } void qDestroyQueryPlan(SQueryPlan* pPlan) { - + nodesDestroyNode(pPlan); } diff --git a/source/libs/planner/src/splitPlan.c b/source/libs/planner/src/splitPlan.c new file mode 100644 index 0000000000..97e2eefb67 --- /dev/null +++ b/source/libs/planner/src/splitPlan.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "plannerInt.h" + +#define SPLIT_FLAG_MASK(n) (1 << n) + +#define SPLIT_FLAG_STS SPLIT_FLAG_MASK(0) + +#define SPLIT_FLAG_SET_MASK(val, mask) (val) |= (mask) +#define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) + +typedef struct SSplitContext { + int32_t errCode; + int32_t groupId; + bool match; + void* pInfo; +} SSplitContext; + +typedef int32_t (*FMatch)(SSplitContext* pCxt, SSubLogicPlan* pSubplan); +typedef int32_t (*FSplit)(SSplitContext* pCxt); + +typedef struct SSplitRule { + char* pName; + FMatch matchFunc; + FSplit splitFunc; +} SSplitRule; + +typedef struct SStsInfo { + SScanLogicNode* pScan; + SSubLogicPlan* pSubplan; +} SStsInfo; + +static SLogicNode* stsMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = stsMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { + if (SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { + return TSDB_CODE_SUCCESS; + } + SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); + CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); + pInfo->pScan = (SScanLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + pCxt->pInfo = pInfo; + pCxt->match = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild; + FOREACH(pChild, pSubplan->pChildren) { + int32_t code = stsMatch(pCxt, (SSubLogicPlan*)pChild); + if (TSDB_CODE_SUCCESS != code || pCxt->match) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { + SSubLogicPlan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return NULL; + } + pSubplan->id.groupId = pCxt->groupId; + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo); + SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); + return pSubplan; +} + +static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubplan, SScanLogicNode* pScan) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + pExchange->node.pTargets = nodesCloneList(pScan->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL == pScan->node.pParent) { + pSubplan->pNode = (SLogicNode*)pExchange; + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pScan->node.pParent->pChildren) { + if (nodesEqualNode(pNode, pScan)) { + REPLACE_NODE(pExchange); + nodesDestroyNode(pNode); + return TSDB_CODE_SUCCESS; + } + } + nodesDestroyNode(pExchange); + return TSDB_CODE_FAILED; +} + +static int32_t stsSplit(SSplitContext* pCxt) { + SStsInfo* pInfo = pCxt->pInfo; + if (NULL == pInfo->pSubplan->pChildren) { + pInfo->pSubplan->pChildren = nodesMakeList(); + if (NULL == pInfo->pSubplan->pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + int32_t code = nodesListStrictAppend(pInfo->pSubplan->pChildren, stsCreateScanSubplan(pCxt, pInfo->pScan)); + if (TSDB_CODE_SUCCESS == code) { + code = stsCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pScan); + } + ++(pCxt->groupId); + return code; +} + +static const SSplitRule splitRuleSet[] = { + { .pName = "SuperTableScan", .matchFunc = stsMatch, .splitFunc = stsSplit } +}; + +static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); + +int32_t applySplitRule(SSubLogicPlan* pSubplan) { + SSplitContext cxt = { .errCode = TSDB_CODE_SUCCESS, .groupId = pSubplan->id.groupId + 1, .match = false, .pInfo = NULL }; + bool split = false; + do { + split = false; + for (int32_t i = 0; i < splitRuleNum; ++i) { + cxt.match = false; + int32_t code = splitRuleSet[i].matchFunc(&cxt, pSubplan); + if (TSDB_CODE_SUCCESS == code && cxt.match) { + code = splitRuleSet[i].splitFunc(&cxt); + split = true; + } + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } while (split); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 8ba80d1ab9..ae25b157f6 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -137,6 +137,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, stSimple) { + setDatabase("root", "test"); + + bind("SELECT * FROM st1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, groupBy) { setDatabase("root", "test"); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fe886dfcdb..81398acc70 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -797,8 +797,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; - qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); + SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, .taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; + qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 0347318ae5..084a76ebde 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -99,7 +99,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); scanPlan->id.queryId = qId; - scanPlan->id.templateId = 0x0000000000000002; + scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003; scanPlan->subplanType = SUBPLAN_TYPE_SCAN; @@ -114,7 +114,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; - mergePlan->id.templateId = schtMergeTemplateId; + mergePlan->id.groupId = schtMergeTemplateId; mergePlan->id.subplanId = 0x5555; mergePlan->subplanType = SUBPLAN_TYPE_MERGE; mergePlan->level = 0; @@ -158,7 +158,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { for (int32_t i = 0; i < scanPlanNum; ++i) { scanPlan[i].id.queryId = qId; - scanPlan[i].id.templateId = 0x0000000000000002; + scanPlan[i].id.groupId = 0x0000000000000002; scanPlan[i].id.subplanId = 0x0000000000000003 + i; scanPlan[i].subplanType = SUBPLAN_TYPE_SCAN; @@ -183,7 +183,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { } mergePlan->id.queryId = qId; - mergePlan->id.templateId = schtMergeTemplateId; + mergePlan->id.groupId = schtMergeTemplateId; mergePlan->id.subplanId = 0x5555; mergePlan->subplanType = SUBPLAN_TYPE_MERGE; mergePlan->level = 0; @@ -216,7 +216,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; - insertPlan[0].id.templateId = 0x0000000000000003; + insertPlan[0].id.groupId = 0x0000000000000003; insertPlan[0].id.subplanId = 0x0000000000000004; insertPlan[0].subplanType = SUBPLAN_TYPE_MODIFY; insertPlan[0].level = 0; @@ -232,7 +232,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; - insertPlan[1].id.templateId = 0x0000000000000003; + insertPlan[1].id.groupId = 0x0000000000000003; insertPlan[1].id.subplanId = 0x0000000000000005; insertPlan[1].subplanType = SUBPLAN_TYPE_MODIFY; insertPlan[1].level = 0; @@ -263,7 +263,7 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { return 0; } -void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { +void schtExecNode(SSubplan* subplan, uint64_t groupId, SQueryNodeAddr* ep) { } From 031b50ff764ca2233b9a8dadbd5d12e6f4e9ea99 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 9 Mar 2022 17:52:59 +0800 Subject: [PATCH 24/39] [TD-13761]: redefine dir api. --- source/os/src/osSocket.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 9330cb7b32..24db2d7c5f 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -809,7 +809,7 @@ int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { - // printf("failed to get hostname, reason:%s", strerror(errno)); + printf("failed to get hostname, reason:%s", strerror(errno)); return -1; } @@ -826,7 +826,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - // printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } From a5fece22c6819ac14f1f561fc9792cd398755a97 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 9 Mar 2022 18:08:02 +0800 Subject: [PATCH 25/39] [TD-13761]: redefine dir api. --- source/os/src/osSocket.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 24db2d7c5f..38f933dcbf 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -827,6 +827,7 @@ int32_t taosGetFqdn(char *fqdn) { int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + assert(0); return -1; } From bd3451a8bd068a8ecdd41522be5f57fe45b210d9 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 9 Mar 2022 18:08:26 +0800 Subject: [PATCH 26/39] [TD-13761]: redefine dir api. --- source/os/src/osSocket.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 38f933dcbf..698ceded16 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -810,6 +810,7 @@ int32_t taosGetFqdn(char *fqdn) { hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { printf("failed to get hostname, reason:%s", strerror(errno)); + assert(0); return -1; } From a455105de39c1e38feb0aceb06a1bc6d297a1171 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 9 Mar 2022 18:33:23 +0800 Subject: [PATCH 27/39] mark expired query window by SSmaStat --- source/dnode/vnode/src/tsdb/tsdbSma.c | 35 ++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 5cfa597eb2..c5f1261282 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -86,7 +86,12 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWin static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { ASSERT(pSmaStat != NULL); - // TODO: lock and create when each put, or create during tsdbNew. + + if (*pSmaStat != NULL) { // no lock + return TSDB_CODE_SUCCESS; + } + + // TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew. if (*pSmaStat == NULL) { *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); if (*pSmaStat == NULL) { @@ -149,6 +154,8 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + tsdbInitSmaStat(&pTsdb->pSmaStat); // lazy mode + // TODO: decode the msg => start const char * indexName = SMA_TEST_INDEX_NAME; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; @@ -157,8 +164,8 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { expiredWindows[i] = now + i; } - // TODO: decode the msg <= end + // TODO: decode the msg <= end SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems; SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName)); @@ -181,8 +188,12 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { int8_t state = TSDB_SMA_STAT_EXPIRED; for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) { - // If error occurs during put expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would tell - // query module to query raw TS data. + // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. taosHashCleanup(pItem->expiredWindows); taosHashRemove(pItemsHash, indexName, sizeof(indexName)); return TSDB_CODE_FAILED; @@ -612,6 +623,22 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow * @return int32_t */ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) { + const char *indexName = param->indexName; + + SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName)); + if (pItem == NULL) { + // mark all window as expired and notify query module to query raw TS data. + return TSDB_CODE_SUCCESS; + } + + int32_t nQueryWin = 0; + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY thisWindow = n; + if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) { + // TODO: mark this window as expired. + } + } + STSmaReadH tReadH = {0}; tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData); From 221a9bf97a36544d45de5bca2aa251b9448fba52 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 9 Mar 2022 18:36:13 +0800 Subject: [PATCH 28/39] revert --- include/common/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 7788dfd871..99360b8d3f 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -50,7 +50,7 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; -typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2} TDUpdateConfig; +typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; typedef enum { TSDB_STATIS_OK = 0, // statis part exist and load successfully TSDB_STATIS_NONE = 1, // statis part not exist From 913e8d70d8738047bb3b826c1c5b12d91f24a130 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 9 Mar 2022 10:49:42 +0800 Subject: [PATCH 29/39] define stream token --- source/dnode/vnode/inc/vnode.h | 18 +++++++++++++++++- source/dnode/vnode/src/inc/tqPush.h | 13 ++++++++----- source/dnode/vnode/src/tq/tq.c | 20 ++++++++++++++++++++ source/dnode/vnode/src/tq/tqPush.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/wal/src/walWrite.c | 3 --- source/libs/wal/test/walMetaTest.cpp | 22 +++++++++++----------- 7 files changed, 58 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 319bd9fde6..b6dd90a4e2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -59,7 +59,7 @@ typedef struct { SWalCfg walCfg; uint32_t hashBegin; uint32_t hashEnd; - int8_t hashMethod; + int8_t hashMethod; } SVnodeCfg; typedef struct { @@ -202,6 +202,22 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); /* ------------------------- TQ READ --------------------------- */ +enum { + TQ_STREAM_TOKEN__DATA = 1, + TQ_STREAM_TOKEN__WATERMARK, + TQ_STREAM_TOKEN__CHECKPOINT, +}; + +typedef struct { + int8_t type; + int8_t reserved[7]; + union { + void *data; + int64_t wmTs; + int64_t checkpointId; + }; +} STqStreamToken; + STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) { diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h index 32fd7c3ddf..a6121c5dc1 100644 --- a/source/dnode/vnode/src/inc/tqPush.h +++ b/source/dnode/vnode/src/inc/tqPush.h @@ -16,9 +16,11 @@ #ifndef _TQ_PUSH_H_ #define _TQ_PUSH_H_ +#include "executor.h" #include "thash.h" #include "trpc.h" #include "ttimer.h" +#include "vnode.h" #ifdef __cplusplus extern "C" { @@ -39,11 +41,12 @@ typedef struct { } STqClientPusher; typedef struct { - int8_t type; - int8_t nodeType; - int8_t reserved[6]; - int64_t streamId; - SEpSet epSet; + int8_t type; + int8_t nodeType; + int8_t reserved[6]; + int64_t streamId; + qTaskInfo_t task; + // TODO sync function } STqStreamPusher; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 16809f1527..d15481b4aa 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -67,6 +67,26 @@ void tqClose(STQ* pTq) { } int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + if (msgType != TDMT_VND_SUBMIT) return 0; + void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); + while (pIter != NULL) { + STqPusher* pusher = *(STqPusher**)pIter; + if (pusher->type == TQ_PUSHER_TYPE__STREAM) { + STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; + // repack + STqStreamToken* token = malloc(sizeof(STqStreamToken)); + if (token == NULL) { + taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + token->type = TQ_STREAM_TOKEN__DATA; + token->data = msg; + // set input + // exec + } + // send msg to ep + } // iterate hash // process all msg // if waiting diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index fea65846be..4186f29e2a 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -73,7 +73,7 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet streamPusher->type = TQ_PUSHER_TYPE__STREAM; streamPusher->nodeType = 0; streamPusher->streamId = streamId; - memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet)); + /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/ if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2f24df0309..92a111298f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "vnode.h" @@ -37,6 +36,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t pMsg->length = htonl(pMsg->length); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // iterate and convert if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 699da75790..0f6ac0b214 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -34,9 +34,6 @@ int32_t walCommit(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) { int code; char fnameStr[WAL_FILE_LEN]; - if (ver == pWal->vers.lastVer) { - return 0; - } if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 230555e016..5bfea9ab5e 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -124,13 +124,8 @@ class WalRetentionEnv : public ::testing::Test { void SetUp() override { SWalCfg cfg; - cfg.rollPeriod = -1, - cfg.segSize = -1, - cfg.retentionPeriod = -1, - cfg.retentionSize = 0, - cfg.rollPeriod = 0, - cfg.vgId = 0, - cfg.level = TAOS_WAL_FSYNC; + cfg.rollPeriod = -1, cfg.segSize = -1, cfg.retentionPeriod = -1, cfg.retentionSize = 0, cfg.rollPeriod = 0, + cfg.vgId = 0, cfg.level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, &cfg); ASSERT(pWal != NULL); } @@ -241,6 +236,12 @@ TEST_F(WalCleanEnv, rollback) { ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); code = walRollback(pWal, 5); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, 4); @@ -324,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) { TEST_F(WalRetentionEnv, repairMeta1) { walResetEnv(); int code; - + int i; for (i = 0; i < 100; i++) { char newStr[100]; @@ -336,14 +337,14 @@ TEST_F(WalRetentionEnv, repairMeta1) { TearDown(); - //getchar(); + // getchar(); char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); taosRemoveFile(buf); sprintf(buf, "%s/meta-ver%d", pathName, 1); taosRemoveFile(buf); SetUp(); - //getchar(); + // getchar(); ASSERT_EQ(pWal->vers.lastVer, 99); @@ -401,5 +402,4 @@ TEST_F(WalRetentionEnv, repairMeta1) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } - } From 8615c97fc6f3dc07a014d558d0c3a2e0e2a34dad Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Mar 2022 06:53:00 -0500 Subject: [PATCH 30/39] TD-13704 super table plan split --- include/libs/nodes/plannodes.h | 7 +- include/libs/planner/planner.h | 12 +- source/libs/planner/src/physicalPlan.c | 165 +++++++++++-------------- source/libs/planner/src/planner.c | 57 +++++++-- 4 files changed, 125 insertions(+), 116 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 084d642292..e52c313b8d 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -105,7 +105,8 @@ typedef struct SSubLogicPlan { typedef struct SQueryLogicPlan { ENodeType type;; - SNodeList* pSubplans; + int32_t totalLevel; + SNodeList* pTopSubplans; } SQueryLogicPlan; typedef struct SSlotDescNode { @@ -221,8 +222,8 @@ typedef struct SSubplan { typedef struct SQueryPlan { ENodeType type;; uint64_t queryId; - int32_t numOfSubplans; - SNodeList* pSubplans; // SNodeListNode. The execution level of subplan, starting from 0. + int32_t numOfSubplans; + SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. } SQueryPlan; #ifdef __cplusplus diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 5d6ec46d85..70e35824b6 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -31,14 +31,14 @@ typedef struct SPlanContext { int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList); // Set datasource of this subplan, multiple calls may be made to a subplan. -// @subplan subplan to be schedule -// @groupId id of a group of datasource subplans of this @subplan -// @ep one execution location of this group of datasource subplans -void qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource); +// @pSubplan subplan to be schedule +// @groupId id of a group of datasource subplans of this @pSubplan +// @pSource one execution location of this group of datasource subplans +int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); // Convert to subplan to string for the scheduler to send to the executor -int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len); -int32_t qStringToSubplan(const char* str, SSubplan** subplan); +int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); +int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan); char* qQueryPlanToString(const SQueryPlan* pPlan); SQueryPlan* qStringToQueryPlan(const char* pStr); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index b0c087b36a..265aac40fb 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -28,7 +28,6 @@ typedef struct SPhysiPlanContext { int16_t nextDataBlockId; SArray* pLocationHelper; SArray* pExecNodeList; - int32_t groupId; int32_t subplanId; } SPhysiPlanContext; @@ -537,6 +536,8 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubpl SSubplan* pSubplan = nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); CHECK_ALLOC(pSubplan, NULL); pSubplan->id = pLogicSubplan->id; + pSubplan->subplanType = pLogicSubplan->subplanType; + pSubplan->level = pLogicSubplan->level; return pSubplan; } @@ -554,8 +555,6 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); pSubplan->msgType = TDMT_VND_QUERY; } - pSubplan->subplanType = pLogicSubplan->subplanType; - pSubplan->level = pLogicSubplan->level; return pSubplan; } @@ -603,7 +602,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l return TSDB_CODE_SUCCESS; } -SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { +static SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { SSubLogicPlan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); CHECK_ALLOC(pDst, NULL); pDst->pNode = nodesCloneNode(pSrc->pNode); @@ -613,13 +612,13 @@ SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* p } pDst->subplanType = pSrc->subplanType; pDst->level = level; - pDst->id.queryId = pCxt->pPlanCxt->queryId; - pDst->id.groupId = pCxt->groupId; + pDst->id.queryId = pSrc->id.queryId; + pDst->id.groupId = pSrc->id.groupId; pDst->id.subplanId = pCxt->subplanId++; return pDst; } -static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { +static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); for (int32_t i = 0; i < numOfVgroups; ++i) { @@ -627,17 +626,12 @@ static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubpla CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; - // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); } return TSDB_CODE_SUCCESS; } -static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { - // SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - // CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); - // return TSDB_CODE_SUCCESS; +static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); } @@ -665,40 +659,60 @@ static int32_t setScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const S return doSetScanVgroup(pCxt, pNode, pVgroup, &found); } -static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SNodeList* pGroup) { +static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { if (pSubplan->pVgroupList) { for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); CHECK_CODE_EXT(setScanVgroup(pCxt, pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i)); - // CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans)); CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); } return TSDB_CODE_SUCCESS; } else { - return scaleOutForMerge(pCxt, pSubplan, level, pLogicPlan, pGroup); + return scaleOutForMerge(pCxt, pSubplan, level, pGroup); } } -// static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup, int32_t level, SQueryLogicPlan* pLogicPlan) { -// FOREACH() { +static int32_t appendWithMakeList(SNodeList** pList, SNodeptr pNode) { + if (NULL == *pList) { + *pList = nodesMakeList(); + if (NULL == *pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return nodesListAppend(*pList, pNode); +} -// } -// } +static int32_t pushHierarchicalPlan(SPhysiPlanContext* pCxt, SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); + SNode* pChild = NULL; + FOREACH(pChild, pCurrentGroup) { + if (topLevel) { + CHECK_CODE_EXT(nodesListAppend(pParentsGroup, pChild)); + } else { + SNode* pParent = NULL; + FOREACH(pParent, pParentsGroup) { + CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pParent)->pChildren), pChild)); + CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pChild)->pParents), pParent)); + } + } + } + return TSDB_CODE_SUCCESS; +} -static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan, SHashObj* pHash, SNodeList* pParentsGroup) { +static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { SNodeList* pCurrentGroup = nodesMakeList(); CHECK_ALLOC(pCurrentGroup, TSDB_CODE_OUT_OF_MEMORY); int32_t code = TSDB_CODE_SUCCESS; switch (pSubplan->subplanType) { case SUBPLAN_TYPE_MERGE: - code = scaleOutForMerge(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); break; case SUBPLAN_TYPE_SCAN: - code = scaleOutForScan(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); break; case SUBPLAN_TYPE_MODIFY: - code = scaleOutForModify(pCxt, pSubplan, level, pLogicPlan, pCurrentGroup); + code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); break; default: break; @@ -706,12 +720,12 @@ static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int3 if (TSDB_CODE_SUCCESS != code) { return code; } - // pushHierarchicalPlan(pParentsGroup, pCurrentGroup, level, pLogicPlan); - CHECK_CODE(taosHashPut(pHash, &pCxt->groupId, sizeof(pCxt->groupId), &pSubplan->id.groupId, sizeof(pSubplan->id.groupId)), TSDB_CODE_OUT_OF_MEMORY); - ++(pCxt->groupId); + + CHECK_CODE_EXT(pushHierarchicalPlan(pCxt, pParentsGroup, pCurrentGroup)); + ++(*pLevel); SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, level + 1, pLogicPlan, pHash, pCurrentGroup)); + CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, pLevel, pCurrentGroup)); } return TSDB_CODE_SUCCESS; @@ -720,75 +734,18 @@ static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int3 static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); CHECK_ALLOC(pLogicPlan, NULL); - pLogicPlan->pSubplans = nodesMakeList(); - if (NULL == pLogicPlan->pSubplans) { + pLogicPlan->pTopSubplans = nodesMakeList(); + if (NULL == pLogicPlan->pTopSubplans) { nodesDestroyNode(pLogicPlan); return NULL; } return pLogicPlan; } -static int32_t doMappingLogicPlan(SLogicNode* pNode, SHashObj* pHash) { - if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pNode)) { - SExchangeLogicNode* pExchange = (SExchangeLogicNode*)pNode; - int32_t* pGroupId = taosHashGet(pHash, &pExchange->srcGroupId, sizeof(pExchange->srcGroupId)); - if (NULL == pGroupId) { - return TSDB_CODE_FAILED; - } - pExchange->srcGroupId = *pGroupId; - return TSDB_CODE_SUCCESS; - } - - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { - doMappingLogicPlan((SLogicNode*)pChild, pHash); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t mappingLogicPlan(SQueryLogicPlan* pLogicPlan, SHashObj* pHash) { - SNode* pNode = NULL; - FOREACH(pNode, pLogicPlan->pSubplans) { - SNode* pSubplan = NULL; - FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { - int32_t code = doMappingLogicPlan(((SSubLogicPlan*)pSubplan)->pNode, pHash); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) { *pLogicPlan = makeQueryLogicPlan(pCxt); CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - SHashObj* pHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); - int32_t code = doScaleOut(pCxt, pRootSubLogicPlan, 0, *pLogicPlan, pHash, NULL); - if (TSDB_CODE_SUCCESS == code) { - code = mappingLogicPlan(*pLogicPlan, pHash); - } - taosHashCleanup(pHash); - return code; -} - -typedef struct SBuildPhysiSubplanCxt { - int32_t errCode; - SQueryPlan* pQueryPlan; - SPhysiPlanContext* pPhyCxt; -} SBuildPhysiSubplanCxt; - -static EDealRes doBuildPhysiSubplan(SNode* pNode, void* pContext) { - SBuildPhysiSubplanCxt* pCxt = (SBuildPhysiSubplanCxt*)pContext; - if (QUERY_NODE_LOGIC_SUBPLAN == nodeType(pNode)) { - SSubplan* pSubplan = createPhysiSubplan(pCxt->pPhyCxt, (SSubLogicPlan*)pNode); - CHECK_ALLOC(pSubplan, DEAL_RES_ERROR); - CHECK_CODE(pushSubplan(pCxt->pPhyCxt, pSubplan, ((SSubLogicPlan*)pNode)->level, pCxt->pQueryPlan->pSubplans), DEAL_RES_ERROR); - ++(pCxt->pQueryPlan->numOfSubplans); - return DEAL_RES_IGNORE_CHILD; - } - return DEAL_RES_CONTINUE; + return doScaleOut(pCxt, pRootSubLogicPlan, &((*pLogicPlan)->totalLevel), (*pLogicPlan)->pTopSubplans); } static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { @@ -803,15 +760,31 @@ static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { return pPlan; } -static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { - SBuildPhysiSubplanCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pQueryPlan = makeQueryPhysiPlan(pCxt), .pPhyCxt = pCxt }; - CHECK_ALLOC(cxt.pQueryPlan, TSDB_CODE_OUT_OF_MEMORY); - nodesWalkList(pLogicPlan->pSubplans, doBuildPhysiSubplan, &cxt); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyNode(cxt.pQueryPlan); - return cxt.errCode; +static int32_t doBuildPhysiPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { + SSubplan* pSubplan = createPhysiSubplan(pCxt, pLogicSubplan); + CHECK_ALLOC(pSubplan, DEAL_RES_ERROR); + CHECK_CODE_EXT(pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans)); + ++(pQueryPlan->numOfSubplans); + if (NULL != pParent) { + CHECK_CODE_EXT(appendWithMakeList(&pParent->pChildren, pSubplan)); + CHECK_CODE_EXT(appendWithMakeList(&pSubplan->pParents, pParent)); + } + + SNode* pChild = NULL; + FOREACH(pChild, pLogicSubplan->pChildren) { + CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pChild, pSubplan, pQueryPlan)); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { + *pPlan = makeQueryPhysiPlan(pCxt); + CHECK_ALLOC(*pPlan, TSDB_CODE_OUT_OF_MEMORY); + SNode* pSubplan = NULL; + FOREACH(pSubplan, pLogicPlan->pTopSubplans) { + CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pSubplan, NULL, *pPlan)); } - *pPlan = cxt.pQueryPlan; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 4fa9b6ba10..15d5c09f04 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -34,31 +34,66 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo return code; } -void qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { +static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDownstreamSourceNode* pSource) { + if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pNode)) { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode; + if (pExchange->srcGroupId == groupId) { + if (NULL == pExchange->pSrcEndPoints) { + pExchange->pSrcEndPoints = nodesMakeList(); + if (NULL == pExchange->pSrcEndPoints) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pExchange->pSrcEndPoints, nodesCloneNode(pSource))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; + } + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + if (TSDB_CODE_SUCCESS != setSubplanExecutionNode((SPhysiNode*)pChild, groupId, pSource)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return TSDB_CODE_SUCCESS; } -int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len) { - if (SUBPLAN_TYPE_MODIFY == subplan->subplanType) { - SDataInserterNode* insert = (SDataInserterNode*)subplan->pDataSink; - *len = insert->size; - *str = insert->pData; +int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { + return setSubplanExecutionNode(subplan->pNode, groupId, pSource); +} + +int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { + if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) { + SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink; + *pLen = insert->size; + *pStr = insert->pData; insert->pData = NULL; return TSDB_CODE_SUCCESS; } - return nodesNodeToString((const SNode*)subplan, false, str, len); + return nodesNodeToString((const SNode*)pSubplan, false, pStr, pLen); } -int32_t qStringToSubplan(const char* str, SSubplan** subplan) { - return nodesStringToNode(str, (SNode**)subplan); +int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan) { + return nodesStringToNode(pStr, (SNode**)pSubplan); } char* qQueryPlanToString(const SQueryPlan* pPlan) { - + char* pStr = NULL; + int32_t len = 0; + if (TSDB_CODE_SUCCESS != nodesNodeToString(pPlan, false, &pStr, &len)) { + return NULL; + } + return pStr; } SQueryPlan* qStringToQueryPlan(const char* pStr) { - + SQueryPlan* pPlan = NULL; + if (TSDB_CODE_SUCCESS != nodesStringToNode(pStr, (SNode**)&pPlan)) { + return NULL; + } + return pPlan; } void qDestroyQueryPlan(SQueryPlan* pPlan) { From c6de1967698180061767a59bd7a6d656c8c5ae11 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 10 Mar 2022 00:36:30 +0800 Subject: [PATCH 31/39] [TD-13765]: redefine rand api. --- contrib/test/craft/raftMain.c | 2 +- contrib/test/craft/simulate_vnode.c | 4 +- examples/c/schemaless.c | 2 +- include/os/osRand.h | 9 ++++ source/client/src/clientEnv.c | 2 +- source/dnode/vnode/test/tqMetaTest.cpp | 18 +++---- source/libs/catalog/test/catalogTests.cpp | 10 ++-- source/libs/executor/src/executorimpl.c | 6 +-- source/libs/executor/test/executorTests.cpp | 8 +-- source/libs/executor/test/lhashTests.cpp | 2 +- source/libs/index/test/indexTests.cc | 2 +- source/libs/qworker/test/qworkerTests.cpp | 50 +++++++++---------- .../libs/scalar/test/filter/filterTests.cpp | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- source/libs/scheduler/test/schedulerTests.cpp | 4 +- source/libs/sync/src/syncEnv.c | 2 +- source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/src/syncUtil.c | 2 +- source/libs/tdb/src/db/tdbUtil.c | 2 +- source/libs/transport/src/rpcMain.c | 2 +- source/libs/wal/test/walMetaTest.cpp | 6 +-- source/os/src/osEnv.c | 2 +- source/os/src/osRand.c | 6 ++- source/util/src/tdes.c | 2 +- source/util/src/tskiplist.c | 8 +-- source/util/test/codingTests.cpp | 4 +- source/util/test/pageBufferTest.cpp | 2 +- source/util/test/skiplistTest.cpp | 8 +-- 28 files changed, 92 insertions(+), 79 deletions(-) diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index b28adfaaca..12be3deb2e 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -377,7 +377,7 @@ void printConf(SRaftServerConfig *pConf) { int main(int argc, char **argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); int32_t ret; exe_name = argv[0]; diff --git a/contrib/test/craft/simulate_vnode.c b/contrib/test/craft/simulate_vnode.c index 668fe638b7..7ee9b9f8f0 100644 --- a/contrib/test/craft/simulate_vnode.c +++ b/contrib/test/craft/simulate_vnode.c @@ -132,7 +132,7 @@ static void proposeValue(struct raft *r) { buf.base = raft_malloc(buf.len); // mock ts value - int vid = rand() % VNODE_COUNT; + int vid = taosRand() % VNODE_COUNT; snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL)); printf("propose value: %s \n", (char*)buf.base); @@ -174,7 +174,7 @@ void usage() { } int main(int argc, char **argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); exe_name = argv[0]; if (argc < 2) { diff --git a/examples/c/schemaless.c b/examples/c/schemaless.c index 21f39213cd..99aa361b0a 100644 --- a/examples/c/schemaless.c +++ b/examples/c/schemaless.c @@ -19,7 +19,7 @@ void shuffle(char**lines, size_t n) size_t i; for (i = 0; i < n - 1; i++) { - size_t j = i + rand() / (RAND_MAX / (n - i) + 1); + size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1); char* t = lines[j]; lines[j] = lines[i]; lines[i] = t; diff --git a/include/os/osRand.h b/include/os/osRand.h index 422ea92a71..09e1f1b41d 100644 --- a/include/os/osRand.h +++ b/include/os/osRand.h @@ -20,7 +20,16 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define rand RAND_FUNC_TAOS_FORBID + #define srand SRAND_FUNC_TAOS_FORBID + #define rand_r RANDR_FUNC_TAOS_FORBID +#endif + +void taosSeedRand(uint32_t seed); uint32_t taosRand(void); +uint32_t taosRandR(uint32_t *pSeed); void taosRandStr(char* str, int32_t size); uint32_t taosSafeRand(void); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 08285c9d26..f5cc034e09 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -208,7 +208,7 @@ void taos_init_imp(void) { atexit(taos_cleanup); errno = TSDB_CODE_SUCCESS; - srand(taosGetTimestampSec()); + taosSeedRand(taosGetTimestampSec()); deltaToUtcInitOnce(); diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index d3c9b50e4a..4f15185254 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -168,10 +168,10 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { } TEST_F(TqMetaUpdateAppendTest, multiplePage) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -202,10 +202,10 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) { } TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -213,14 +213,14 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } for (int i = 500; i < 1000; i++) { - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -235,7 +235,7 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { ASSERT(pMeta); for (int i = 500; i < 1000; i++) { - v[i] = rand(); + v[i] = taosRand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); @@ -250,10 +250,10 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { } TEST_F(TqMetaUpdateAppendTest, dupCommit) { - srand(0); + taosSeedRand(0); std::vector v; for (int i = 0; i < 1000; i++) { - v.push_back(rand()); + v.push_back(taosRand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index c7867c4da5..00f6a508b7 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -723,7 +723,7 @@ void *ctgTestGetDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); @@ -747,7 +747,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -771,7 +771,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -801,7 +801,7 @@ void *ctgTestGetCtableMetaThread(void *param) { tfree(tbMeta); if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { @@ -838,7 +838,7 @@ void *ctgTestSetCtableMetaThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand() % 5); + usleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index eeb48a1f3d..e30b51cbdf 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -61,7 +61,7 @@ typedef enum SResultTsInterpType { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 1000 <= 0) { return NULL; @@ -71,7 +71,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { } static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 1000 <= 0) { return NULL; } else { @@ -80,7 +80,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { } static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { - uint32_t v = rand(); + uint32_t v = taosRand(); if (v % 5 <= 1) { return NULL; } else { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index ff29d5f355..72ef13124a 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -869,7 +869,7 @@ TEST(testCase, external_sort_Test) { #if 0 su* v = static_cast(calloc(1000000, sizeof(su))); for(int32_t i = 0; i < 1000000; ++i) { - v[i].v = rand(); + v[i].v = taosRand(); v[i].c = static_cast(malloc(4)); *(int32_t*) v[i].c = i; } @@ -882,7 +882,7 @@ TEST(testCase, external_sort_Test) { return; #endif - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; @@ -943,7 +943,7 @@ TEST(testCase, external_sort_Test) { } TEST(testCase, sorted_merge_Test) { - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; @@ -1015,7 +1015,7 @@ TEST(testCase, sorted_merge_Test) { } TEST(testCase, time_interval_Operator_Test) { - srand(time(NULL)); + taosSeedRand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {0}; diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp index 66ef3b0877..88cf713727 100644 --- a/source/libs/executor/test/lhashTests.cpp +++ b/source/libs/executor/test/lhashTests.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" TEST(testCase, linear_hash_Tests) { - srand(time(NULL)); + taosSeedRand(time(NULL)); _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); #if 0 diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ce3f7fe25e..699c785be5 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -699,7 +699,7 @@ class IndexObj { for (int i = 0; i < numOfTable; i++) { for (int k = 0; k < 10 && k < colVal.size(); k++) { // opt - tColVal[rand() % colValSize] = 'a' + k % 26; + tColVal[taosRand() % colValSize] = 'a' + k % 26; } SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 231f0c7fff..8658c4cead 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -266,7 +266,7 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum); qwtTestSinkBlockNum = 0; - qwtTestSinkMaxBlockNum = rand() % 100 + 1; + qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; if (0 == idx) { @@ -295,15 +295,15 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } else { if (qwtTestSinkQueryEnd) { *pRes = NULL; - *useconds = rand() % 10; + *useconds = taosRand() % 10; return 0; } - endExec = rand() % 5; + endExec = taosRand() % 5; int32_t runTime = 0; if (qwtTestEnableSleep && qwtTestMaxExecTaskUsec > 0) { - runTime = rand() % qwtTestMaxExecTaskUsec; + runTime = taosRand() % qwtTestMaxExecTaskUsec; } if (qwtTestEnableSleep) { @@ -314,10 +314,10 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (endExec) { *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); - (*pRes)->info.rows = rand() % 1000; + (*pRes)->info.rows = taosRand() % 1000; } else { *pRes = NULL; - *useconds = rand() % 10; + *useconds = taosRand() % 10; } } @@ -376,7 +376,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { taosWLockLatch(&qwtTestSinkLock); if (qwtTestSinkBlockNum > 0) { - *pLen = rand() % 100 + 1; + *pLen = taosRand() % 100 + 1; qwtTestSinkBlockNum--; } else { *pLen = 0; @@ -392,7 +392,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { taosWLockLatch(&qwtTestSinkLock); if (qwtTestSinkLastLen > 0) { - pOutput->numOfRows = rand() % 10 + 1; + pOutput->numOfRows = taosRand() % 10 + 1; pOutput->compressed = 1; pOutput->queryEnd = qwtTestSinkQueryEnd; if (qwtTestSinkBlockNum == 0) { @@ -402,7 +402,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { } else { pOutput->bufStatus = DS_BUF_FULL; } - pOutput->useconds = rand() % 10 + 1; + pOutput->useconds = taosRand() % 10 + 1; pOutput->precision = 1; } else if (qwtTestSinkLastLen == 0) { pOutput->numOfRows = 0; @@ -416,7 +416,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { } else { pOutput->bufStatus = DS_BUF_FULL; } - pOutput->useconds = rand() % 10 + 1; + pOutput->useconds = taosRand() % 10 + 1; pOutput->precision = 1; } else { assert(0); @@ -590,7 +590,7 @@ void *queryThread(void *param) { qwtBuildQueryReqMsg(&queryRpc); qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); @@ -612,7 +612,7 @@ void *readyThread(void *param) { qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("ready:%d\n", n); @@ -634,7 +634,7 @@ void *fetchThread(void *param) { qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("fetch:%d\n", n); @@ -656,7 +656,7 @@ void *dropThread(void *param) { qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("drop:%d\n", n); @@ -678,7 +678,7 @@ void *statusThread(void *param) { qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); if (qwtTestEnableSleep) { - usleep(rand()%5); + usleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("status:%d\n", n); @@ -748,7 +748,7 @@ void *queryQueueThread(void *param) { if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { - int32_t delay = rand() % qwtTestReqMaxDelayUsec; + int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { usleep(delay); @@ -804,7 +804,7 @@ void *fetchQueueThread(void *param) { taosWUnLockLatch(&qwtTestFetchQueueLock); if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { - int32_t delay = rand() % qwtTestReqMaxDelayUsec; + int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { usleep(delay); @@ -963,7 +963,7 @@ TEST(seqTest, randCase) { stubSetRpcSendResponse(); stubSetCreateExecTask(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -971,7 +971,7 @@ TEST(seqTest, randCase) { int32_t t = 0; int32_t maxr = 10001; while (true) { - int32_t r = rand() % maxr; + int32_t r = taosRand() % maxr; if (r >= 0 && r < maxr/5) { printf("Query,%d\n", t++); @@ -1025,7 +1025,7 @@ TEST(seqTest, multithreadRand) { stubSetStringToPlan(); stubSetRpcSendResponse(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -1076,7 +1076,7 @@ TEST(rcTest, shortExecshortDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1157,7 +1157,7 @@ TEST(rcTest, longExecshortDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1240,7 +1240,7 @@ TEST(rcTest, shortExeclongDelay) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); qwtTestStop = false; qwtTestQuitThreadNum = 0; @@ -1324,7 +1324,7 @@ TEST(rcTest, dropTest) { stubSetPutDataBlock(); stubSetGetDataBlock(); - srand(time(NULL)); + taosSeedRand(time(NULL)); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -1358,7 +1358,7 @@ TEST(rcTest, dropTest) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 08210aa2f0..c69a1eb247 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -1286,7 +1286,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index faf13f0a82..6518dbec87 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1427,7 +1427,7 @@ TEST(columnTest, greater_and_lower) { int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 8ed963d875..f17dcbd1f5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -532,7 +532,7 @@ void* schtRunJobThread(void *aa) { void* schtFreeJobThread(void *aa) { while (!schtTestStop) { - usleep(rand() % 100); + usleep(taosRand() % 100); schtFreeQueryJob(1); } } @@ -713,7 +713,7 @@ TEST(multiThread, forceFree) { } int main(int argc, char** argv) { - srand(time(NULL)); + taosSeedRand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 6917df1597..fa58bda76f 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -28,7 +28,7 @@ static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer); int32_t syncEnvStart() { int32_t ret; - srand(time(NULL)); + taosSeedRand(time(NULL)); gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); assert(gSyncEnv != NULL); ret = doSyncEnvStart(gSyncEnv); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index d37c821a24..7cb42c9f87 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -44,7 +44,7 @@ int32_t syncIOStart(char *host, uint16_t port) { gSyncIO = syncIOCreate(host, port); assert(gSyncIO != NULL); - srand(time(NULL)); + taosSeedRand(time(NULL)); int32_t ret = syncIOStartInternal(gSyncIO); assert(ret == 0); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 3fb430a714..216dccd62c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -95,7 +95,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { // ---- misc ---- -int32_t syncUtilRand(int32_t max) { return rand() % max; } +int32_t syncUtilRand(int32_t max) { return taosRand() % max; } int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index fe0f3befd6..237a39e47d 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -27,7 +27,7 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { ((uint64_t *)fileid)[0] = stDev; ((uint64_t *)fileid)[1] = stIno; if (unique) { - ((uint64_t *)fileid)[2] = rand(); + ((uint64_t *)fileid)[2] = taosRand(); } return 0; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index e1319da162..615c576a9b 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -749,7 +749,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { memcpy(pConn->user, pHead->user, tListLen(pConn->user)); pConn->pRpc = pRpc; pConn->sid = sid; - pConn->tranId = (uint16_t)(rand() & 0xFFFF); + pConn->tranId = (uint16_t)(taosRand() & 0xFFFF); pConn->ownId = htonl(pConn->sid); pConn->linkUid = pHead->linkUid; if (pRpc->afp) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 5bfea9ab5e..f44fc71964 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -300,7 +300,7 @@ TEST_F(WalKeepEnv, readHandleRead) { ASSERT_EQ(code, 0); } for (int i = 0; i < 1000; i++) { - int ver = rand() % 100; + int ver = taosRand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); @@ -352,7 +352,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT(pRead != NULL); for (int i = 0; i < 1000; i++) { - int ver = rand() % 100; + int ver = taosRand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); @@ -382,7 +382,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { } for (int i = 0; i < 1000; i++) { - int ver = rand() % 200; + int ver = taosRand() % 200; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 0b2fe904b3..63fa600217 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -38,7 +38,7 @@ float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; void osInit() { - srand(taosSafeRand()); + taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemTimezone(tsTimezone); taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index b81e41b3cf..f3dd9b74c5 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -21,8 +21,12 @@ #include #endif +void taosSeedRand(uint32_t seed) { return srand(seed); } + uint32_t taosRand(void) { return rand(); } +uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); } + uint32_t taosSafeRand(void) { TdFilePtr pFile; int seed; diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index 105dd7f95f..d12b47efe8 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -32,7 +32,7 @@ void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* #if 0 int64_t taosDesGenKey() { uint32_t iseed = (uint32_t)time(NULL); - srand(iseed); + taosSeedRand(iseed); uint8_t key[8] = {0}; generate_key(key); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 6b89ed2c43..d9d6e4e3da 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -51,7 +51,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->len = keyLen; pSkipList->flags = flags; pSkipList->keyFn = fn; - pSkipList->seed = rand(); + pSkipList->seed = taosRand(); #if 0 // the function getkeycomparfunc is defined in common @@ -82,7 +82,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } } - srand((uint32_t)time(NULL)); + taosSeedRand((uint32_t)time(NULL)); #if SKIP_LIST_RECORD_PERFORMANCE pSkipList->state.nTotalMemSize += sizeof(SSkipList); @@ -560,9 +560,9 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { int32_t n = 1; #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) { + while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) { #else - while ((rand_r(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { + while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { #endif n++; } diff --git a/source/util/test/codingTests.cpp b/source/util/test/codingTests.cpp index 0cd9524646..b991411047 100644 --- a/source/util/test/codingTests.cpp +++ b/source/util/test/codingTests.cpp @@ -150,7 +150,7 @@ static bool test_variant_int64(int64_t value) { } TEST(codingTest, fixed_encode_decode) { - srand(time(0)); + taosSeedRand(time(0)); // uint16_t for (uint16_t value = 0; value <= UINT16_MAX; value++) { @@ -204,7 +204,7 @@ TEST(codingTest, fixed_encode_decode) { } TEST(codingTest, variant_encode_decode) { - srand(time(0)); + taosSeedRand(time(0)); // uint16_t for (uint16_t value = 0; value <= UINT16_MAX; value++) { diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index f392aac7d1..e63e6f04a1 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -161,7 +161,7 @@ void recyclePageTest() { TEST(testCase, resultBufferTest) { - srand(time(NULL)); + taosSeedRand(time(NULL)); simpleTest(); writeDownTest(); recyclePageTest(); diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index f2e696b0e5..f61ebfd890 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -47,7 +47,7 @@ void doubleSkipListTest() { SSkipListKey sk; for (int32_t i = 0; i < 100; ++i) { sk.nType = TSDB_DATA_TYPE_DOUBLE; - int32_t idx = abs((i * rand()) % 1000); + int32_t idx = abs((i * taosRand()) % 1000); sk.dKey = doubleVal[idx]; @@ -74,7 +74,7 @@ void randKeyTest() { false, getkey); int32_t size = 200000; - srand(time(NULL)); + taosSeedRand(time(NULL)); printf("generated %d keys is: \n", size); @@ -87,7 +87,7 @@ void randKeyTest() { d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); - key[0] = rand() % 1000000000; + key[0] = taosRand() % 1000000000; key[1] = key[0]; @@ -337,7 +337,7 @@ void duplicatedKeyTest() { TEST(testCase, skiplist_test) { assert(sizeof(SSkipListKey) == 8); - srand(time(NULL)); + taosSeedRand(time(NULL)); stringKeySkiplistTest(); doubleSkipListTest(); From 8345a5429bf30c1745cf3e1c4b117dd8c7e99e9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Mar 2022 11:12:44 +0800 Subject: [PATCH 32/39] [td-13039] add min/max/systable-scanner. --- include/libs/function/function.h | 12 +- source/libs/executor/inc/executorimpl.h | 32 ++- source/libs/executor/src/executorimpl.c | 159 +++++++------ source/libs/function/inc/builtinsimpl.h | 6 + source/libs/function/src/builtins.c | 20 ++ source/libs/function/src/builtinsimpl.c | 280 +++++++++++++++++++++- source/libs/function/src/taggfunction.c | 296 +++++------------------- 7 files changed, 463 insertions(+), 342 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 21013ef906..c01e267c42 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -132,11 +132,11 @@ struct SqlFunctionCtx; struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified -typedef struct SExtTagsInfo { - int16_t tagsLen; // keep the tags data for top/bottom query result - int16_t numOfTagCols; - struct SqlFunctionCtx **pTagCtxList; -} SExtTagsInfo; +typedef struct SSubsidiaryResInfo { + int16_t bufLen; // keep the tags data for top/bottom query result + int16_t numOfCols; + struct SqlFunctionCtx **pCtx; +} SSubsidiaryResInfo; typedef struct SResultDataInfo { int16_t precision; @@ -187,7 +187,7 @@ typedef struct SqlFunctionCtx { void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SVariant tag; struct SResultRowEntryInfo *resultInfo; - SExtTagsInfo tagInfo; + SSubsidiaryResInfo subsidiaryRes; SPoint1 start; SPoint1 end; SFuncExecFuncs fpSet; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d46978b26d..c8b8feb3af 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -441,17 +441,24 @@ typedef struct SStreamBlockScanInfo { } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { - void *pTransporter; - SEpSet epSet; - int32_t type; // show type - tsem_t ready; - void *readHandle; - SSchema *pSchema; - SSDataBlock *pRes; - int64_t numOfBlocks; // extract basic running information. - int64_t totalRows; - int64_t elapsedTime; - int64_t totalBytes; + union { + void* pTransporter; + void* readHandle; + }; + + void *pCur; // cursor + SRetrieveTableReq* pReq; + SEpSet epSet; + int32_t type; // show type + tsem_t ready; + SSchema* pSchema; + SSDataBlock* pRes; + + int32_t capacity; + int64_t numOfBlocks; // extract basic running information. + int64_t totalRows; + int64_t elapsedTime; + int64_t totalBytes; } SSysTableScanInfo; typedef struct SOptrBasicInfo { @@ -630,7 +637,8 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSystemScanOperatorInfo(void* pSystemTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, + int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6c710a5932..ab4eee4b86 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -212,6 +212,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); +static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; @@ -1920,9 +1921,9 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { } } if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; + p->subsidiaryRes.pCtx = pTagCtx; + p->subsidiaryRes.numOfCols = num; + p->subsidiaryRes.bufLen = tagLen; } else { tfree(pTagCtx); } @@ -2127,7 +2128,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC return pFuncCtx; } -static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -2138,7 +2139,7 @@ static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].tagInfo.pTagCtxList); + tfree(pCtx[i].subsidiaryRes.pCtx); } tfree(pCtx); @@ -2222,46 +2223,6 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { } } -static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; - - //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - - //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); -// destroyUdfInfo(pRuntimeEnv->pUdfInfo); - destroyDiskbasedBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pRuntimeEnv); - - destroyTsComp(pRuntimeEnv, pQueryAttr); - - pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - taosHashCleanup(pRuntimeEnv->pResultRowHashTable); - pRuntimeEnv->pResultRowHashTable = NULL; - - taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); - pRuntimeEnv->pTableRetrieveTsMap = NULL; - - taosHashCleanup(pRuntimeEnv->pResultRowListSet); - pRuntimeEnv->pResultRowListSet = NULL; - - destroyOperatorInfo(pRuntimeEnv->proot); - - pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); - taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); - taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); - pRuntimeEnv->prevResult = NULL; -} - -static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { - return pQInfo->rspContext != NULL; -} - bool isTaskKilled(SExecTaskInfo *pTaskInfo) { // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. @@ -5475,38 +5436,67 @@ static SSDataBlock* doSysTableScan(void* param, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; - SRetrieveTableReq* req = calloc(1, sizeof(SRetrieveTableReq)); - if (req == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + // retrieve local table list info from vnode + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); + } + + SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, 0); + + char * name = NULL; + int32_t numOfRows = 0; + while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { + colDataAppend(pTableNameCol, numOfRows, name, false); + numOfRows += 1; + if (numOfRows >= pInfo->capacity) { + break; + } + } + + pInfo->totalRows += numOfRows; + pInfo->pRes->info.rows = numOfRows; + +// pInfo->elapsedTime; +// pInfo->totalBytes; + return (pInfo->pRes->info.rows == 0)? NULL:pInfo->pRes; + } else { // load the meta from mnode of the given epset + if (pInfo->pReq == NULL) { + pInfo->pReq = calloc(1, sizeof(SRetrieveTableReq)); + if (pInfo->pReq == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pInfo->pReq->type = pInfo->type; + } + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pMsgSendInfo->param = NULL; + pMsgSendInfo->msgInfo.pData = pInfo->pReq; + pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); + + tsem_wait(&pInfo->ready); + // handle the response and return to the caller } - req->type = pInfo->type; - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pMsgSendInfo->param = NULL; - pMsgSendInfo->msgInfo.pData = req; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); - pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); - - tsem_wait(&pInfo->ready); - // handle the response and return to the caller - return NULL; } -SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, + int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = calloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5516,6 +5506,17 @@ SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SAr return NULL; } + // todo: create the schema of result data block + pInfo->capacity = 4096; + pInfo->type = tableType; + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + pInfo->readHandle = pSysTableReadHandle; + blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = epset; + } + pInfo->readHandle = pSysTableReadHandle; pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; @@ -5524,6 +5525,7 @@ SOperatorInfo* createSystemScanOperatorInfo(void* pSysTableReadHandle, const SAr pOperator->info = pInfo; pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->nextDataFn = doSysTableScan; + pOperator->closeFn = destroySysTableScannerOperatorInfo; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -7165,7 +7167,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); - destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); tfree(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); @@ -7185,6 +7187,7 @@ static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } + static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); @@ -7233,6 +7236,16 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } +static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*) param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + metaCloseTbCursor(pInfo->pCur); + } +} + SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 18d1ff41e2..7ba7d7bdcc 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -31,6 +31,12 @@ void countFunction(SqlFunctionCtx *pCtx); bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); void sumFunction(SqlFunctionCtx *pCtx); +bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +void minFunction(SqlFunctionCtx* pCtx); +void maxFunction(SqlFunctionCtx *pCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7a73bdb4e8..cc2f3c94f9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -41,6 +41,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = sumFunction, .finalizeFunc = functionFinalizer }, + { + .name = "min", + .type = FUNCTION_TYPE_MIN, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = minFunctionSetup, + .processFunc = minFunction, + .finalizeFunc = functionFinalizer + }, + { + .name = "max", + .type = FUNCTION_TYPE_MAX, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalizer + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fb30cce6a9..e514943f47 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include #include "taggfunction.h" #include "tdatablock.h" @@ -27,7 +28,6 @@ } while (0) typedef struct SSumRes { -// int8_t hasResult; union { int64_t isum; uint64_t usum; @@ -115,7 +115,7 @@ void countFunction(SqlFunctionCtx *pCtx) { } \ } while (0) -static void do_sum(SqlFunctionCtx *pCtx) { +void sumFunction(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -179,14 +179,272 @@ bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -void sumFunction(SqlFunctionCtx *pCtx) { - do_sum(pCtx); - // keep the result data in output buffer, not in the intermediate buffer -// SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult == DATA_SET_FLAG) { - // set the flag for super table query -// SSumRes *pSum = (SSumRes *)pCtx->pOutput; -// pSum->hasResult = DATA_SET_FLAG; -// } +bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + char* buf = GET_ROWCELL_INTERBUF(pResultInfo); + switch (pCtx->input.pData[0]->info.type) { + case TSDB_DATA_TYPE_INT: + *((int32_t *)buf) = INT32_MIN; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_FLOAT: + *((float *)buf) = -FLT_MAX; + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(((double *)buf), -DBL_MAX); + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)buf) = INT64_MIN; + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)buf) = INT16_MIN; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)buf) = 0; + break; + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)buf) = INT8_MIN; + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)buf) = 0; + break; + default: + assert(0); + } + return true; } + +bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; // not initialized since it has been initialized + } + + char* buf = GET_ROWCELL_INTERBUF(pResultInfo); + switch (pCtx->input.pData[0]->info.type) { + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)buf) = INT8_MAX; + break; + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *) buf = UINT8_MAX; + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)buf) = INT16_MAX; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)buf) = UINT16_MAX; + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)buf) = INT32_MAX; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)buf) = UINT32_MAX; + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)buf) = INT64_MAX; + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)buf) = UINT64_MAX; + break; + case TSDB_DATA_TYPE_FLOAT: + *((float *)buf) = FLT_MAX; + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(((double *)buf), DBL_MAX); + break; + default: + assert(0); + } + + return true; +} + +bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = sizeof(int64_t); + return true; +} + +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) + +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0); + +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0) + +#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ + } while (0) + +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ + do { \ + _t* d = (_t*)((_col)->pData); \ + for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + } \ + TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \ + UPDATE_DATA(ctx, val, d[i], num, sign, ts); \ + } \ + } while (0) + +int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { + int32_t numOfElems = 0; + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + + SColumnInfoData* pCol = pInput->pData[0]; + int32_t type = pCol->info.type; + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + + // data in current data block are qualified to the query + if (pInput->colDataAggIsSet) { + numOfElems = pInput->numOfRows - pAgg->numOfNull; + ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0); + + if (numOfElems == 0) { + return numOfElems; + } + + void* tval = NULL; + int16_t index = 0; + + if (isMinFunc) { + tval = &pInput->pColumnDataAgg[0]->min; + index = pInput->pColumnDataAgg[0]->minIndex; + } else { + tval = &pInput->pColumnDataAgg[0]->max; + index = pInput->pColumnDataAgg[0]->maxIndex; + } + + TSKEY key = TSKEY_INITIAL_VAL; + if (pCtx->ptsList != NULL) { + // the index is the original position, not the relative position + key = pCtx->ptsList[index]; + } + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + int64_t val = GET_INT64_VAL(tval); + +#if defined(_DEBUG_VIEW) + qDebug("max value updated according to pre-cal:%d", *data); +#endif + + if ((*(int64_t*)buf < val) ^ isMinFunc) { + *(int64_t*) buf = val; + for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor + __ctx->tag.i = key; + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; + } + + __ctx->fpSet.process(__ctx); + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + uint64_t val = GET_UINT64_VAL(tval); + UPDATE_DATA(pCtx, *(uint64_t*)buf, val, numOfElems, isMinFunc, key); + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + double val = GET_DOUBLE_VAL(tval); + UPDATE_DATA(pCtx, *(double*)buf, val, numOfElems, isMinFunc, key); + } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + double val = GET_DOUBLE_VAL(tval); + UPDATE_DATA(pCtx, *(float*)buf, (float)val, numOfElems, isMinFunc, key); + } + + return numOfElems; + } + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { + LOOPCHECK_N(*(int64_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { + LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { + int32_t *pData = (int32_t*)pCol->pData; + int64_t *val = (int64_t*) buf; + + for (int32_t i = 0; i < pCtx->size; ++i) { + if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + if ((*val < pData[i]) ^ isMinFunc) { + *val = pData[i]; + TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i) : 0; + DO_UPDATE_SUBSID_RES(pCtx, ts); + } + + numOfElems += 1; + } + +#if defined(_DEBUG_VIEW) + qDebug("max value updated:%d", *retVal); +#endif + } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { + LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int64_t, numOfRows, start, isMinFunc, numOfElems); + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint64_t, numOfRows, start, isMinFunc, numOfElems); + } + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + LOOPCHECK_N(*(double*) buf, pCol, pCtx, double, numOfRows, start, isMinFunc, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + LOOPCHECK_N(*(float*) buf, pCol, pCtx, float, numOfRows, start, isMinFunc, numOfElems); + } + + return numOfElems; +} + +void minFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = doMinMaxHelper(pCtx, 1); + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); +} + +void maxFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = doMinMaxHelper(pCtx, 0); + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); +} \ No newline at end of file diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 47d09ec2dc..4360515328 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -958,157 +958,6 @@ static void avg_finalizer(SqlFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// -static void minMax_function(SqlFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - // data in current data block are qualified to the query - if (pCtx->isAggSet) { - *notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(*notNullElems >= 0); - - if (*notNullElems == 0) { - return; - } - - void* tval = NULL; - int16_t index = 0; - - if (isMin) { - tval = &pCtx->agg.min; - index = pCtx->agg.minIndex; - } else { - tval = &pCtx->agg.max; - index = pCtx->agg.maxIndex; - } - - TSKEY key = TSKEY_INITIAL_VAL; - if (pCtx->ptsList != NULL) { - /** - * NOTE: work around the bug caused by invalid pre-calculated function. - * Here the selectivity + ts will not return correct value. - * - * The following codes of 3 lines will be removed later. - */ -// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { -// index = 0; -// } - - // the index is the original position, not the relative position - key = pCtx->ptsList[index]; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t val = GET_INT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *data = (int8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *data = (int16_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *data = (int32_t *)pOutput; -#if defined(_DEBUG_VIEW) - qDebug("max value updated according to pre-cal:%d", *data); -#endif - - if ((*data < val) ^ isMin) { - *data = (int32_t)val; - for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - if (__ctx->functionId == FUNCTION_TS_DUMMY) { - __ctx->tag.i = key; - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - } - - aggFunc[FUNCTION_TAG].addInput(__ctx); - } - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *data = (int64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t val = GET_UINT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - uint8_t *data = (uint8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - uint16_t *data = (uint16_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - uint32_t *data = (uint32_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - uint64_t *data = (uint64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *data = (float *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, (float)val, notNullElems, isMin, key); - } - - return; - } - - void *p = GET_INPUT_DATA_LIST(pCtx); - TSKEY *tsList = GET_TS_LIST(pCtx); - - *notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = (int32_t*) pOutput; - - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { - continue; - } - - if ((*retVal < pData[i]) ^ isMin) { - *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - *notNullElems += 1; - } -#if defined(_DEBUG_VIEW) - qDebug("max value updated:%d", *retVal); -#endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } -} - static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized @@ -1204,43 +1053,9 @@ static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf /* * the output result of min/max function is the final output buffer, not the intermediate result buffer */ -static void min_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static void max_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; - +#if 0 GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); @@ -1319,7 +1134,8 @@ static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *outp break; } } - +#endif + return notNullElems; } @@ -1618,7 +1434,7 @@ static void first_function(SqlFunctionCtx *pCtx) { memcpy(pCtx->pOutput, data, pCtx->inputBytes); if (pCtx->ptsList != NULL) { TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); +// DO_UPDATE_TAG_COLUMNS(pCtx, k); } SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); @@ -1642,7 +1458,7 @@ static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t in pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1696,7 +1512,7 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { pCtx->param[1].i = pInput->ts; pCtx->param[1].nType = pCtx->resDataInfo.type; - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); @@ -1730,7 +1546,7 @@ static void last_function(SqlFunctionCtx *pCtx) { memcpy(pCtx->pOutput, data, pCtx->inputBytes); TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed on this column @@ -1777,7 +1593,7 @@ static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t ind pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1833,7 +1649,7 @@ static void last_dist_func_merge(SqlFunctionCtx *pCtx) { pCtx->param[1].i = pInput->ts; pCtx->param[1].nType = pCtx->resDataInfo.type; - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); @@ -1860,10 +1676,10 @@ static void last_row_function(SqlFunctionCtx *pCtx) { pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1883,25 +1699,25 @@ static void last_row_finalizer(SqlFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SExtTagsInfo *pTagInfo, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, int16_t stage) { dst->v.nType = type; dst->v.i = *(int64_t *)val; dst->timestamp = tsKey; int32_t size = 0; if (stage == MERGE_STAGE) { - memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); +// memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields - for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; - if (ctx->functionId == FUNCTION_TS_DUMMY) { - ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - ctx->tag.i = tsKey; - } - - taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); - size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; - } +// for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { +// SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; +// if (ctx->functionId == FUNCTION_TS_DUMMY) { +// ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; +// ctx->tag.i = tsKey; +// } +// +// taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); +// size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; +// } } } @@ -1956,7 +1772,7 @@ static void topBotSwapFn(void *dst, void *src, const void *param) } static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1966,7 +1782,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); +// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); pInfo->num++; } else { @@ -1974,13 +1790,13 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pList[0]->v.u) || (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) { valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); +// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); } } } static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1990,7 +1806,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); +// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); pInfo->num++; } else { @@ -1998,7 +1814,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pList[0]->v.u) || (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) { valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); +// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); } } } @@ -2113,21 +1929,21 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // set the corresponding tag data for each record // todo check malloc failure - char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; - } +// char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); +// for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { +// pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; +// } - for (int32_t i = 0; i < len; ++i, output += step) { - int16_t offset = 0; - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); - offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; - pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; - } - } +// for (int32_t i = 0; i < len; ++i, output += step) { +// int16_t offset = 0; +// for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { +// memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); +// offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; +// pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; +// } +// } - tfree(pData); +// tfree(pData); } /* @@ -2161,13 +1977,13 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i; - size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; +// size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - for (int32_t i = 0; i < pCtx->param[0].i; ++i) { - pTopBotInfo->res[i] = (tValuePair*) tmp; - pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); - tmp += size; - } +// for (int32_t i = 0; i < pCtx->param[0].i; ++i) { +// pTopBotInfo->res[i] = (tValuePair*) tmp; +// pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); +// tmp += size; +// } } bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { @@ -2256,7 +2072,7 @@ static void top_function(SqlFunctionCtx *pCtx) { // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2283,8 +2099,8 @@ static void top_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, - type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); +// do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, +// type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } SET_VAL(pCtx, pInput->num, pOutput->num); @@ -2313,7 +2129,7 @@ static void bottom_function(SqlFunctionCtx *pCtx) { notNullElems++; // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2340,8 +2156,8 @@ static void bottom_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, - &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); +// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, +// &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } SET_VAL(pCtx, pInput->num, pOutput->num); @@ -4448,7 +4264,7 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_MIN, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, min_func_setup, - min_function, + NULL, function_finalizer, min_func_merge, statisRequired, @@ -4461,7 +4277,7 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_MAX, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, max_func_setup, - max_function, + NULL, function_finalizer, max_func_merge, statisRequired, From 1648754f025e9ab7e4266625ef56a2db74cbb848 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Mar 2022 22:31:23 -0500 Subject: [PATCH 33/39] TD-13747 bugfix --- source/libs/planner/src/physicalPlan.c | 41 +++++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index da1e691945..fa5732d667 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -115,6 +115,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { pIndex = taosHashGet(pCxt->pRightHash, name, len); } // pIndex is definitely not NULL, otherwise it is a bug + CHECK_ALLOC(pIndex, DEAL_RES_ERROR); ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->slotId = pIndex->slotId; CHECK_ALLOC(pNode, DEAL_RES_ERROR); @@ -177,6 +178,8 @@ static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SData FOREACH(pNode, pTargets) { int32_t len = getSlotKey(pNode, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); + // pIndex is definitely not NULL, otherwise it is a bug + CHECK_ALLOC(pIndex, TSDB_CODE_FAILED); ((SSlotDescNode*)nodesListGetNode(pDataBlockDesc->pSlots, pIndex->slotId))->output = true; } @@ -191,32 +194,30 @@ static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { pCol->tableId = tableId; pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pCol->colType = COLUMN_TYPE_COLUMN; + strcpy(pCol->colName, "#primarykey"); return pCol; } -static int32_t addPrimaryKeyCol(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode) { - if (NULL == pScanPhysiNode->pScanCols) { - pScanPhysiNode->pScanCols = nodesMakeList(); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); - return TSDB_CODE_SUCCESS; - } - SNode* pNode; - FOREACH(pNode, pScanPhysiNode->pScanCols) { - if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { - return TSDB_CODE_SUCCESS; - } - } +static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { + pScanPhysiNode->pScanCols = nodesMakeList(); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); + + SNode* pNode; + FOREACH(pNode, pScanCols) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { + SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0); + strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias); + strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); + continue; + } + CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); + } return TSDB_CODE_SUCCESS; } static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - if (NULL != pScanLogicNode->pScanCols) { - pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - } - CHECK_CODE(addPrimaryKeyCol(pCxt, pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); @@ -505,6 +506,10 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, default: break; } + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + nodesDestroyNode(pPhyNode); + return NULL; + } pPhyNode->pChildren = pChildren; SNode* pChild; From ddbe4095ed741042ec1fb91f09ba53d60dcb385c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Mar 2022 15:24:05 +0800 Subject: [PATCH 34/39] [td-13039] fix bug in agg query of ordinary table. --- include/common/tcommon.h | 1 - source/libs/executor/inc/executil.h | 3 +- source/libs/executor/inc/executorimpl.h | 2 + source/libs/executor/src/executil.c | 13 ++- source/libs/executor/src/executorimpl.c | 41 ++++------ source/libs/function/src/builtins.c | 31 ++++++- source/libs/function/src/builtinsimpl.c | 104 ++++++++++++------------ 7 files changed, 110 insertions(+), 85 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index f2129341ea..b3b47b4c68 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -256,7 +256,6 @@ typedef struct SFunctParam { // the structure for sql function in select clause typedef struct SExprBasicInfo { SSchema resSchema; // TODO refactor - int32_t interBytes; // inter result buffer size, TODO remove it int16_t numOfParams; // argument value of each function SFunctParam *pParam; // SVariant param[3]; // parameters are not more than 3 diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f35cfaee70..b34067ba4e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -92,10 +92,11 @@ typedef struct SResultRowPool { struct STaskAttr; struct STaskRuntimeEnv; struct SUdfInfo; +struct SqlFunctionCtx; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(SArray* pExprInfo); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c8b8feb3af..58839a0711 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -469,12 +469,14 @@ typedef struct SOptrBasicInfo { int32_t capacity; } SOptrBasicInfo; +//TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; typedef struct STableIntervalOperatorInfo { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1b901ee9f6..9d77e23d38 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -46,7 +46,7 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - size += pQueryAttr->pExpr1[i].base.interBytes; +// size += pQueryAttr->pExpr1[i].base.interBytes; } assert(size >= 0); @@ -172,9 +172,14 @@ SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_ return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); } -size_t getResultRowSize(SArray* pExprInfo) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - return (numOfOutput * sizeof(SResultRowEntryInfo)) + /*pQueryAttr->interBufSize +*/ sizeof(SResultRow); +size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow); + + for(int32_t i = 0; i < numOfOutput; ++i) { + rowSize += pCtx[i].resDataInfo.interBufSize; + } + + return rowSize; } SResultRowPool* initResultRowPool(size_t size) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 12426b2a39..cce05f93f9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1970,7 +1970,7 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->order = pQueryAttr->order.order; // pCtx->functionId = pFunct->functionId; pCtx->stableQuery = pQueryAttr->stableQuery; - pCtx->resDataInfo.interBufSize = pFunct->interBytes; +// pCtx->resDataInfo.interBufSize = pFunct->interBytes; pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -2052,7 +2052,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SExprBasicInfo *pFunct = &pExpr->base; SqlFunctionCtx* pCtx = &pFuncCtx[i]; - fmGetFuncExecFuncs(pExpr->pExpr->_function.functionId, &pCtx->fpSet); + fmGetFuncExecFuncs(pExpr->pExpr->_function.pFunctNode->funcId, &pCtx->fpSet); pCtx->input.numOfInputCols = pFunct->numOfParams; pCtx->input.pData = calloc(pFunct->numOfParams, POINTER_BYTES); @@ -2062,8 +2062,6 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; pCtx->order = TSDB_ORDER_ASC; -// pCtx->functionId = pExpr->pExpr->_function.pFunctNode->;//TODO remove it - pCtx->stableQuery = false; // TODO pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -2120,8 +2118,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC } for(int32_t i = 1; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1); - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i].resDataInfo.interBufSize); } setCtxTagColumnInfo(pFuncCtx, numOfOutput); @@ -3347,15 +3344,11 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); cleanupResultRowEntry(pEntry); pCtx[i].resultInfo = pEntry; - pCtx[i].pOutput = pData->pData; + pCtx[i].pOutput = pData->pData; // todo remove it pCtx[i].currentStage = stage; // set the timestamp output buffer for top/bottom/diff query @@ -5663,7 +5656,7 @@ SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { return pOrderColumns; } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SArray* pExprInfo); +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput); static void clearupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { @@ -6044,7 +6037,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t goto _error; } - int32_t code = doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -7085,13 +7078,14 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SArray* pExprInfo) { +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pAggSup->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pAggSup->pool = initResultRowPool(pAggSup->resultRowSize); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || @@ -7115,7 +7109,7 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n pInfo->binfo.pRes = pResultBlock; pInfo->binfo.capacity = numOfRows; - doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, taosArrayGetSize(pExprInfo)); pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); int32_t index = 0; @@ -7353,14 +7347,15 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - doInitAggInfoSup(&pInfo->aggSup, pExprInfo); + size_t numOfOutput = taosArrayGetSize(pExprInfo); + doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); - pInfo->order = TSDB_ORDER_ASC; + pInfo->order = TSDB_ORDER_ASC; pInfo->precision = TSDB_TIME_PRECISION_MICRO; - pInfo->win = pTaskInfo->window; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->interval = *pInterval; - int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); @@ -8039,8 +8034,8 @@ SArray* createExprInfo(SAggPhysiNode* pPhyNode, int32_t* resultRowSize) { pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; - ASSERT(LIST_LENGTH(pPhyNode->pAggFuncs) == 1); - STargetNode* pTargetNode = (STargetNode*) nodesListGetNode(pPhyNode->pAggFuncs, 0); + STargetNode* pTargetNode = (STargetNode*) nodesListGetNode(pPhyNode->pAggFuncs, i); + ASSERT(pTargetNode->slotId == i); SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; pExp->base.resSchema = createSchema(pFuncNode->node.resType.type, pFuncNode->node.resType.bytes, pTargetNode->slotId, pFuncNode->node.aliasName); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index cc2f3c94f9..714bf7e146 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -44,7 +44,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "min", .type = FUNCTION_TYPE_MIN, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .checkFunc = stubCheckAndGetResultType, .getEnvFunc = getMinmaxFuncEnv, .initFunc = minFunctionSetup, @@ -54,7 +54,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "max", .type = FUNCTION_TYPE_MAX, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .checkFunc = stubCheckAndGetResultType, .getEnvFunc = getMinmaxFuncEnv, .initFunc = maxFunctionSetup, @@ -78,8 +78,33 @@ const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFun int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { switch(pFunc->funcType) { case FUNCTION_TYPE_COUNT: pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};break; - default: + case FUNCTION_TYPE_SUM: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + + int32_t resType = 0; + if (IS_SIGNED_NUMERIC_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_BIGINT; + } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_UBIGINT; + } else if (IS_FLOAT_TYPE(paraType)) { + resType = TSDB_DATA_TYPE_DOUBLE; + } else { + ASSERT(0); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType }; break; + } + case FUNCTION_TYPE_MIN: + case FUNCTION_TYPE_MAX: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + break; + } + default: + ASSERT(0); // to found the fault ASAP. } return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e514943f47..aaaee6d56c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,7 +14,7 @@ */ #include "builtinsimpl.h" -#include +#include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -123,17 +123,18 @@ void sumFunction(SqlFunctionCtx *pCtx) { SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; + SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (pInput->colDataAggIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; ASSERT(numOfElem >= 0); - SSumRes* pSumInfo = (SSumRes*) pCtx->pOutput; if (IS_SIGNED_NUMERIC_TYPE(type)) { - pSumInfo->isum += pAgg->sum; + pSumRes->isum += pAgg->sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pSumInfo->usum += pAgg->sum; + pSumRes->usum += pAgg->sum; } else if (IS_FLOAT_TYPE(type)) { - pSumInfo->dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + pSumRes->dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); } } else { // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -141,32 +142,30 @@ void sumFunction(SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - SSumRes* pSum = (SSumRes*) pCtx->pOutput; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int8_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int16_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int32_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(pSum->isum, pCol, start, numOfRows, int64_t, numOfElem); + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_TINYINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int8_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int16_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_INT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int32_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_BIGINT) { + LIST_ADD_N(pSumRes->isum, pCol, start, numOfRows, int64_t, numOfElem); } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint8_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint16_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint32_t, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(pSum->usum, pCol, start, numOfRows, uint64_t, numOfElem); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_UTINYINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint8_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint16_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_UINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint32_t, numOfElem); + } else if (type == TSDB_DATA_TYPE_UBIGINT) { + LIST_ADD_N(pSumRes->usum, pCol, start, numOfRows, uint64_t, numOfElem); } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N(pSum->dsum, pCol, start, numOfRows, double, numOfElem); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N(pSum->dsum, pCol, start, numOfRows, float, numOfElem); + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, double, numOfElem); + } else if (type == TSDB_DATA_TYPE_FLOAT) { + LIST_ADD_N(pSumRes->dsum, pCol, start, numOfRows, float, numOfElem); } } @@ -179,14 +178,13 @@ bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } - bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } char* buf = GET_ROWCELL_INTERBUF(pResultInfo); - switch (pCtx->input.pData[0]->info.type) { + switch (pCtx->resDataInfo.type) { case TSDB_DATA_TYPE_INT: *((int32_t *)buf) = INT32_MIN; break; @@ -229,7 +227,7 @@ bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { } char* buf = GET_ROWCELL_INTERBUF(pResultInfo); - switch (pCtx->input.pData[0]->info.type) { + switch (pCtx->resDataInfo.type) { case TSDB_DATA_TYPE_TINYINT: *((int8_t *)buf) = INT8_MAX; break; @@ -374,13 +372,13 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { __ctx->fpSet.process(__ctx); } } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t val = GET_UINT64_VAL(tval); UPDATE_DATA(pCtx, *(uint64_t*)buf, val, numOfElems, isMinFunc, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + } else if (type == TSDB_DATA_TYPE_DOUBLE) { double val = GET_DOUBLE_VAL(tval); UPDATE_DATA(pCtx, *(double*)buf, val, numOfElems, isMinFunc, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + } else if (type == TSDB_DATA_TYPE_FLOAT) { double val = GET_DOUBLE_VAL(tval); UPDATE_DATA(pCtx, *(float*)buf, (float)val, numOfElems, isMinFunc, key); } @@ -391,14 +389,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LOOPCHECK_N(*(int64_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_TINYINT) { + LOOPCHECK_N(*(int8_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + LOOPCHECK_N(*(int16_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_INT) { int32_t *pData = (int32_t*)pCol->pData; - int64_t *val = (int64_t*) buf; + int32_t *val = (int32_t*) buf; for (int32_t i = 0; i < pCtx->size; ++i) { if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { @@ -417,22 +415,22 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { #if defined(_DEBUG_VIEW) qDebug("max value updated:%d", *retVal); #endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { + } else if (type == TSDB_DATA_TYPE_BIGINT) { LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int64_t, numOfRows, start, isMinFunc, numOfElems); } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (type == TSDB_DATA_TYPE_UTINYINT) { + LOOPCHECK_N(*(uint8_t*) buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + LOOPCHECK_N(*(uint16_t*) buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_UINT) { + LOOPCHECK_N(*(uint32_t*) buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); + } else if (type == TSDB_DATA_TYPE_UBIGINT) { LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint64_t, numOfRows, start, isMinFunc, numOfElems); } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + } else if (type == TSDB_DATA_TYPE_DOUBLE) { LOOPCHECK_N(*(double*) buf, pCol, pCtx, double, numOfRows, start, isMinFunc, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + } else if (type == TSDB_DATA_TYPE_FLOAT) { LOOPCHECK_N(*(float*) buf, pCol, pCtx, float, numOfRows, start, isMinFunc, numOfElems); } From da0fa26717d53155637755956783f3743bc97a74 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Mar 2022 02:36:06 -0500 Subject: [PATCH 35/39] TD-13747 src file name organize --- source/client/src/clientImpl.c | 3 +- .../nodes/inc/{nodesint.h => nodesUtil.h} | 0 source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/inc/insertParser.h | 31 - source/libs/parser/inc/new_sql.y | 557 ------- .../parser/inc/{astCreateFuncs.h => parAst.h} | 4 +- .../inc/{dataBlockMgt.h => parInsertData.h} | 0 .../libs/parser/inc/{parserInt.h => parInt.h} | 1 + .../libs/parser/inc/{ttoken.h => parToken.h} | 0 .../parser/inc/{parserUtil.h => parUtil.h} | 1 - source/libs/parser/inc/sql.y | 1462 ++++++----------- .../src/{astCreateFuncs.c => parAstCreater.c} | 4 +- .../parser/src/{astParse.c => parAstParser.c} | 26 +- .../src/{insertParser.c => parInsert.c} | 9 +- .../src/{dataBlockMgt.c => parInsertData.c} | 4 +- .../src/{ttokenizer.c => parTokenizer.c} | 3 +- .../src/{astTranslate.c => parTranslater.c} | 4 +- .../parser/src/{parserUtil.c => parUtil.c} | 2 +- source/libs/parser/src/parser.c | 16 +- source/libs/parser/src/{new_sql.c => sql.c} | 144 +- .../{parserTest.cpp => parserAstTest.cpp} | 2 +- ...ertParserTest.cpp => parserInsertTest.cpp} | 3 +- source/libs/parser/test/tokenizerTest.cpp | 730 -------- .../planner/inc/{plannerInt.h => planInt.h} | 0 .../src/{logicPlan.c => planLogicCreater.c} | 2 +- .../{physicalPlan.c => planPhysiCreater.c} | 2 +- .../src/{splitPlan.c => planSpliter.c} | 2 +- source/libs/planner/src/planner.c | 2 +- source/libs/planner/test/plannerTest.cpp | 2 +- 31 files changed, 660 insertions(+), 2362 deletions(-) rename source/libs/nodes/inc/{nodesint.h => nodesUtil.h} (100%) delete mode 100644 source/libs/parser/inc/insertParser.h delete mode 100644 source/libs/parser/inc/new_sql.y rename source/libs/parser/inc/{astCreateFuncs.h => parAst.h} (99%) rename source/libs/parser/inc/{dataBlockMgt.h => parInsertData.h} (100%) rename source/libs/parser/inc/{parserInt.h => parInt.h} (93%) rename source/libs/parser/inc/{ttoken.h => parToken.h} (100%) rename source/libs/parser/inc/{parserUtil.h => parUtil.h} (98%) rename source/libs/parser/src/{astCreateFuncs.c => parAstCreater.c} (99%) rename source/libs/parser/src/{astParse.c => parAstParser.c} (80%) rename source/libs/parser/src/{insertParser.c => parInsert.c} (99%) rename source/libs/parser/src/{dataBlockMgt.c => parInsertData.c} (99%) rename source/libs/parser/src/{ttokenizer.c => parTokenizer.c} (99%) rename source/libs/parser/src/{astTranslate.c => parTranslater.c} (99%) rename source/libs/parser/src/{parserUtil.c => parUtil.c} (99%) rename source/libs/parser/src/{new_sql.c => sql.c} (97%) rename source/libs/parser/test/{parserTest.cpp => parserAstTest.cpp} (99%) rename source/libs/parser/test/{insertParserTest.cpp => parserInsertTest.cpp} (99%) delete mode 100644 source/libs/parser/test/tokenizerTest.cpp rename source/libs/planner/inc/{plannerInt.h => planInt.h} (100%) rename source/libs/planner/src/{logicPlan.c => planLogicCreater.c} (99%) rename source/libs/planner/src/{physicalPlan.c => planPhysiCreater.c} (99%) rename source/libs/planner/src/{splitPlan.c => planSpliter.c} (99%) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fbe9186e6e..70557ec06e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -173,6 +173,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; pRequest->type = pMsgInfo->msgType; pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; + pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management STscObj* pTscObj = pRequest->pTscObj; SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); @@ -248,7 +249,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { } SRequestObj* pRequest = NULL; - SQuery* pQuery; + SQuery* pQuery = NULL; SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/inc/nodesint.h b/source/libs/nodes/inc/nodesUtil.h similarity index 100% rename from source/libs/nodes/inc/nodesint.h rename to source/libs/nodes/inc/nodesUtil.h diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index e11a7a6833..5a3d533145 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "taos.h" diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 71fa4f1f5d..5e5c3d862e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "query.h" diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index c2b96b1c99..f943b63b80 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -14,7 +14,7 @@ */ #include "cmdnodes.h" -#include "nodesint.h" +#include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" #include "taos.h" diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h deleted file mode 100644 index 311db19bcd..0000000000 --- a/source/libs/parser/inc/insertParser.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_INSERTPARSER_H -#define TDENGINE_INSERTPARSER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "parser.h" - -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y deleted file mode 100644 index 6e09048810..0000000000 --- a/source/libs/parser/inc/new_sql.y +++ /dev/null @@ -1,557 +0,0 @@ -//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql -//usage: lemon sql.y - -%name NewParse - -%token_prefix TK_ -%token_type { SToken } -%default_type { SNode* } -%default_destructor { nodesDestroyNode($$); } - -%extra_argument { SAstCreateContext* pCxt } - -%include { -#include -#include -#include -#include -#include - -#include "nodes.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "astCreateFuncs.h" -} - -%syntax_error { - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); - } - pCxt->valid = false; -} - -%left OR. -%left AND. -//%right NOT. -%left UNION ALL MINUS EXCEPT INTERSECT. -%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. -%left NK_PLUS NK_MINUS. -//%left DIVIDE TIMES. -%left NK_STAR NK_SLASH NK_REM. -%left NK_CONCAT. -//%right NK_BITNOT. - -/************************************************ create/alter/drop/show user *****************************************/ -cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B);} -cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B);} -cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B);} -cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } -cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } - -/************************************************ create/drop/show dnode **********************************************/ -cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL);} -cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B);} -cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} -cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} -cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } - -%type dnode_endpoint { SToken } -%destructor dnode_endpoint { } -dnode_endpoint(A) ::= NK_STRING(B). { A = B; } - -%type dnode_host_name { SToken } -%destructor dnode_host_name { } -dnode_host_name(A) ::= NK_ID(B). { A = B; } -dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } - -/************************************************ create/drop/show/use database ***************************************/ -cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C);} -cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } -cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } -cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);} - -%type not_exists_opt { bool } -%destructor not_exists_opt { } -not_exists_opt(A) ::= IF NOT EXISTS. { A = true; } -not_exists_opt(A) ::= . { A = false; } - -%type exists_opt { bool } -%destructor exists_opt { } -exists_opt(A) ::= IF EXISTS. { A = true; } -exists_opt(A) ::= . { A = false; } - -%type db_options { SDatabaseOptions* } -%destructor db_options { tfree($$); } -db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } -db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } -db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } -db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } -db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } -db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } -db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } -db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } -db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } -db_options(A) ::= db_options(B) KEEP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); } -db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } -db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); } -db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } -db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } -db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } -db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } -db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } -db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } - -/************************************************ create/drop/show table/stable ***************************************/ -cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} -cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } -cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } -cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } -cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } - -%type multi_create_clause { SNodeList* } -%destructor multi_create_clause { nodesDestroyList($$); } -multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } -multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); } - -create_subtable_clause(A) ::= - not_exists_opt(B) full_table_name(C) USING full_table_name(D) - specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP. { A = createCreateSubTableClause(pCxt, B, C, D, E, F); } - -%type multi_drop_clause { SNodeList* } -%destructor multi_drop_clause { nodesDestroyList($$); } -multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } -multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } - -drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); } - -%type specific_tags_opt { SNodeList* } -%destructor specific_tags_opt { nodesDestroyList($$); } -specific_tags_opt(A) ::= . { A = NULL; } -specific_tags_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } - -full_table_name(A) ::= table_name(B). { A = createRealTableNode(pCxt, NULL, &B, NULL); } -full_table_name(A) ::= db_name(B) NK_DOT table_name(C). { A = createRealTableNode(pCxt, &B, &C, NULL); } - -%type column_def_list { SNodeList* } -%destructor column_def_list { nodesDestroyList($$); } -column_def_list(A) ::= column_def(B). { A = createNodeList(pCxt, B); } -column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } - -column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } -column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } - -%type type_name { SDataType } -%destructor type_name { } -type_name(A) ::= BOOL. { A = createDataType(TSDB_DATA_TYPE_BOOL); } -type_name(A) ::= TINYINT. { A = createDataType(TSDB_DATA_TYPE_TINYINT); } -type_name(A) ::= SMALLINT. { A = createDataType(TSDB_DATA_TYPE_SMALLINT); } -type_name(A) ::= INT. { A = createDataType(TSDB_DATA_TYPE_INT); } -type_name(A) ::= INTEGER. { A = createDataType(TSDB_DATA_TYPE_INT); } -type_name(A) ::= BIGINT. { A = createDataType(TSDB_DATA_TYPE_BIGINT); } -type_name(A) ::= FLOAT. { A = createDataType(TSDB_DATA_TYPE_FLOAT); } -type_name(A) ::= DOUBLE. { A = createDataType(TSDB_DATA_TYPE_DOUBLE); } -type_name(A) ::= BINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &B); } -type_name(A) ::= TIMESTAMP. { A = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } -type_name(A) ::= NCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &B); } -type_name(A) ::= TINYINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UTINYINT); } -type_name(A) ::= SMALLINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_USMALLINT); } -type_name(A) ::= INT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UINT); } -type_name(A) ::= BIGINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UBIGINT); } -type_name(A) ::= JSON. { A = createDataType(TSDB_DATA_TYPE_JSON); } -type_name(A) ::= VARCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &B); } -type_name(A) ::= MEDIUMBLOB. { A = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } -type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } -type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } -type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } -type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } - -%type tags_def_opt { SNodeList* } -%destructor tags_def_opt { nodesDestroyList($$); } -tags_def_opt(A) ::= . { A = NULL; } -tags_def_opt(A) ::= tags_def(B). { A = B; } - -%type tags_def { SNodeList* } -%destructor tags_def { nodesDestroyList($$); } -tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } - -%type table_options { STableOptions* } -%destructor table_options { tfree($$); } -table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} -table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } -table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } -table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } -table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } - -%type col_name_list { SNodeList* } -%destructor col_name_list { nodesDestroyList($$); } -col_name_list(A) ::= col_name(B). { A = createNodeList(pCxt, B); } -col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } - -col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } - -/************************************************ show vgroups ********************************************************/ -cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } -cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } - -/************************************************ select **************************************************************/ -cmd ::= query_expression(A). { pCxt->pRootNode = A; } - -/************************************************ literal *************************************************************/ -literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } -literal(A) ::= NK_FLOAT(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); } -literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); } -literal(A) ::= NK_BOOL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); } -literal(A) ::= TIMESTAMP(B) NK_STRING(C). { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); } -literal(A) ::= duration_literal(B). { A = B; } - -duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } - -%type literal_list { SNodeList* } -%destructor literal_list { nodesDestroyList($$); } -literal_list(A) ::= literal(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } - -/************************************************ names and identifiers ***********************************************/ -%type db_name { SToken } -%destructor db_name { } -db_name(A) ::= NK_ID(B). { A = B; } - -%type table_name { SToken } -%destructor table_name { } -table_name(A) ::= NK_ID(B). { A = B; } - -%type column_name { SToken } -%destructor column_name { } -column_name(A) ::= NK_ID(B). { A = B; } - -%type function_name { SToken } -%destructor function_name { } -function_name(A) ::= NK_ID(B). { A = B; } - -%type table_alias { SToken } -%destructor table_alias { } -table_alias(A) ::= NK_ID(B). { A = B; } - -%type column_alias { SToken } -%destructor column_alias { } -column_alias(A) ::= NK_ID(B). { A = B; } - -%type user_name { SToken } -%destructor user_name { } -user_name(A) ::= NK_ID(B). { A = B; } - -/************************************************ expression **********************************************************/ -expression(A) ::= literal(B). { A = B; } -//expression(A) ::= NK_QUESTION(B). { A = B; } -//expression(A) ::= pseudo_column(B). { A = B; } -expression(A) ::= column_reference(B). { A = B; } -expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } -expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); } -//expression(A) ::= cast_expression(B). { A = B; } -//expression(A) ::= case_expression(B). { A = B; } -expression(A) ::= subquery(B). { A = B; } -expression(A) ::= NK_LP(B) expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); } -expression(A) ::= NK_PLUS(B) expression(C). { - SToken t = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C)); - } -expression(A) ::= NK_MINUS(B) expression(C). { - SToken t = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); - } -expression(A) ::= expression(B) NK_PLUS expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_MINUS expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_STAR expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_SLASH expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -expression(A) ::= expression(B) NK_REM expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } - -%type expression_list { SNodeList* } -%destructor expression_list { nodesDestroyList($$); } -expression_list(A) ::= expression(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } - -column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } -column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } - -//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } - -/************************************************ predicate ***********************************************************/ -predicate(A) ::= expression(B) compare_op(C) expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } -//predicate(A) ::= expression(B) compare_op sub_type expression(B). -predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); - } -predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } -predicate(A) ::= expression(B) IS NULL(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, B), NULL)); - } -predicate(A) ::= expression(B) IS NOT NULL(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, B), NULL)); - } -predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); - } - -%type compare_op { EOperatorType } -%destructor compare_op { } -compare_op(A) ::= NK_LT. { A = OP_TYPE_LOWER_THAN; } -compare_op(A) ::= NK_GT. { A = OP_TYPE_GREATER_THAN; } -compare_op(A) ::= NK_LE. { A = OP_TYPE_LOWER_EQUAL; } -compare_op(A) ::= NK_GE. { A = OP_TYPE_GREATER_EQUAL; } -compare_op(A) ::= NK_NE. { A = OP_TYPE_NOT_EQUAL; } -compare_op(A) ::= NK_EQ. { A = OP_TYPE_EQUAL; } -compare_op(A) ::= LIKE. { A = OP_TYPE_LIKE; } -compare_op(A) ::= NOT LIKE. { A = OP_TYPE_NOT_LIKE; } -compare_op(A) ::= MATCH. { A = OP_TYPE_MATCH; } -compare_op(A) ::= NMATCH. { A = OP_TYPE_NMATCH; } - -%type in_op { EOperatorType } -%destructor in_op { } -in_op(A) ::= IN. { A = OP_TYPE_IN; } -in_op(A) ::= NOT IN. { A = OP_TYPE_NOT_IN; } - -in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } - -/************************************************ boolean_value_expression ********************************************/ -boolean_value_expression(A) ::= boolean_primary(B). { A = B; } -boolean_value_expression(A) ::= NOT(C) boolean_primary(B). { - SToken e = getTokenFromRawExprNode(pCxt, B); - A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); - } -boolean_value_expression(A) ::= - boolean_value_expression(B) OR boolean_value_expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } -boolean_value_expression(A) ::= - boolean_value_expression(B) AND boolean_value_expression(C). { - SToken s = getTokenFromRawExprNode(pCxt, B); - SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); - } - -boolean_primary(A) ::= predicate(B). { A = B; } -boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); } - -/************************************************ common_expression ********************************************/ -common_expression(A) ::= expression(B). { A = B; } -common_expression(A) ::= boolean_value_expression(B). { A = B; } - -/************************************************ from_clause *********************************************************/ -from_clause(A) ::= FROM table_reference_list(B). { A = B; } - -table_reference_list(A) ::= table_reference(B). { A = B; } -table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } - -/************************************************ table_reference *****************************************************/ -table_reference(A) ::= table_primary(B). { A = B; } -table_reference(A) ::= joined_table(B). { A = B; } - -table_primary(A) ::= table_name(B) alias_opt(C). { A = createRealTableNode(pCxt, NULL, &B, &C); } -table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { A = createRealTableNode(pCxt, &B, &C, &D); } -table_primary(A) ::= subquery(B) alias_opt(C). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); } -table_primary(A) ::= parenthesized_joined_table(B). { A = B; } - -%type alias_opt { SToken } -%destructor alias_opt { } -alias_opt(A) ::= . { A = nil_token; } -alias_opt(A) ::= table_alias(B). { A = B; } -alias_opt(A) ::= AS table_alias(B). { A = B; } - -parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { A = B; } -parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { A = B; } - -/************************************************ joined_table ********************************************************/ -joined_table(A) ::= - table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { A = createJoinTableNode(pCxt, C, B, D, E); } - -%type join_type { EJoinType } -%destructor join_type { } -join_type(A) ::= . { A = JOIN_TYPE_INNER; } -join_type(A) ::= INNER. { A = JOIN_TYPE_INNER; } - -/************************************************ query_specification *************************************************/ -query_specification(A) ::= - SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) - partition_by_clause_opt(F) twindow_clause_opt(G) - group_by_clause_opt(H) having_clause_opt(I). { - A = createSelectStmt(pCxt, B, C, D); - A = addWhereClause(pCxt, A, E); - A = addPartitionByClause(pCxt, A, F); - A = addWindowClauseClause(pCxt, A, G); - A = addGroupByClause(pCxt, A, H); - A = addHavingClause(pCxt, A, I); - } - -%type set_quantifier_opt { bool } -%destructor set_quantifier_opt { } -set_quantifier_opt(A) ::= . { A = false; } -set_quantifier_opt(A) ::= DISTINCT. { A = true; } -set_quantifier_opt(A) ::= ALL. { A = false; } - -%type select_list { SNodeList* } -%destructor select_list { nodesDestroyList($$); } -select_list(A) ::= NK_STAR. { A = NULL; } -select_list(A) ::= select_sublist(B). { A = B; } - -%type select_sublist { SNodeList* } -%destructor select_sublist { nodesDestroyList($$); } -select_sublist(A) ::= select_item(B). { A = createNodeList(pCxt, B); } -select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); } - -select_item(A) ::= common_expression(B). { - SToken t = getTokenFromRawExprNode(pCxt, B); - A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); - } -select_item(A) ::= common_expression(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } -select_item(A) ::= common_expression(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } -select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { A = createColumnNode(pCxt, &B, &C); } - -where_clause_opt(A) ::= . { A = NULL; } -where_clause_opt(A) ::= WHERE search_condition(B). { A = B; } - -%type partition_by_clause_opt { SNodeList* } -%destructor partition_by_clause_opt { nodesDestroyList($$); } -partition_by_clause_opt(A) ::= . { A = NULL; } -partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { A = B; } - -twindow_clause_opt(A) ::= . { A = NULL; } -twindow_clause_opt(A) ::= - SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } -twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } -twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, B, NULL, C, D); } -twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP - sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, B, C, D, E); } - -sliding_opt(A) ::= . { A = NULL; } -sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = B; } - -fill_opt(A) ::= . { A = NULL; } -fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } - -%type fill_mode { EFillMode } -%destructor fill_mode { } -fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; } -fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; } -fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; } -fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; } -fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; } - -%type group_by_clause_opt { SNodeList* } -%destructor group_by_clause_opt { nodesDestroyList($$); } -group_by_clause_opt(A) ::= . { A = NULL; } -group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { A = B; } - -%type group_by_list { SNodeList* } -%destructor group_by_list { nodesDestroyList($$); } -group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } -group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } - -having_clause_opt(A) ::= . { A = NULL; } -having_clause_opt(A) ::= HAVING search_condition(B). { A = B; } - -/************************************************ query_expression ****************************************************/ -query_expression(A) ::= - query_expression_body(B) - order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { - A = addOrderByClause(pCxt, B, C); - A = addSlimitClause(pCxt, A, D); - A = addLimitClause(pCxt, A, E); - } - -query_expression_body(A) ::= query_primary(B). { A = B; } -query_expression_body(A) ::= - query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } - -query_primary(A) ::= query_specification(B). { A = B; } -//query_primary(A) ::= -// NK_LP query_expression_body(B) -// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} - -%type order_by_clause_opt { SNodeList* } -%destructor order_by_clause_opt { nodesDestroyList($$); } -order_by_clause_opt(A) ::= . { A = NULL; } -order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { A = B; } - -slimit_clause_opt(A) ::= . { A = NULL; } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -limit_clause_opt(A) ::= . { A = NULL; } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -/************************************************ subquery ************************************************************/ -subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, C); } - -/************************************************ search_condition ****************************************************/ -search_condition(A) ::= common_expression(B). { A = releaseRawExprNode(pCxt, B); } - -/************************************************ sort_specification_list *********************************************/ -%type sort_specification_list { SNodeList* } -%destructor sort_specification_list { nodesDestroyList($$); } -sort_specification_list(A) ::= sort_specification(B). { A = createNodeList(pCxt, B); } -sort_specification_list(A) ::= - sort_specification_list(B) NK_COMMA sort_specification(C). { A = addNodeToList(pCxt, B, C); } - -sort_specification(A) ::= - expression(B) ordering_specification_opt(C) null_ordering_opt(D). { A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); } - -%type ordering_specification_opt EOrder -%destructor ordering_specification_opt { } -ordering_specification_opt(A) ::= . { A = ORDER_ASC; } -ordering_specification_opt(A) ::= ASC. { A = ORDER_ASC; } -ordering_specification_opt(A) ::= DESC. { A = ORDER_DESC; } - -%type null_ordering_opt ENullOrder -%destructor null_ordering_opt { } -null_ordering_opt(A) ::= . { A = NULL_ORDER_DEFAULT; } -null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } -null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/parAst.h similarity index 99% rename from source/libs/parser/inc/astCreateFuncs.h rename to source/libs/parser/inc/parAst.h index bc43b0f4e9..11c56ddf3c 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/parAst.h @@ -22,9 +22,9 @@ extern "C" { #include "cmdnodes.h" #include "parser.h" -#include "parserUtil.h" +#include "parToken.h" +#include "parUtil.h" #include "querynodes.h" -#include "ttoken.h" typedef struct SAstCreateContext { SParseContext* pQueryCxt; diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/parInsertData.h similarity index 100% rename from source/libs/parser/inc/dataBlockMgt.h rename to source/libs/parser/inc/parInsertData.h diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parInt.h similarity index 93% rename from source/libs/parser/inc/parserInt.h rename to source/libs/parser/inc/parInt.h index 3fb3bbcc7f..af0d78717e 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parInt.h @@ -22,6 +22,7 @@ extern "C" { #include "parser.h" +int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/parToken.h similarity index 100% rename from source/libs/parser/inc/ttoken.h rename to source/libs/parser/inc/parToken.h diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parUtil.h similarity index 98% rename from source/libs/parser/inc/parserUtil.h rename to source/libs/parser/inc/parUtil.h index 50c99bb08b..4d7a8e2a18 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -22,7 +22,6 @@ extern "C" { #include "os.h" #include "query.h" -#include "ttoken.h" typedef struct SMsgBuf { int32_t len; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index fa713d5fb4..29e99c360a 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1,24 +1,12 @@ //lemon parser file to generate sql parse by using finite-state-machine code used to parse sql //usage: lemon sql.y + %token_prefix TK_ +%token_type { SToken } +%default_type { SNode* } +%default_destructor { nodesDestroyNode($$); } -%token_type {SToken} -%default_type {SToken} -%extra_argument {SSqlInfo* pInfo} - -%fallback ID BOOL INTEGER FLOAT STRING TIMESTAMP. - -%left OR. -%left AND. -%right NOT. -%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -%left GT GE LT LE. -%left BITAND BITOR LSHIFT RSHIFT. -%left PLUS MINUS. -%left DIVIDE TIMES. -%left STAR SLASH REM. -%left CONCAT. -%right UMINUS UPLUS BITNOT. +%extra_argument { SAstCreateContext* pCxt } %include { #include @@ -26,920 +14,542 @@ #include #include #include -#include "astGenerator.h" -#include "tmsgtype.h" -#include "ttoken.h" + +#include "nodes.h" +#include "parToken.h" #include "ttokendef.h" -#include "tvariant.h" -#include "parserInt.h" +#include "parAst.h" } -%syntax_error { - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - +%syntax_error { if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - - assert(len <= outputBufLen); + pCxt->valid = false; } -%parse_accept {} - -program ::= cmd. {} - -//////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// -cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} -cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -cmd ::= SHOW USERS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} - -cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TRANS, 0, 0); } -cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } - -cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -cmd ::= SHOW VNODES ids(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } - - -%type dbPrefix {SToken} -dbPrefix(A) ::=. {A.n = 0; A.type = 0;} -dbPrefix(A) ::= ids(X) DOT. {A = X; } - -%type cpxName {SToken} -cpxName(A) ::= . {A.n = 0; } -cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } - -cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); -} -cmd ::= SHOW CREATE STABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &X); -} - -cmd ::= SHOW CREATE DATABASE ids(X). { - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); -} - -cmd ::= SHOW dbPrefix(X) TABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, &Y); -} - -cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &Y); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS. { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); -} - -//drop configure for tables -cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1); -} - -//drop stable -cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE); -} - -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); } -cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); } -cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); } - -cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } -cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } -cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } - -/////////////////////////////////THE USE STATEMENT////////////////////////////////////////// -cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} - -/////////////////////////////////THE DESCRIBE STATEMENT///////////////////////////////////// -cmd ::= DESCRIBE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -cmd ::= DESC ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -/////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } -cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } -cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -//cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} - -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} - -////////////////////////////// COMPACT STATEMENT ////////////////////////////////////////////// - -cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, Y);} - -// An IDENTIFIER can be a generic identifier, or one of several keywords. -// Any non-standard keyword can also be an identifier. -// And "ids" is an identifer-or-string. -%type ids {SToken} -ids(A) ::= ID(X). {A = X; } -//ids(A) ::= STRING(X). {A = X; } - -%type ifexists {SToken} -ifexists(X) ::= IF EXISTS. { X.n = 1;} -ifexists(X) ::= . { X.n = 0;} - -%type ifnotexists {SToken} -ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} -ifnotexists(X) ::= . { X.n = 0;} - -/////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// -//create option for dnode/db/user/account -cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);} -cmd ::= CREATE DNODE IPTOKEN(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);} -cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -//cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);} -cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} - -bufsize(Y) ::= . { Y.n = 0; } -bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; } - -pps(Y) ::= . { Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). { Y = X; } - -tseries(Y) ::= . { Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } - -dbs(Y) ::= . { Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). { Y = X; } - -streams(Y) ::= . { Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). { Y = X; } - -storage(Y) ::= . { Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). { Y = X; } - -qtime(Y) ::= . { Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). { Y = X; } - -users(Y) ::= . { Y.n = 0; } -users(Y) ::= USERS INTEGER(X). { Y = X; } - -conns(Y) ::= . { Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). { Y = X; } - -state(Y) ::= . { Y.n = 0; } -state(Y) ::= STATE ids(X). { Y = X; } - -%type acct_optr {SCreateAcctInfo} -acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { - Y.maxUsers = (K.n>0)?atoi(K.z):-1; - Y.maxDbs = (E.n>0)?atoi(E.z):-1; - Y.maxTimeSeries = (D.n>0)?atoi(D.z):-1; - Y.maxStreams = (F.n>0)?atoi(F.z):-1; - Y.maxPointsPerSecond = (C.n>0)?atoi(C.z):-1; - Y.maxStorage = (P.n>0)?strtoll(P.z, NULL, 10):-1; - Y.maxQueryTime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1; - Y.maxConnections = (L.n>0)?atoi(L.z):-1; - Y.stat = M; -} - -%type intitemlist {SArray*} -%destructor intitemlist {taosArrayDestroy($$);} - -%type intitem {SVariant} -intitemlist(A) ::= intitemlist(X) COMMA intitem(Y). { A = tListItemAppend(X, &Y, -1); } -intitemlist(A) ::= intitem(X). { A = tListItemAppend(NULL, &X, -1); } - -intitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } - -%type keep {SArray*} -%destructor keep {taosArrayDestroy($$);} -keep(Y) ::= KEEP intitemlist(X). { Y = X; } - -cache(Y) ::= CACHE INTEGER(X). { Y = X; } -replica(Y) ::= REPLICA INTEGER(X). { Y = X; } -quorum(Y) ::= QUORUM INTEGER(X). { Y = X; } -days(Y) ::= DAYS INTEGER(X). { Y = X; } -minrows(Y) ::= MINROWS INTEGER(X). { Y = X; } -maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; } -blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; } -ctime(Y) ::= CTIME INTEGER(X). { Y = X; } -wal(Y) ::= WAL INTEGER(X). { Y = X; } -fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } -comp(Y) ::= COMP INTEGER(X). { Y = X; } -prec(Y) ::= PRECISION STRING(X). { Y = X; } -update(Y) ::= UPDATE INTEGER(X). { Y = X; } -cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } -vgroups(Y) ::= VGROUPS INTEGER(X). { Y = X; } -//partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } -stream_mode(Y) ::= STREAM MODE INTEGER(X). { Y = X; } - -%type db_optr {SCreateDbInfo} -db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} - -db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) days(X). { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) minrows(X). { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } -db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) vgroups(X). { Y = Z; Y.numOfVgroups = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) stream_mode(X). { Y = Z; Y.streamMode = strtol(X.z, NULL, 10); } - -//%type topic_optr {SCreateDbInfo} -// -//topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type alter_db_optr {SCreateDbInfo} -alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} - -alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -// dynamically update the following two parameters are not allowed. -//alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -//alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } not support yet - -//%type alter_topic_optr {SCreateDbInfo} - -//alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type typename {SField} -typename(A) ::= ids(X). { - X.type = 0; - tSetColumnType (&A, &X); -} - -//define binary type, e.g., binary(10), nchar(10) -typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSetColumnType(&A, &X); - } -} - -// define the unsigned number type -typename(A) ::= ids(X) UNSIGNED(Z). { - X.type = 0; - X.n = ((Z.z + Z.n) - X.z); - tSetColumnType (&A, &X); -} - -%type signed {int64_t} -signed(A) ::= INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} - -////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} -cmd ::= CREATE TABLE create_stable_args. {} -cmd ::= CREATE STABLE create_stable_args. {} -cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} - -%type create_table_list{SCreateTableSql*} -%destructor create_table_list{destroyCreateTableSql($$);} -create_table_list(A) ::= create_from_stable(Z). { - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &Z); - pCreateTable->type = TSDB_SQL_CREATE_TABLE; - A = pCreateTable; -} - -create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { - taosArrayPush(X->childTableInfo, &Z); - A = X; -} - -%type create_table_args{SCreateTableSql*} -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateTableInfo(X, NULL, NULL, TSDB_SQL_CREATE_TABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create super table -%type create_stable_args{SCreateTableSql*} -create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateTableInfo(X, Y, NULL, TSDB_SQL_CREATE_STABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_STABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create table by using super table -// create table table_name using super_table_name tags(tag_values1, tag_values2) -%type create_from_stable{SCreatedTableInfo} -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist1(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, NULL, Y, &V, &U); -} - -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist1(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, P, Y, &V, &U); -} - -%type tagNamelist{SArray*} -%destructor tagNamelist {taosArrayDestroy($$);} -tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } -tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &X);} - -// create stream -// create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { -// A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); -// setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); -// -// V.n += Z.n; -// setCreatedTableName(pInfo, &V, &U); -} - -%type column{SField} -%type columnlist{SArray*} -%destructor columnlist {taosArrayDestroy($$);} -columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; } -columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);} - -// The information used for a column is the name and type of column: -// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) -column(A) ::= ids(X) typename(Y). { - tSetColumnInfo(&A, &X, &Y); -} - -%type tagitemlist1 {SArray*} -%destructor tagitemlist1 {taosArrayDestroy($$);} - -tagitemlist1(A) ::= tagitemlist1(X) COMMA tagitem1(Y). { taosArrayPush(X, &Y); A = X;} -tagitemlist1(A) ::= tagitem1(Y). { A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &Y); } - -%type tagitem1 {SToken} -tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} -tagitem1(A) ::= INTEGER(X). { A = X; } -tagitem1(A) ::= FLOAT(X). { A = X; } -tagitem1(A) ::= STRING(X). { A = X; } -tagitem1(A) ::= BOOL(X). { A = X; } -tagitem1(A) ::= NULL(X). { A = X; } -tagitem1(A) ::= NOW(X). { A = X; } - -%type tagitemlist {SArray*} -%destructor tagitemlist {taosArrayDestroy($$);} - -%type tagitem {SVariant} -tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tListItemAppend(X, &Y, -1); } -tagitemlist(A) ::= tagitem(X). { A = tListItemAppend(NULL, &X, -1); } - -tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= STRING(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= BOOL(X). { toTSDBType(X.type); taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= NULL(X). { X.type = 0; taosVariantCreate(&A, X.z, X.n, X.type); } -tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&A, X.z, X.n, X.type);} - -tagitem(A) ::= MINUS(X) INTEGER(Y).{ - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= MINUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= PLUS(X) INTEGER(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -tagitem(A) ::= PLUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, X.z, X.n, X.type); -} - -//////////////////////// The SELECT statement ///////////////////////////////// -%type select {SSqlNode*} -%destructor select {destroySqlNode($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sliding_opt(S) session_option(H) windowstate_option(D) fill_opt(F)groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N); -} - -select(A) ::= LP select(B) RP. {A = B;} - -%type union {SSubclause*} -%destructor union {destroyAllSqlNode($$);} -union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNIONALL, X); } -union(Y) ::= union(Z) UNION select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNION, X); } -cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } - -// Support for the SQL exprssion without from & where subclauses, e.g., -// select database() -// select server_version() -// select client_version() -// select server_state() -select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} - -// selcollist is a list of expressions that are to become the return -// values of the SELECT statement. The "*" in statements like -// "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. -%type selcollist {SArray*} -%destructor selcollist {tSqlExprListDestroy($$);} - -%type sclp {SArray*} -%destructor sclp {tSqlExprListDestroy($$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} -sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { - A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); -} - -selcollist(A) ::= sclp(P) STAR. { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - A = tSqlExprListAppend(P, pNode, 0, 0); -} - -// An option "AS " phrase that can follow one of the expressions that -// define the result set, or one of the tables in the FROM clause. -%type as {SToken} -as(X) ::= AS ids(Y). { X = Y; } -as(X) ::= ids(Y). { X = Y; } -as(X) ::= . { X.n = 0; } - -%type distinct {SToken} -distinct(X) ::= DISTINCT(Y). { X = Y; } -distinct(X) ::= . { X.n = 0;} - -// A complete FROM clause. -%type from {SRelationInfo*} -%destructor from {destroyRelationInfo($$);} -from(A) ::= FROM tablelist(X). {A = X;} -from(A) ::= FROM sub(X). {A = X;} - -%type sub {SRelationInfo*} -%destructor sub {destroyRelationInfo($$);} -sub(A) ::= LP union(Y) RP. {A = addSubquery(NULL, Y, NULL);} -sub(A) ::= LP union(Y) RP ids(Z). {A = addSubquery(NULL, Y, &Z);} -sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubquery(X, Y, &Z);} - -%type tablelist {SRelationInfo*} -%destructor tablelist {destroyRelationInfo($$);} -tablelist(A) ::= ids(X) cpxName(Y). { - X.n += Y.n; - A = setTableNameList(NULL, &X, NULL); -} - -tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - X.n += Y.n; - A = setTableNameList(NULL, &X, &Z); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { - X.n += Z.n; - A = setTableNameList(Y, &X, NULL); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - X.n += Z.n; - A = setTableNameList(Y, &X, &F); -} - -// The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SToken} -tmvar(A) ::= VARIABLE(X). {A = X;} - -%type interval_option {SIntervalVal} -interval_option(N) ::= intervalKey(A) LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.token = A;} -interval_option(N) ::= intervalKey(A) LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X; N.token = A;} -interval_option(N) ::= . {memset(&N, 0, sizeof(N));} - -%type intervalKey {int32_t} -intervalKey(A) ::= INTERVAL. {A = TK_INTERVAL;} -intervalKey(A) ::= EVERY. {A = TK_EVERY; } - -%type session_option {SSessionWindowVal} -session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} -session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { - V.n += Z.n; - X.col = V; - X.gap = Y; -} - -%type windowstate_option {SWindowStateVal} -windowstate_option(X) ::= . { X.col.n = 0; X.col.z = NULL;} -windowstate_option(X) ::= STATE_WINDOW LP ids(V) RP. { X.col = V; } - -%type fill_opt {SArray*} -%destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . { N = 0; } -fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { - SVariant A = {0}; - toTSDBType(Y.type); - taosVariantCreate(&A, Y.z, Y.n, Y.type); - - tListItemInsert(X, &A, -1, 0); - N = X; -} - -fill_opt(N) ::= FILL LP ID(Y) RP. { - toTSDBType(Y.type); - N = tListItemAppendToken(NULL, &Y, -1); -} - -%type sliding_opt {SToken} -sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } -sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } - -%type orderby_opt {SArray*} -%destructor orderby_opt {taosArrayDestroy($$);} - -%type sortlist {SArray*} -%destructor sortlist {taosArrayDestroy($$);} - -%type sortitem {SVariant} -%destructor sortitem {taosVariantDestroy(&$$);} - -orderby_opt(A) ::= . {A = 0;} -orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} - -sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). { - A = tListItemAppend(X, &Y, Z); -} - -sortlist(A) ::= item(Y) sortorder(Z). { - A = tListItemAppend(NULL, &Y, Z); -} - -%type item {SVariant} -item(A) ::= ids(X) cpxName(Y). { - toTSDBType(X.type); - X.n += Y.n; - - taosVariantCreate(&A, X.z, X.n, X.type); -} - -%type sortorder {int} -sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} -sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default - -//group by clause -%type groupby_opt {SArray*} -%destructor groupby_opt {taosArrayDestroy($$);} -%type grouplist {SArray*} -%destructor grouplist {taosArrayDestroy($$);} - -groupby_opt(A) ::= . { A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} - -grouplist(A) ::= grouplist(X) COMMA item(Y). { - A = tListItemAppend(X, &Y, -1); -} - -grouplist(A) ::= item(X). { - A = tListItemAppend(NULL, &X, -1); -} - -//having clause, ignore the input condition in having -%type having_opt {tSqlExpr*} -%destructor having_opt {tSqlExprDestroy($$);} -having_opt(A) ::=. {A = 0;} -having_opt(A) ::= HAVING expr(X). {A = X;} - -//limit-offset subclause -%type limit_opt {SLimit} -limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - { A.limit = X; A.offset = Y;} -limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - { A.limit = Y; A.offset = X;} - -%type slimit_opt {SLimit} -slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X). {A.limit = X; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y). - {A.limit = X; A.offset = Y;} -slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). - {A.limit = Y; A.offset = X;} - -%type where_opt {tSqlExpr*} -%destructor where_opt {tSqlExprDestroy($$);} - -where_opt(A) ::= . {A = 0;} -where_opt(A) ::= WHERE expr(X). {A = X;} - -/////////////////////////// Expression Processing ///////////////////////////// -// -%type expr {tSqlExpr*} -%destructor expr {tSqlExprDestroy($$);} - -expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);} - -expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} -expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} - -// ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); } - -// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } - -// is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} - -// relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} - -expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} - -expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } - -// binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } - -// like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } - -// match expression -expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); } -expr(A) ::= expr(X) NMATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_NMATCH); } - -//in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } - -%type exprlist {SArray*} -%destructor exprlist {tSqlExprListDestroy($$);} - -%type expritem {tSqlExpr*} -%destructor expritem {tSqlExprDestroy($$);} - -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} -exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} - -///////////////////////////////////reset query cache////////////////////////////////////// -cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} - -///////////////////////////////////sync replica database////////////////////////////////// -cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);} - -///////////////////////////////////ALTER TABLE statement////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - toTSDBType(A.type); - SArray* K = tListItemAppendToken(NULL, &A, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tListItemAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - A = tListItemAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -///////////////////////////////////ALTER STABLE statement////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tListItemAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tListItemAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tListItemAppendToken(NULL, &Y, -1); - A = tListItemAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} - -%fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED - DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD - LIKE MATCH NMATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. +%left OR. +%left AND. +//%right NOT. +%left UNION ALL MINUS EXCEPT INTERSECT. +%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. +%left NK_PLUS NK_MINUS. +//%left DIVIDE TIMES. +%left NK_STAR NK_SLASH NK_REM. +%left NK_CONCAT. +//%right NK_BITNOT. + +/************************************************ create/alter/drop/show user *****************************************/ +cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B);} +cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B);} +cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B);} +cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } +cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } + +/************************************************ create/drop/show dnode **********************************************/ +cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL);} +cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B);} +cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} +cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} +cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } + +%type dnode_endpoint { SToken } +%destructor dnode_endpoint { } +dnode_endpoint(A) ::= NK_STRING(B). { A = B; } + +%type dnode_host_name { SToken } +%destructor dnode_host_name { } +dnode_host_name(A) ::= NK_ID(B). { A = B; } +dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } + +/************************************************ create/drop/show/use database ***************************************/ +cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C);} +cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } +cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } +cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);} + +%type not_exists_opt { bool } +%destructor not_exists_opt { } +not_exists_opt(A) ::= IF NOT EXISTS. { A = true; } +not_exists_opt(A) ::= . { A = false; } + +%type exists_opt { bool } +%destructor exists_opt { } +exists_opt(A) ::= IF EXISTS. { A = true; } +exists_opt(A) ::= . { A = false; } + +%type db_options { SDatabaseOptions* } +%destructor db_options { tfree($$); } +db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } +db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } +db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } +db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } +db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } +db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } +db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } +db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } +db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } +db_options(A) ::= db_options(B) KEEP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); } +db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } +db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); } +db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } +db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } +db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } +db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } +db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } +db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } + +/************************************************ create/drop/show table/stable ***************************************/ +cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) + NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} +cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} +cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) + NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} +cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } +cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } +cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } +cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } + +%type multi_create_clause { SNodeList* } +%destructor multi_create_clause { nodesDestroyList($$); } +multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } +multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); } + +create_subtable_clause(A) ::= + not_exists_opt(B) full_table_name(C) USING full_table_name(D) + specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP. { A = createCreateSubTableClause(pCxt, B, C, D, E, F); } + +%type multi_drop_clause { SNodeList* } +%destructor multi_drop_clause { nodesDestroyList($$); } +multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } +multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } + +drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); } + +%type specific_tags_opt { SNodeList* } +%destructor specific_tags_opt { nodesDestroyList($$); } +specific_tags_opt(A) ::= . { A = NULL; } +specific_tags_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + +full_table_name(A) ::= table_name(B). { A = createRealTableNode(pCxt, NULL, &B, NULL); } +full_table_name(A) ::= db_name(B) NK_DOT table_name(C). { A = createRealTableNode(pCxt, &B, &C, NULL); } + +%type column_def_list { SNodeList* } +%destructor column_def_list { nodesDestroyList($$); } +column_def_list(A) ::= column_def(B). { A = createNodeList(pCxt, B); } +column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } + +column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } +column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } + +%type type_name { SDataType } +%destructor type_name { } +type_name(A) ::= BOOL. { A = createDataType(TSDB_DATA_TYPE_BOOL); } +type_name(A) ::= TINYINT. { A = createDataType(TSDB_DATA_TYPE_TINYINT); } +type_name(A) ::= SMALLINT. { A = createDataType(TSDB_DATA_TYPE_SMALLINT); } +type_name(A) ::= INT. { A = createDataType(TSDB_DATA_TYPE_INT); } +type_name(A) ::= INTEGER. { A = createDataType(TSDB_DATA_TYPE_INT); } +type_name(A) ::= BIGINT. { A = createDataType(TSDB_DATA_TYPE_BIGINT); } +type_name(A) ::= FLOAT. { A = createDataType(TSDB_DATA_TYPE_FLOAT); } +type_name(A) ::= DOUBLE. { A = createDataType(TSDB_DATA_TYPE_DOUBLE); } +type_name(A) ::= BINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &B); } +type_name(A) ::= TIMESTAMP. { A = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +type_name(A) ::= NCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &B); } +type_name(A) ::= TINYINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UTINYINT); } +type_name(A) ::= SMALLINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_USMALLINT); } +type_name(A) ::= INT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UINT); } +type_name(A) ::= BIGINT UNSIGNED. { A = createDataType(TSDB_DATA_TYPE_UBIGINT); } +type_name(A) ::= JSON. { A = createDataType(TSDB_DATA_TYPE_JSON); } +type_name(A) ::= VARCHAR NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &B); } +type_name(A) ::= MEDIUMBLOB. { A = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } +type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } +type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } +type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } + +%type tags_def_opt { SNodeList* } +%destructor tags_def_opt { nodesDestroyList($$); } +tags_def_opt(A) ::= . { A = NULL; } +tags_def_opt(A) ::= tags_def(B). { A = B; } + +%type tags_def { SNodeList* } +%destructor tags_def { nodesDestroyList($$); } +tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } + +%type table_options { STableOptions* } +%destructor table_options { tfree($$); } +table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} +table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } +table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } +table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } +table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } + +%type col_name_list { SNodeList* } +%destructor col_name_list { nodesDestroyList($$); } +col_name_list(A) ::= col_name(B). { A = createNodeList(pCxt, B); } +col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } + +col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } + +/************************************************ show vgroups ********************************************************/ +cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } +cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } + +/************************************************ select **************************************************************/ +cmd ::= query_expression(A). { pCxt->pRootNode = A; } + +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } +literal(A) ::= NK_FLOAT(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); } +literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); } +literal(A) ::= NK_BOOL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); } +literal(A) ::= TIMESTAMP(B) NK_STRING(C). { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); } +literal(A) ::= duration_literal(B). { A = B; } + +duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } + +%type literal_list { SNodeList* } +%destructor literal_list { nodesDestroyList($$); } +literal_list(A) ::= literal(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + +/************************************************ names and identifiers ***********************************************/ +%type db_name { SToken } +%destructor db_name { } +db_name(A) ::= NK_ID(B). { A = B; } + +%type table_name { SToken } +%destructor table_name { } +table_name(A) ::= NK_ID(B). { A = B; } + +%type column_name { SToken } +%destructor column_name { } +column_name(A) ::= NK_ID(B). { A = B; } + +%type function_name { SToken } +%destructor function_name { } +function_name(A) ::= NK_ID(B). { A = B; } + +%type table_alias { SToken } +%destructor table_alias { } +table_alias(A) ::= NK_ID(B). { A = B; } + +%type column_alias { SToken } +%destructor column_alias { } +column_alias(A) ::= NK_ID(B). { A = B; } + +%type user_name { SToken } +%destructor user_name { } +user_name(A) ::= NK_ID(B). { A = B; } + +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { A = B; } +//expression(A) ::= NK_QUESTION(B). { A = B; } +//expression(A) ::= pseudo_column(B). { A = B; } +expression(A) ::= column_reference(B). { A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } +expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); } +//expression(A) ::= cast_expression(B). { A = B; } +//expression(A) ::= case_expression(B). { A = B; } +expression(A) ::= subquery(B). { A = B; } +expression(A) ::= NK_LP(B) expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); } +expression(A) ::= NK_PLUS(B) expression(C). { + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C)); + } +expression(A) ::= NK_MINUS(B) expression(C). { + SToken t = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); + } +expression(A) ::= expression(B) NK_PLUS expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_MINUS expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_STAR expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_SLASH expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_REM expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } + +%type expression_list { SNodeList* } +%destructor expression_list { nodesDestroyList($$); } +expression_list(A) ::= expression(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + +column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } + +//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } + +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) IS NULL(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) IS NOT NULL(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } + +%type compare_op { EOperatorType } +%destructor compare_op { } +compare_op(A) ::= NK_LT. { A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { A = OP_TYPE_NMATCH; } + +%type in_op { EOperatorType } +%destructor in_op { } +in_op(A) ::= IN. { A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { A = OP_TYPE_NOT_IN; } + +in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } + +/************************************************ boolean_value_expression ********************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { A = B; } +boolean_value_expression(A) ::= NOT(C) boolean_primary(B). { + SToken e = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); + } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } + +boolean_primary(A) ::= predicate(B). { A = B; } +boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); } + +/************************************************ common_expression ********************************************/ +common_expression(A) ::= expression(B). { A = B; } +common_expression(A) ::= boolean_value_expression(B). { A = B; } + +/************************************************ from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { A = B; } + +table_reference_list(A) ::= table_reference(B). { A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } + +/************************************************ table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { A = B; } +table_reference(A) ::= joined_table(B). { A = B; } + +table_primary(A) ::= table_name(B) alias_opt(C). { A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +table_primary(A) ::= parenthesized_joined_table(B). { A = B; } + +%type alias_opt { SToken } +%destructor alias_opt { } +alias_opt(A) ::= . { A = nil_token; } +alias_opt(A) ::= table_alias(B). { A = B; } +alias_opt(A) ::= AS table_alias(B). { A = B; } + +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { A = B; } + +/************************************************ joined_table ********************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { A = createJoinTableNode(pCxt, C, B, D, E); } + +%type join_type { EJoinType } +%destructor join_type { } +join_type(A) ::= . { A = JOIN_TYPE_INNER; } +join_type(A) ::= INNER. { A = JOIN_TYPE_INNER; } + +/************************************************ query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } + +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { } +set_quantifier_opt(A) ::= . { A = false; } +set_quantifier_opt(A) ::= DISTINCT. { A = true; } +set_quantifier_opt(A) ::= ALL. { A = false; } + +%type select_list { SNodeList* } +%destructor select_list { nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { A = NULL; } +select_list(A) ::= select_sublist(B). { A = B; } + +%type select_sublist { SNodeList* } +%destructor select_sublist { nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { A = addNodeToList(pCxt, B, C); } + +select_item(A) ::= common_expression(B). { + SToken t = getTokenFromRawExprNode(pCxt, B); + A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); + } +select_item(A) ::= common_expression(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= common_expression(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { A = createColumnNode(pCxt, &B, &C); } + +where_clause_opt(A) ::= . { A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { A = B; } + +twindow_clause_opt(A) ::= . { A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = B; } + +fill_opt(A) ::= . { A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { } +fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { A = NULL; } +group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { A = B; } + +%type group_by_list { SNodeList* } +%destructor group_by_list { nodesDestroyList($$); } +group_by_list(A) ::= expression(B). { A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } +group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } + +having_clause_opt(A) ::= . { A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { A = B; } + +/************************************************ query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { A = B; } +//query_primary(A) ::= +// NK_LP query_expression_body(B) +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { A = B; } + +slimit_clause_opt(A) ::= . { A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +/************************************************ subquery ************************************************************/ +subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, C); } + +/************************************************ search_condition ****************************************************/ +search_condition(A) ::= common_expression(B). { A = releaseRawExprNode(pCxt, B); } + +/************************************************ sort_specification_list *********************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); } + +%type ordering_specification_opt EOrder +%destructor ordering_specification_opt { } +ordering_specification_opt(A) ::= . { A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { A = ORDER_DESC; } + +%type null_ordering_opt ENullOrder +%destructor null_ordering_opt { } +null_ordering_opt(A) ::= . { A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/parAstCreater.c similarity index 99% rename from source/libs/parser/src/astCreateFuncs.c rename to source/libs/parser/src/parAstCreater.c index 17a3d1f765..f26c4c3549 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/parAstCreater.c @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#include "astCreateFuncs.h" -#include "parserUtil.h" +#include "parAst.h" +#include "parUtil.h" #define CHECK_OUT_OF_MEM(p) \ do { \ diff --git a/source/libs/parser/src/astParse.c b/source/libs/parser/src/parAstParser.c similarity index 80% rename from source/libs/parser/src/astParse.c rename to source/libs/parser/src/parAstParser.c index 8be03b54b3..e992f150aa 100644 --- a/source/libs/parser/src/astParse.c +++ b/source/libs/parser/src/parAstParser.c @@ -13,28 +13,28 @@ * along with this program. If not, see . */ -#include "parserInt.h" +#include "parInt.h" -#include "astCreateFuncs.h" -#include "ttoken.h" +#include "parAst.h" +#include "parToken.h" typedef void* (*FMalloc)(size_t); typedef void (*FFree)(void*); -extern void* NewParseAlloc(FMalloc); -extern void NewParse(void*, int, SToken, void*); -extern void NewParseFree(void*, FFree); -extern void NewParseTrace(FILE*, char*); +extern void* ParseAlloc(FMalloc); +extern void Parse(void*, int, SToken, void*); +extern void ParseFree(void*, FFree); +extern void ParseTrace(FILE*, char*); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void *pParser = NewParseAlloc(malloc); + void *pParser = ParseAlloc(malloc); int32_t i = 0; while (1) { SToken t0 = {0}; if (cxt.pQueryCxt->pSql[i] == 0) { - NewParse(pParser, 0, t0, &cxt); + Parse(pParser, 0, t0, &cxt); goto abort_parse; } t0.n = tGetToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); @@ -47,7 +47,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { break; } case TK_NK_SEMI: { - NewParse(pParser, 0, t0, &cxt); + Parse(pParser, 0, t0, &cxt); goto abort_parse; } case TK_NK_QUESTION: @@ -64,8 +64,8 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { goto abort_parse; } default: - NewParse(pParser, t0.type, t0, &cxt); - // NewParseTrace(stdout, ""); + Parse(pParser, t0.type, t0, &cxt); + // ParseTrace(stdout, ""); if (!cxt.valid) { goto abort_parse; } @@ -73,7 +73,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - NewParseFree(pParser, free); + ParseFree(pParser, free); if (cxt.valid) { *pQuery = calloc(1, sizeof(SQuery)); if (NULL == *pQuery) { diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/parInsert.c similarity index 99% rename from source/libs/parser/src/insertParser.c rename to source/libs/parser/src/parInsert.c index 3241dc196b..43cc308483 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/parInsert.c @@ -13,13 +13,12 @@ * along with this program. If not, see . */ -#include "insertParser.h" - -#include "dataBlockMgt.h" -#include "parserUtil.h" +#include "parInsertData.h" +#include "parInt.h" +#include "parUtil.h" +#include "parToken.h" #include "tglobal.h" #include "ttime.h" -#include "ttoken.h" #include "ttypes.h" #define NEXT_TOKEN(pSql, sToken) \ diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/parInsertData.c similarity index 99% rename from source/libs/parser/src/dataBlockMgt.c rename to source/libs/parser/src/parInsertData.c index 34e440dd19..da5a652018 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/parInsertData.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "dataBlockMgt.h" +#include "parInsertData.h" #include "catalog.h" -#include "parserUtil.h" +#include "parUtil.h" #include "querynodes.h" #define IS_RAW_PAYLOAD(t) \ diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/parTokenizer.c similarity index 99% rename from source/libs/parser/src/ttokenizer.c rename to source/libs/parser/src/parTokenizer.c index b6435e95ad..089b9c30b5 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -14,10 +14,9 @@ */ #include "os.h" - +#include "parToken.h" #include "thash.h" #include "taosdef.h" -#include "ttoken.h" #include "ttokendef.h" // All the keywords of the SQL language are stored in a hash table diff --git a/source/libs/parser/src/astTranslate.c b/source/libs/parser/src/parTranslater.c similarity index 99% rename from source/libs/parser/src/astTranslate.c rename to source/libs/parser/src/parTranslater.c index be683c42cf..b86500bb08 100644 --- a/source/libs/parser/src/astTranslate.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "parserInt.h" +#include "parInt.h" #include "catalog.h" #include "cmdnodes.h" #include "functionMgt.h" -#include "parserUtil.h" +#include "parUtil.h" #include "ttime.h" static bool afterGroupBy(ESqlClause clause) { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parUtil.c similarity index 99% rename from source/libs/parser/src/parserUtil.c rename to source/libs/parser/src/parUtil.c index 73e62edc3c..aa2516e2b9 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "parserUtil.h" +#include "parUtil.h" static char* getSyntaxErrFormat(int32_t errCode) { switch (errCode) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 64418ad961..868bd75520 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -15,9 +15,8 @@ #include "parser.h" -#include "insertParser.h" -#include "parserInt.h" -#include "ttoken.h" +#include "parInt.h" +#include "parToken.h" static bool isInsertSql(const char* pStr, size_t length) { int32_t index = 0; @@ -47,5 +46,14 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { } void qDestroyQuery(SQuery* pQueryNode) { - // todo + if (NULL == pQueryNode) { + return; + } + nodesDestroyNode(pQueryNode->pRoot); + tfree(pQueryNode->pResSchema); + if (NULL != pQueryNode->pCmdMsg) { + tfree(pQueryNode->pCmdMsg->pMsg); + tfree(pQueryNode->pCmdMsg); + } + tfree(pQueryNode); } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/sql.c similarity index 97% rename from source/libs/parser/src/new_sql.c rename to source/libs/parser/src/sql.c index 058e1b35cd..34d38c1c06 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/sql.c @@ -33,9 +33,9 @@ #include #include "nodes.h" -#include "ttoken.h" +#include "parToken.h" #include "ttokendef.h" -#include "astCreateFuncs.h" +#include "parAst.h" /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -59,7 +59,7 @@ ** YYACTIONTYPE is the data type used for "action codes" - numbers ** that indicate what to do in response to the next ** token. -** NewParseTOKENTYPE is the data type used for minor type for terminal +** ParseTOKENTYPE is the data type used for minor type for terminal ** symbols. Background: A "minor type" is a semantic ** value associated with a terminal or non-terminal ** symbols. For example, for an "ID" terminal symbol, @@ -70,16 +70,16 @@ ** symbols. ** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of -** which is NewParseTOKENTYPE. The entry in the union +** which is ParseTOKENTYPE. The entry in the union ** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() -** NewParseARG_SDECL A static variable declaration for the %extra_argument -** NewParseARG_PDECL A parameter declaration for the %extra_argument -** NewParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** NewParseARG_STORE Code to store %extra_argument into yypParser -** NewParseARG_FETCH Code to extract %extra_argument from yypParser -** NewParseCTX_* As NewParseARG_ except for %extra_context +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -101,10 +101,10 @@ #define YYCODETYPE unsigned char #define YYNOCODE 208 #define YYACTIONTYPE unsigned short int -#define NewParseTOKENTYPE SToken +#define ParseTOKENTYPE SToken typedef union { int yyinit; - NewParseTOKENTYPE yy0; + ParseTOKENTYPE yy0; ENullOrder yy9; SDatabaseOptions* yy103; SToken yy161; @@ -121,16 +121,16 @@ typedef union { #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif -#define NewParseARG_SDECL SAstCreateContext* pCxt ; -#define NewParseARG_PDECL , SAstCreateContext* pCxt -#define NewParseARG_PARAM ,pCxt -#define NewParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; -#define NewParseARG_STORE yypParser->pCxt =pCxt ; -#define NewParseCTX_SDECL -#define NewParseCTX_PDECL -#define NewParseCTX_PARAM -#define NewParseCTX_FETCH -#define NewParseCTX_STORE +#define ParseARG_SDECL SAstCreateContext* pCxt ; +#define ParseARG_PDECL , SAstCreateContext* pCxt +#define ParseARG_PARAM ,pCxt +#define ParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; +#define ParseARG_STORE yypParser->pCxt =pCxt ; +#define ParseCTX_SDECL +#define ParseCTX_PDECL +#define ParseCTX_PARAM +#define ParseCTX_FETCH +#define ParseCTX_STORE #define YYNSTATE 278 #define YYNRULE 236 #define YYNTOKEN 134 @@ -544,8 +544,8 @@ struct yyParser { #ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ #endif - NewParseARG_SDECL /* A place to hold %extra_argument */ - NewParseCTX_SDECL /* A place to hold %extra_context */ + ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -581,7 +581,7 @@ static char *yyTracePrompt = 0; ** Outputs: ** None. */ -void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ +void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ yyTraceFILE = TraceFILE; yyTracePrompt = zTracePrompt; if( yyTraceFILE==0 ) yyTracePrompt = 0; @@ -1082,7 +1082,7 @@ static int yyGrowStack(yyParser *p){ #endif /* Datatype of the argument to the memory allocated passed as the -** second argument to NewParseAlloc() below. This can be changed by +** second argument to ParseAlloc() below. This can be changed by ** putting an appropriate #define in the %include section of the input ** grammar. */ @@ -1092,9 +1092,9 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ +void ParseInit(void *yypRawParser ParseCTX_PDECL){ yyParser *yypParser = (yyParser*)yypRawParser; - NewParseCTX_STORE + ParseCTX_STORE #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif @@ -1118,7 +1118,7 @@ void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ #endif } -#ifndef NewParse_ENGINEALWAYSONSTACK +#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -1129,18 +1129,18 @@ void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ ** ** Outputs: ** A pointer to a parser. This pointer is used in subsequent calls -** to NewParse and NewParseFree. +** to Parse and ParseFree. */ -void *NewParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) NewParseCTX_PDECL){ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ yyParser *yypParser; yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( yypParser ){ - NewParseCTX_STORE - NewParseInit(yypParser NewParseCTX_PARAM); + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); } return (void*)yypParser; } -#endif /* NewParse_ENGINEALWAYSONSTACK */ +#endif /* Parse_ENGINEALWAYSONSTACK */ /* The following function deletes the "minor type" or semantic value @@ -1155,8 +1155,8 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1321,7 +1321,7 @@ static void yy_pop_parser_stack(yyParser *pParser){ /* ** Clear all secondary memory allocations from the parser */ -void NewParseFinalize(void *p){ +void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 @@ -1329,7 +1329,7 @@ void NewParseFinalize(void *p){ #endif } -#ifndef NewParse_ENGINEALWAYSONSTACK +#ifndef Parse_ENGINEALWAYSONSTACK /* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. @@ -1338,23 +1338,23 @@ void NewParseFinalize(void *p){ ** is defined in a %include section of the input grammar) then it is ** assumed that the input pointer is never NULL. */ -void NewParseFree( +void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ #ifndef YYPARSEFREENEVERNULL if( p==0 ) return; #endif - NewParseFinalize(p); + ParseFinalize(p); (*freeProc)(p); } -#endif /* NewParse_ENGINEALWAYSONSTACK */ +#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. */ #ifdef YYTRACKMAXSTACKDEPTH -int NewParseStackPeak(void *p){ +int ParseStackPeak(void *p){ yyParser *pParser = (yyParser*)p; return pParser->yyhwm; } @@ -1378,7 +1378,7 @@ static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; ** Return the number of missed state/lookahead combinations. */ #if defined(YYCOVERAGE) -int NewParseCoverage(FILE *out){ +int ParseCoverage(FILE *out){ int stateno, iLookAhead, i; int nMissed = 0; for(stateno=0; statenoyytos++; @@ -1840,14 +1840,14 @@ static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - NewParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - NewParseCTX_PDECL /* %extra_context */ + ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - NewParseARG_FETCH + ParseARG_FETCH (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; @@ -2680,8 +2680,8 @@ static YYACTIONTYPE yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -2692,8 +2692,8 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -2703,10 +2703,10 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - NewParseTOKENTYPE yyminor /* The minor type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ @@ -2717,8 +2717,8 @@ static void yy_syntax_error( } pCxt->valid = false; /************ End %syntax_error code ******************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* @@ -2727,8 +2727,8 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - NewParseARG_FETCH - NewParseCTX_FETCH + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -2742,13 +2742,13 @@ static void yy_accept( ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - NewParseCTX_STORE + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* The main parser program. ** The first argument is a pointer to a structure obtained from -** "NewParseAlloc" which describes the current state of the parser. +** "ParseAlloc" which describes the current state of the parser. ** The second argument is the major token number. The third is ** the minor token. The fourth optional argument is whatever the ** user wants (and specified in the grammar) and is available for @@ -2765,11 +2765,11 @@ static void yy_accept( ** Outputs: ** None. */ -void NewParse( +void Parse( void *yyp, /* The parser */ int yymajor, /* The major token code number */ - NewParseTOKENTYPE yyminor /* The value for the token */ - NewParseARG_PDECL /* Optional %extra_argument parameter */ + ParseTOKENTYPE yyminor /* The value for the token */ + ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; YYACTIONTYPE yyact; /* The parser action. */ @@ -2780,8 +2780,8 @@ void NewParse( int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif yyParser *yypParser = (yyParser*)yyp; /* The parser */ - NewParseCTX_FETCH - NewParseARG_STORE + ParseCTX_FETCH + ParseARG_STORE assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) @@ -2806,7 +2806,7 @@ void NewParse( yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor NewParseCTX_PARAM); + yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY @@ -2939,7 +2939,7 @@ void NewParse( ** Return the fallback token corresponding to canonical token iToken, or ** 0 if iToken has no fallback. */ -int NewParseFallback(int iToken){ +int ParseFallback(int iToken){ #ifdef YYFALLBACK if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ return yyFallback[iToken]; diff --git a/source/libs/parser/test/parserTest.cpp b/source/libs/parser/test/parserAstTest.cpp similarity index 99% rename from source/libs/parser/test/parserTest.cpp rename to source/libs/parser/test/parserAstTest.cpp index d154ed09a8..4d29475ad7 100644 --- a/source/libs/parser/test/parserTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -18,7 +18,7 @@ #include -#include "parserInt.h" +#include "parInt.h" using namespace std; using namespace testing; diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/parserInsertTest.cpp similarity index 99% rename from source/libs/parser/test/insertParserTest.cpp rename to source/libs/parser/test/parserInsertTest.cpp index 7c46981028..3d4a6e0eb8 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -15,8 +15,7 @@ #include -#include "insertParser.h" -// #include "mockCatalog.h" +#include "parInt.h" using namespace std; using namespace testing; diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp deleted file mode 100644 index c7225639c9..0000000000 --- a/source/libs/parser/test/tokenizerTest.cpp +++ /dev/null @@ -1,730 +0,0 @@ -#if 0 - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include "os.h" - -#include "taos.h" -#include "tvariant.h" -#include "tdef.h" -#include "ttoken.h" -#include "astGenerator.h" -#include "parserUtil.h" -#include "parserInt.h" - -namespace { -int32_t testValidateName(char* name) { - SToken token = {0}; - token.z = name; - token.n = strlen(name); - token.type = 0; - - tGetToken(name, &token.type); - return parserValidateIdToken(&token); -} - -SToken createToken(char* s) { - SToken t = {0}; - - t.type = TK_STRING; - t.z = s; - t.n = strlen(s); - return t; -} -} // namespace - -static void _init_tvariant_bool(SVariant* t) { - t->i = TSDB_FALSE; - t->nType = TSDB_DATA_TYPE_BOOL; -} - -static void _init_tvariant_tinyint(SVariant* t) { - t->i = -27; - t->nType = TSDB_DATA_TYPE_TINYINT; -} - -static void _init_tvariant_int(SVariant* t) { - t->i = -23997659; - t->nType = TSDB_DATA_TYPE_INT; -} - -static void _init_tvariant_bigint(SVariant* t) { - t->i = -3333333333333; - t->nType = TSDB_DATA_TYPE_BIGINT; -} - -static void _init_tvariant_float(SVariant* t) { - t->d = -8991212199.8987878776; - t->nType = TSDB_DATA_TYPE_FLOAT; -} - -static void _init_tvariant_binary(SVariant* t) { - taosVariantDestroy(t); - - t->pz = (char*)calloc(1, 20); //"2e3"); - t->nType = TSDB_DATA_TYPE_BINARY; - strcpy(t->pz, "2e5"); - t->nLen = strlen(t->pz); -} - -static void _init_tvariant_nchar(SVariant* t) { - taosVariantDestroy(t); - - t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); - t->nType = TSDB_DATA_TYPE_NCHAR; - wcscpy(t->wpz, L"-2000000.8765"); - t->nLen = twcslen(t->wpz); -} - -TEST(testCase, validateToken_test) { - char t01[] = "abc"; - EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); - - char t110[] = "`1233abc.911`"; - EXPECT_EQ(testValidateName(t110), TSDB_CODE_SUCCESS); - - char t02[] = "'abc'"; - EXPECT_EQ(testValidateName(t02), TSDB_CODE_TSC_INVALID_OPERATION); - - char t1[] = "abc.def"; - EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); - printf("%s\n", t1); - - char t98[] = "abc.DeF"; - EXPECT_EQ(testValidateName(t98), TSDB_CODE_SUCCESS); - EXPECT_STREQ(t98, "abc.def"); - printf("%s\n", t98); - - char t97[] = "257.abc"; - EXPECT_EQ(testValidateName(t97), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t97); - - char t96[] = "_257.aBc"; - EXPECT_EQ(testValidateName(t96), TSDB_CODE_SUCCESS); - EXPECT_STREQ(t96, "_257.abc"); - printf("%s\n", t96); - - char t99[] = "abc . def"; - EXPECT_EQ(testValidateName(t99), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t99); - - char t2[] = "'abc.def'"; - EXPECT_EQ(testValidateName(t2), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t2); - - char t3[] = "'abc'.def"; - EXPECT_EQ(testValidateName(t3), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t3); - - char t4[] = "'abc'.'def'"; - EXPECT_EQ(testValidateName(t4), TSDB_CODE_TSC_INVALID_OPERATION); - - char t5[] = "table.'def'"; - EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION); - - char t6[] = "'table'.'def'"; - EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION); - - char t7[] = "'_ab1234'.'def'"; - EXPECT_EQ(testValidateName(t7), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t7); - - char t8[] = "'_ab&^%1234'.'def'"; - EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION); - - char t9[] = "'_123'.'gtest中文'"; - EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10[] = "abc.'gtest中文'"; - EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10_1[] = "abc.'中文gtest'"; - EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t11[] = "'192.168.0.1'.abc"; - EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION); - - char t12[] = "192.168.0.1.abc"; - EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION); - - char t13[] = "abc."; - EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION); - - char t14[] = ".abc"; - EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION); - - char t15[] = ".'abc'"; - EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION); - - char t16[] = ".abc'"; - EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION); - - char t17[] = "123a.\"abc\""; - EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t17); - - char t18[] = "a.\"abc\""; - EXPECT_EQ(testValidateName(t18), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t18); - - char t19[] = "'_ab1234'.'def'.'ab123'"; - EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION); - - char t20[] = "'_ab1234*&^'"; - EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION); - - char t21[] = "'1234_abc'"; - EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION); - - // =======Containing capital letters================= - char t30[] = "ABC"; - EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); - - char t31[] = "'ABC'"; - EXPECT_EQ(testValidateName(t31), TSDB_CODE_TSC_INVALID_OPERATION); - - char t32[] = "ABC.def"; - EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); - - char t33[] = "'ABC.def"; - EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION); - - char t33_0[] = "abc.DEF'"; - EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t34[] = "'ABC.def'"; - // int32_t tmp0 = testValidateName(t34); - EXPECT_EQ(testValidateName(t34), TSDB_CODE_TSC_INVALID_OPERATION); - - char t35[] = "'ABC'.def"; - EXPECT_EQ(testValidateName(t35), TSDB_CODE_TSC_INVALID_OPERATION); - - char t36[] = "ABC.DEF"; - EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); - - char t37[] = "abc.DEF"; - EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); - - char t37_1[] = "abc._123DEF"; - EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); - - char t38[] = "'abc'.\"DEF\""; - EXPECT_EQ(testValidateName(t38), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t39[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION); - - char t40[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION); - - char t41[] = "'_abXYZ1234'.'deFF'"; - EXPECT_EQ(testValidateName(t41), TSDB_CODE_TSC_INVALID_OPERATION); - - char t42[] = "'_abDEF&^%1234'.'DIef'"; - EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION); - - char t43[] = "'_123'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION); - - char t44[] = "'aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION); - - char t45[] = "'ABC'."; - EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION); - - char t46[] = ".'ABC'"; - EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION); - - char t47[] = "a.\"aTWc\""; - EXPECT_EQ(testValidateName(t47), TSDB_CODE_TSC_INVALID_OPERATION); - - // ================has space ================= - char t60[] = " ABC "; - EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION); - - char t60_1[] = " ABC "; - EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t62[] = " ABC . def "; - EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63[] = "' ABC . def "; - EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63_0[] = " abc . DEF ' "; - EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t64[] = " ' ABC . def ' "; - // int32_t tmp1 = testValidateName(t64); - EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION); - - char t65[] = " ' ABC '. def "; - EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION); - - char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION); - - char t67[] = "abc . ' DEF '"; - EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION); - - char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t69[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION); - - char t70[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION); - - char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION); - - char t72[] = "'_abDEF&^%1234'.' DIef'"; - EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION); - - char t73[] = "'_123'.' Gtest中文'"; - EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION); - - char t74[] = "' aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION); - - char t75[] = "' ABC '."; - EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION); - - char t76[] = ".' ABC'"; - EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION); - - char t77[] = " a . \"aTWc\" "; - EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION); - - char t78[] = " a.\"aTWc \""; - EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION); - - // ===============muti string by space =================== - // There's no such case. - // char t160[] = "A BC"; - // EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION); - // printf("end:%s\n", t160); - - // There's no such case. - // char t161[] = "' A BC '"; - // EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION); - - char t162[] = " AB C . de f "; - EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163[] = "' AB C . de f "; - EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163_0[] = " ab c . DE F ' "; - EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t164[] = " ' AB C . de f ' "; - // int32_t tmp2 = testValidateName(t164); - EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION); - - char t165[] = " ' A BC '. de f "; - EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION); - - char t166[] = "' AB C '.' DE F '"; - EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION); - - char t167[] = "ab c . ' D EF '"; - EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION); - - char t168[] = "' a bc '.' DE F '"; - EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION); -} - -#if 0 -TEST(testCase, tvariant_convert) { - // 1. bool data to all other data types - SVariant t = {0}; - _init_tvariant_bool(&t); - - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "FALSE"); - taosVariantDestroy(&t); - - _init_tvariant_bool(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"FALSE"); - taosVariantDestroy(&t); - - // 2. tinyint to other data types - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-27"); - taosVariantDestroy(&t); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-27"); - taosVariantDestroy(&t); - - // 3. int to other data - // types////////////////////////////////////////////////////////////////// - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-23997659"); - taosVariantDestroy(&t); - - _init_tvariant_int(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-23997659"); - taosVariantDestroy(&t); - - // 4. bigint to other data - // type////////////////////////////////////////////////////////////////////////////// - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.d, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.d, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-3333333333333"); - taosVariantDestroy(&t); - - _init_tvariant_bigint(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-3333333333333"); - taosVariantDestroy(&t); - - // 5. float to other data - // types//////////////////////////////////////////////////////////////////////// - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -8991212199); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-8991212199.898788"); - taosVariantDestroy(&t); - - _init_tvariant_float(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-8991212199.898788"); - taosVariantDestroy(&t); - - // 6. binary to other data types - // ////////////////////////////////////////////////////////////////// - t.pz = "true"; - t.nLen = strlen(t.pz); - t.nType = TSDB_DATA_TYPE_BINARY; - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 1); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "2e5"); - taosVariantDestroy(&t); - - _init_tvariant_binary(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"2e5"); - taosVariantDestroy(&t); - - // 7. nchar to other data types - // ////////////////////////////////////////////////////////////////// - t.wpz = L"FALSE"; - t.nLen = wcslen(t.wpz); - t.nType = TSDB_DATA_TYPE_NCHAR; - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i, 0); - - _init_tvariant_nchar(&t); - EXPECT_LE(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i, -2000000); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.d, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.d, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-2000000.8765"); - taosVariantDestroy(&t); - - _init_tvariant_nchar(&t); - EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-2000000.8765"); - taosVariantDestroy(&t); -} -#endif - -TEST(testCase, tGetToken_Test) { - char* s = ".123 "; - uint32_t type = 0; - - int32_t len = tGetToken(s, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s) - 1); - - char s1[] = "1.123e10 "; - len = tGetToken(s1, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s1) - 1); - - char s4[] = "0xff "; - len = tGetToken(s4, &type); - EXPECT_EQ(type, TK_HEX); - EXPECT_EQ(len, strlen(s4) - 1); - - // invalid data type - char s2[] = "e10 "; - len = tGetToken(s2, &type); - EXPECT_FALSE(type == TK_FLOAT); - - char s3[] = "1.1.1.1"; - len = tGetToken(s3, &type); - EXPECT_EQ(type, TK_IPTOKEN); - EXPECT_EQ(len, strlen(s3)); - - char s5[] = "0x "; - len = tGetToken(s5, &type); - EXPECT_FALSE(type == TK_HEX); -} - -TEST(testCase, isValidNumber_test) { - SToken t1 = createToken("123abc"); - - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("0xabc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createToken("0b11101"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createToken(".134abc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("1e1 "); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("1+2"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createToken("-0x123"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createToken("-1"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER); - - t1 = createToken("-0b1110"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createToken("-.234"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT); -} - -TEST(testCase, generateAST_test) { - SSqlInfo info = doGenerateAST("select * from t1 where ts < now"); - ASSERT_EQ(info.valid, true); - - SSqlInfo info1 = doGenerateAST("select * from `t.1abc` where ts. */ -#include "plannerInt.h" +#include "planInt.h" #include "functionMgt.h" diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/planPhysiCreater.c similarity index 99% rename from source/libs/planner/src/physicalPlan.c rename to source/libs/planner/src/planPhysiCreater.c index fa5732d667..456f00090e 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "plannerInt.h" +#include "planInt.h" #include "functionMgt.h" diff --git a/source/libs/planner/src/splitPlan.c b/source/libs/planner/src/planSpliter.c similarity index 99% rename from source/libs/planner/src/splitPlan.c rename to source/libs/planner/src/planSpliter.c index 97e2eefb67..5a5e1d46c6 100644 --- a/source/libs/planner/src/splitPlan.c +++ b/source/libs/planner/src/planSpliter.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "plannerInt.h" +#include "planInt.h" #define SPLIT_FLAG_MASK(n) (1 << n) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 15d5c09f04..fa0dc549c8 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -15,7 +15,7 @@ #include "planner.h" -#include "plannerInt.h" +#include "planInt.h" int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index ae25b157f6..3748d37d74 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -18,7 +18,7 @@ #include #include "parser.h" -#include "plannerInt.h" +#include "planInt.h" using namespace std; using namespace testing; From b57f7414e2866ee81fba0a16264eccefa4340f0a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Mar 2022 16:18:35 +0800 Subject: [PATCH 36/39] [td-13039] refactor. --- include/common/tcommon.h | 28 ++++++---- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 71 ++++++++++++++----------- 3 files changed, 59 insertions(+), 42 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index b3b47b4c68..b5bd088006 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -56,7 +56,7 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; - int32_t tupleSize; + int32_t rowSize; int32_t numOfCols; union {int64_t uid; int64_t blockId;}; } SDataBlockInfo; @@ -70,10 +70,10 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg* pBlockAgg; - SArray* pDataBlock; // SArray - SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg *pBlockAgg; + SArray *pDataBlock; // SArray + SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { @@ -218,18 +218,17 @@ typedef struct SColumn { int64_t dataBlockId; }; - char name[TSDB_COL_NAME_LEN]; - int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) union { int16_t colId; int16_t slotId; }; + char name[TSDB_COL_NAME_LEN]; + int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) int16_t type; int32_t bytes; uint8_t precision; uint8_t scale; - // SColumnInfo info; } SColumn; typedef struct SLimit { @@ -254,11 +253,20 @@ typedef struct SFunctParam { } SFunctParam; // the structure for sql function in select clause +typedef struct SResSchame { + int8_t type; + int32_t colId; + int32_t bytes; + int32_t precision; + int32_t scale; + char name[TSDB_COL_NAME_LEN]; +} SResSchema; + +// TODO move away to executor.h typedef struct SExprBasicInfo { - SSchema resSchema; // TODO refactor + SResSchema resSchema; int16_t numOfParams; // argument value of each function SFunctParam *pParam; -// SVariant param[3]; // parameters are not more than 3 } SExprBasicInfo; typedef struct SExprInfo { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 58839a0711..0e0a5f9333 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -632,7 +632,7 @@ typedef struct SOrderOperatorInfo { uint64_t totalElapsed; // total elapsed time } SOrderOperatorInfo; -SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cce05f93f9..30a5bd3f80 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -347,6 +347,7 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.blockId = pNode->dataBlockId; + pBlock->info.rowSize = pNode->resultRowSize; for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; @@ -683,12 +684,6 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t w.ekey = w.skey + pInterval->interval - 1; } } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ -// ASSERT(win->skey <= win->ekey); // todo no need this return w; } @@ -3348,14 +3343,13 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t cleanupResultRowEntry(pEntry); pCtx[i].resultInfo = pEntry; - pCtx[i].pOutput = pData->pData; // todo remove it pCtx[i].currentStage = stage; // set the timestamp output buffer for top/bottom/diff query - int32_t fid = pCtx[i].functionId; - if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } +// int32_t fid = pCtx[i].functionId; +// if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { +// if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; +// } } initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); @@ -3663,7 +3657,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.tupleSize); + int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { return; } @@ -5187,9 +5181,10 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { #endif } +// TODO remove it static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); -SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -5200,9 +5195,9 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return NULL; } - size_t numOfSources = taosArrayGetSize(pSources); + size_t numOfSources = LIST_LENGTH(pSources); - pInfo->pSources = taosArrayDup(pSources); +// pInfo->pSources = taosArrayDup(pSources); pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { tfree(pInfo); @@ -5222,8 +5217,8 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); } - size_t size = taosArrayGetSize(pExprInfo); - pInfo->pResult = createResultDataBlock(pExprInfo); + size_t size = taosArrayGetSize(pExprInfo); + pInfo->pResult = createResultDataBlock(pExprInfo); pInfo->seqLoadData = true; tsem_init(&pInfo->ready, 0, 0); @@ -5277,11 +5272,12 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { SColumnInfoData colInfoData = {0}; SExprInfo* p = taosArrayGetP(pExprInfo, i); - SSchema* pSchema = &p->base.resSchema; + SResSchema* pSchema = &p->base.resSchema; colInfoData.info.type = pSchema->type; colInfoData.info.colId = pSchema->colId; colInfoData.info.bytes = pSchema->bytes; - + colInfoData.info.scale = pSchema->scale; + colInfoData.info.precision = pSchema->precision; taosArrayPush(pResult, &colInfoData); } @@ -8016,9 +8012,19 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -SArray* createExprInfo(SAggPhysiNode* pPhyNode, int32_t* resultRowSize) { - *resultRowSize = pPhyNode->node.pOutputDataBlockDesc->resultRowSize; +static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { + SResSchema s = {0}; + s.scale = scale; + s.precision = precision; + s.type = type; + s.bytes = bytes; + s.colId = slotId; + strncpy(s.name, name, tListLen(s.name)); + return s; +} + +SArray* createExprInfo(SAggPhysiNode* pPhyNode) { int32_t numOfAggFuncs = LIST_LENGTH(pPhyNode->pAggFuncs); SArray* pArray = taosArrayInit(numOfAggFuncs, POINTER_BYTES); @@ -8038,9 +8044,12 @@ SArray* createExprInfo(SAggPhysiNode* pPhyNode, int32_t* resultRowSize) { ASSERT(pTargetNode->slotId == i); SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - pExp->base.resSchema = createSchema(pFuncNode->node.resType.type, pFuncNode->node.resType.bytes, pTargetNode->slotId, pFuncNode->node.aliasName); - pExp->pExpr->_function.pFunctNode = pFuncNode; + SDataType *pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, + pType->scale, pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.pFunctNode = pFuncNode; strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); // TODO: value parameter needs to be handled @@ -8089,16 +8098,17 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; - size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); - tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); + size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, (uint64_t)queryId, taskId); int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, + pScanPhyNode->reverse, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { - // SExchangePhysiNode* pEx = (SExchangePhysiNode*) pPhyNode; - // return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); + SExchangePhysiNode* pEx = (SExchangePhysiNode*)pPhyNode; + return createExchangeOperatorInfo(pEx->pSrcEndPoints, NULL, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0}; int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); @@ -8136,8 +8146,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); - int32_t resultRowSize = 0; - SArray* pExprInfo = createExprInfo((SAggPhysiNode*)pPhyNode, &resultRowSize); + SArray* pExprInfo = createExprInfo((SAggPhysiNode*)pPhyNode); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); return createAggregateOperatorInfo(op, pExprInfo, pResBlock, pTaskInfo, pTableGroupInfo); } From 54afca4d5794b645ed9be97089c178ac1c93d902 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Mar 2022 03:34:31 -0500 Subject: [PATCH 37/39] TD-13747 bugfix --- source/libs/function/src/builtins.c | 7 +++++- source/libs/nodes/src/nodesCloneFuncs.c | 9 +++++++ source/libs/nodes/src/nodesUtilFuncs.c | 2 ++ source/libs/parser/src/parTranslater.c | 28 +++++++++++++++------- source/libs/planner/src/planPhysiCreater.c | 2 +- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 714bf7e146..edb0acf075 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -77,7 +77,9 @@ const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFun int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { switch(pFunc->funcType) { - case FUNCTION_TYPE_COUNT: pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};break; + case FUNCTION_TYPE_COUNT: + pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + break; case FUNCTION_TYPE_SUM: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraType = pParam->node.resType.type; @@ -103,6 +105,9 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; break; } + case FUNCTION_TYPE_CONCAT: + // todo + break; default: ASSERT(0); // to found the fault ASAP. } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 5a3d533145..a60366c0d6 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -271,6 +271,13 @@ static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { return (SNode*)pDst; } +static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { + COPY_SCALAR_FIELD(addr); + COPY_SCALAR_FIELD(taskId); + COPY_SCALAR_FIELD(schedId); + return (SNode*)pDst; +} + SNodeptr nodesCloneNode(const SNodeptr pNode) { if (NULL == pNode) { return NULL; @@ -306,6 +313,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return dataBlockDescCopy((const SDataBlockDescNode*)pNode, (SDataBlockDescNode*)pDst); case QUERY_NODE_SLOT_DESC: return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanCopy((const SScanLogicNode*)pNode, (SScanLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_AGG: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f943b63b80..dc533d1423 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -74,6 +74,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSlotDescNode)); case QUERY_NODE_COLUMN_DEF: return makeNode(type, sizeof(SColumnDefNode)); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return makeNode(type, sizeof(SDownstreamSourceNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b86500bb08..0ed73ea1f6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -891,7 +891,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt if (TSDB_SUPER_TABLE == pTableMeta->tableType) { code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } else { - // todo; + // todo : drop normal table or child table code = TSDB_CODE_FAILED; } } @@ -1203,6 +1203,9 @@ static int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) { } static void destroyTranslateContext(STranslateContext* pCxt) { + if (NULL != pCxt->pNsLevel) { + + } taosArrayDestroy(pCxt->pNsLevel); if (NULL != pCxt->pCmdMsg) { tfree(pCxt->pCmdMsg->pMsg); @@ -1222,6 +1225,11 @@ static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema strcpy(pSchema->name, pCol->colName); } +static void destroyCreateTbReq(SVCreateTbReq* pReq) { + tfree(pReq->name); + tfree(pReq->ntbCfg.pSchema); +} + static int32_t buildNormalTableBatchReq( const char* pTableName, const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { SVCreateTbReq req = {0}; @@ -1230,6 +1238,7 @@ static int32_t buildNormalTableBatchReq( req.ntbCfg.nCols = LIST_LENGTH(pColumns); req.ntbCfg.pSchema = calloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } SNode* pCol; @@ -1242,6 +1251,7 @@ static int32_t buildNormalTableBatchReq( pBatch->info = *pVgroupInfo; pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } taosArrayPush(pBatch->req.pArray, &req); @@ -1311,18 +1321,20 @@ static int32_t rewriteToVnodeModifOpStmt(SQuery* pQuery, SArray* pBufArray) { } static int32_t buildCreateTableDataBlock(const SCreateTableStmt* pStmt, const SVgroupInfo* pInfo, SArray** pBufArray) { + *pBufArray = taosArrayInit(1, POINTER_BYTES); + if (NULL == *pBufArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SVgroupTablesBatch tbatch = {0}; int32_t code = buildNormalTableBatchReq(pStmt->tableName, pStmt->pCols, pInfo, &tbatch); - if (TSDB_CODE_SUCCESS == code) { - *pBufArray = taosArrayInit(1, POINTER_BYTES); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } if (TSDB_CODE_SUCCESS == code) { code = serializeVgroupTablesBatch(&tbatch, *pBufArray); } destroyCreateTbReqBatch(&tbatch); + if (TSDB_CODE_SUCCESS != code) { + // todo : destroyCreateTbReqArray(*pBufArray); + } return code; } @@ -1331,7 +1343,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { SVgroupInfo info = {0}; int32_t code = getTableHashVgroup(pCxt->pParseCxt, pStmt->dbName, pStmt->tableName, &info); - SArray* pBufArray; + SArray* pBufArray = NULL; if (TSDB_CODE_SUCCESS == code) { code = buildCreateTableDataBlock(pStmt, &info, &pBufArray); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 456f00090e..5327cf89c5 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -48,7 +48,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_ pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = false; - pSlot->output = false; + pSlot->output = true; return (SNode*)pSlot; } From e055dd5fd766c8d49991661d4bdcdb3d6a1307e8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Mar 2022 03:58:51 -0500 Subject: [PATCH 38/39] TD-13747 bugfix --- include/libs/nodes/plannodes.h | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 37 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b2ae1b4c67..0f0a0176fb 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -143,6 +143,7 @@ typedef struct SScanPhysiNode { int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC int32_t count; // repeat count int32_t reverse; // reverse scan count + SName tableName; } SScanPhysiNode; typedef SScanPhysiNode SSystemTableScanPhysiNode; @@ -217,7 +218,6 @@ typedef struct SSubplan { SNodeList* pParents; // the data destination subplan, get data from current subplan SPhysiNode* pNode; // physical plan of current subplan SDataSinkNode* pDataSink; // data of the subplan flow into the datasink - SName tableName; // scan table } SSubplan; typedef struct SQueryPlan { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 5e5c3d862e..875f19b660 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1261,6 +1261,38 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkDownstreamSourceAddr = "Addr"; +static const char* jkDownstreamSourceTaskId = "TaskId"; +static const char* jkDownstreamSourceSchedId = "SchedId"; + +static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { + const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkDownstreamSourceAddr, queryNodeAddrToJson, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceSchedId, pNode->schedId); + } + + return code; +} + +static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj; + + int32_t code = tjsonToObject(pJson, jkDownstreamSourceAddr, jsonToQueryNodeAddr, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceSchedId, &pNode->schedId); + } + + return code; +} + static const char* jkDataBlockDescDataBlockId = "DataBlockId"; static const char* jkDataBlockDescSlots = "Slots"; static const char* jkDataBlockResultRowSize = "ResultRowSize"; @@ -1381,6 +1413,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SLOT_DESC: return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_COLUMN_DEF: + break; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return downstreamSourceNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: break; case QUERY_NODE_SELECT_STMT: @@ -1470,6 +1505,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDataBlockDescNode(pJson, pObj); case QUERY_NODE_SLOT_DESC: return jsonToSlotDescNode(pJson, pObj); + case QUERY_NODE_DOWNSTREAM_SOURCE: + return jsonToDownstreamSourceNode(pJson, pObj); // case QUERY_NODE_SET_OPERATOR: // break; // case QUERY_NODE_SELECT_STMT: From 8196b6ee9ac79a74587d504b2e816386b6eb233c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Mar 2022 18:07:07 +0800 Subject: [PATCH 39/39] [td-13039] refactor. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 28 +++++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0e0a5f9333..e95457b91e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -632,7 +632,7 @@ typedef struct SOrderOperatorInfo { uint64_t totalElapsed; // total elapsed time } SOrderOperatorInfo; -SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 30a5bd3f80..bc257f797a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5181,10 +5181,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { #endif } -// TODO remove it -static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); - -SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +// TODO handle the error +SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -5196,8 +5194,19 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArra } size_t numOfSources = LIST_LENGTH(pSources); + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfSources; ++i) { + SNodeListNode* pNode = nodesListGetNode((SNodeList*) pSources, i); + taosArrayPush(pInfo->pSources, pNode); + } -// pInfo->pSources = taosArrayDup(pSources); pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { tfree(pInfo); @@ -5217,8 +5226,8 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, const SArra taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); } - size_t size = taosArrayGetSize(pExprInfo); - pInfo->pResult = createResultDataBlock(pExprInfo); + size_t size = pBlock->info.numOfCols; + pInfo->pResult = pBlock; pInfo->seqLoadData = true; tsem_init(&pInfo->ready, 0, 0); @@ -8105,8 +8114,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { - SExchangePhysiNode* pEx = (SExchangePhysiNode*)pPhyNode; - return createExchangeOperatorInfo(pEx->pSrcEndPoints, NULL, pTaskInfo); + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); + return createExchangeOperatorInfo(pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0};