From 3936128742e8f468e99c6d7119d3ebe5ee561159 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Dec 2021 07:37:54 -0500 Subject: [PATCH 01/11] TD-12034 Physical plan code. --- include/libs/planner/planner.h | 1 - source/libs/planner/inc/plannerInt.h | 38 ++++++------ source/libs/planner/src/physicalPlan.c | 80 +++++++++++++++++++++++--- source/libs/planner/src/planner.c | 31 +++------- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 8f217a0deb..be00ed65f0 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -54,7 +54,6 @@ enum OPERATOR_TYPE_E { }; struct SEpSet; -struct SQueryPlanNode; struct SPhyNode; struct SQueryStmtInfo; diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2231c93362..6dcd19782c 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -25,18 +25,20 @@ extern "C" { #include "planner.h" #include "taosmsg.h" -enum LOGIC_PLAN_E { - LP_SCAN = 1, - LP_SESSION = 2, - LP_STATE = 3, - LP_INTERVAL = 4, - LP_FILL = 5, - LP_AGG = 6, - LP_JOIN = 7, - LP_PROJECT = 8, - LP_DISTINCT = 9, - LP_ORDER = 10 -}; +#define QNODE_TAGSCAN 1 +#define QNODE_TABLESCAN 2 +#define QNODE_PROJECT 3 +#define QNODE_AGGREGATE 4 +#define QNODE_GROUPBY 5 +#define QNODE_LIMIT 6 +#define QNODE_JOIN 7 +#define QNODE_DISTINCT 8 +#define QNODE_SORT 9 +#define QNODE_UNION 10 +#define QNODE_TIMEWINDOW 11 +#define QNODE_SESSIONWINDOW 12 +#define QNODE_STATEWINDOW 13 +#define QNODE_FILL 14 typedef struct SQueryNodeBasicInfo { int32_t type; // operator type @@ -64,10 +66,10 @@ typedef struct SQueryPlanNode { SArray *pExpr; // the query functions or sql aggregations int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information - // previous operator to generated result for current node to process + // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. - SArray *pPrevNodes; // upstream nodes - struct SQueryPlanNode *nextNode; + SArray *pChildren; // upstream nodes + struct SQueryPlanNode *pParent; } SQueryPlanNode; typedef SSchema SSlotSchema; @@ -86,11 +88,13 @@ typedef struct SPhyNode { // children plan to generated result for current node to process // in case of join, multiple plan nodes exist. SArray *pChildren; + struct SPhyNode *pParent; } SPhyNode; typedef struct SScanPhyNode { - SPhyNode node; - uint64_t uid; // unique id of the table + SPhyNode node; + STimeWindow window; + uint64_t uid; // unique id of the table } SScanPhyNode; typedef SScanPhyNode STagScanPhyNode; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 2bdc159af8..c4564a9a09 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -15,20 +15,84 @@ #include "plannerInt.h" +// typedef struct SQueryPlanNode { +// void *pExtInfo; // additional information +// SArray *pPrevNodes; // children +// struct SQueryPlanNode *nextNode; // parent +// } SQueryPlanNode; + +// typedef struct SSubplan { +// int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN +// SArray *pDatasource; // the datasource subplan,from which to fetch the result +// struct SPhyNode *pNode; // physical plan of current subplan +// } SSubplan; + +// typedef struct SQueryDag { +// SArray **pSubplans; +// } SQueryDag; + +// typedef struct SScanPhyNode { +// SPhyNode node; +// STimeWindow window; +// uint64_t uid; // unique id of the table +// } SScanPhyNode; + +// typedef SScanPhyNode STagScanPhyNode; + +void fillDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { + dataBlockSchema->index = 0; // todo + SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); + dataBlockSchema->numOfCols = pPlanNode->numOfCols; +} + +void fillPhyNode(SQueryPlanNode* pPlanNode, int32_t type, const char* name, SPhyNode* node) { + node->info.type = type; + node->info.name = name; + SWAP(node->pTargets, pPlanNode->pExpr, SArray*); + fillDataBlockSchema(pPlanNode, &(node->targetSchema)); +} + +SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { + STagScanPhyNode* node = calloc(1, sizeof(STagScanPhyNode)); + fillPhyNode(pPlanNode, OP_TagScan, "TagScan", (SPhyNode*)node); + return (SPhyNode*)node; +} + SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) { - return NULL; + STagScanPhyNode* node = calloc(1, sizeof(STagScanPhyNode)); + fillPhyNode(pPlanNode, OP_TableScan, "SingleTableScan", (SPhyNode*)node); + return (SPhyNode*)node; } -SPhyNode* createPhyNode(SQueryPlanNode* node) { - switch (node->info.type) { - case LP_SCAN: - return createScanNode(node); +SPhyNode* createPhyNode(SQueryPlanNode* pPlanNode) { + SPhyNode* node = NULL; + switch (pPlanNode->info.type) { + case QNODE_TAGSCAN: + node = createTagScanNode(pPlanNode); + break; + case QNODE_TABLESCAN: + node = createScanNode(pPlanNode); + break; + default: + assert(false); } - return NULL; + if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) { + node->pChildren = taosArrayInit(4, POINTER_BYTES); + size_t size = taosArrayGetSize(pPlanNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + SPhyNode* child = createPhyNode(taosArrayGet(pPlanNode->pChildren, i)); + child->pParent = node; + taosArrayPush(node->pChildren, &child); + } + } + return node; } -SPhyNode* createSubplan(SQueryPlanNode* pSubquery) { - return NULL; +SSubplan* createSubplan(SQueryPlanNode* pSubquery) { + SSubplan* subplan = calloc(1, sizeof(SSubplan)); + subplan->pNode = createPhyNode(pSubquery); + // todo + return subplan; } int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e54b847230..b18dd29257 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,21 +18,6 @@ #include "parser.h" #include "plannerInt.h" -#define QNODE_TAGSCAN 1 -#define QNODE_TABLESCAN 2 -#define QNODE_PROJECT 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DISTINCT 8 -#define QNODE_SORT 9 -#define QNODE_UNION 10 -#define QNODE_TIMEWINDOW 11 -#define QNODE_SESSIONWINDOW 12 -#define QNODE_STATEWINDOW 13 -#define QNODE_FILL 14 - typedef struct SFillEssInfo { int32_t fillType; // fill type int64_t *val; // fill value @@ -104,9 +89,9 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla taosArrayPush(pNode->pExpr, &pExpr[i]); } - pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES); + pNode->pChildren = taosArrayInit(4, POINTER_BYTES); for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pPrevNodes, &prev[i]); + taosArrayPush(pNode->pChildren, &prev[i]); } switch(type) { @@ -386,14 +371,14 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { tfree(pQueryNode->info.name); // dropAllExprInfo(pQueryNode->pExpr); - if (pQueryNode->pPrevNodes != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); + if (pQueryNode->pChildren != NULL) { + int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); for(int32_t i = 0; i < size; ++i) { - SQueryPlanNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i); + SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); doDestroyQueryNode(p); } - taosArrayDestroy(pQueryNode->pPrevNodes); + taosArrayDestroy(pQueryNode->pChildren); } tfree(pQueryNode); @@ -607,8 +592,8 @@ int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) { - SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i); + for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { + SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); len = len1; } From 7b6c58fd7ea84b5f6a44cbfb9c7872f5b1d84da8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 17:49:55 +0800 Subject: [PATCH 02/11] remove files --- include/common/tglobal.h | 59 +- include/util/tlog.h | 1 - source/common/src/tglobal.c | 435 ------------ source/dnode/mgmt/impl/test/sut/deploy.cpp | 4 - source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- source/dnode/vnode/impl/CMakeLists.txt | 2 +- source/libs/executor/src/executorimpl.c | 4 +- source/util/src/tlog.c | 4 - src/balance/CMakeLists.txt | 11 - src/balance/inc/bnInt.h | 57 -- src/balance/inc/bnScore.h | 34 - src/balance/inc/bnThread.h | 33 - src/balance/src/bnMain.c | 728 --------------------- src/balance/src/bnScore.c | 321 --------- src/balance/src/bnThread.c | 137 ---- src/inc/http.h | 35 - src/inc/monitor.h | 63 -- src/inc/tacct.h | 37 -- src/inc/tadmin.h | 32 - src/inc/tbn.h | 40 -- src/inc/tgrant.h | 51 -- src/inc/tmqtt.h | 32 - src/inc/tp.h | 31 - src/inc/tsync.h | 127 ---- src/plugins/monitor/CMakeLists.txt | 15 - src/plugins/monitor/src/monMain.c | 436 ------------ src/wal/CMakeLists.txt | 11 - src/wal/test/CMakeLists.txt | 21 - 29 files changed, 6 insertions(+), 2759 deletions(-) delete mode 100644 src/balance/CMakeLists.txt delete mode 100644 src/balance/inc/bnInt.h delete mode 100644 src/balance/inc/bnScore.h delete mode 100644 src/balance/inc/bnThread.h delete mode 100644 src/balance/src/bnMain.c delete mode 100644 src/balance/src/bnScore.c delete mode 100644 src/balance/src/bnThread.c delete mode 100644 src/inc/http.h delete mode 100644 src/inc/monitor.h delete mode 100644 src/inc/tacct.h delete mode 100644 src/inc/tadmin.h delete mode 100644 src/inc/tbn.h delete mode 100644 src/inc/tgrant.h delete mode 100644 src/inc/tmqtt.h delete mode 100644 src/inc/tp.h delete mode 100644 src/inc/tsync.h delete mode 100644 src/plugins/monitor/CMakeLists.txt delete mode 100644 src/plugins/monitor/src/monMain.c delete mode 100644 src/wal/CMakeLists.txt delete mode 100644 src/wal/test/CMakeLists.txt diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a9b0e60761..42956b6bdd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,12 +29,7 @@ extern char tsLocalFqdn[]; extern char tsLocalEp[]; extern uint16_t tsServerPort; extern int32_t tsStatusInterval; -extern int32_t tsNumOfMnodes; -extern int8_t tsEnableVnodeBak; extern int8_t tsEnableTelemetryReporting; -extern char tsArbitrator[]; -extern int8_t tsArbOnline; -extern int64_t tsArbOnlineTimestamp; // common extern int tsRpcTimer; @@ -60,7 +55,6 @@ extern int tsCompatibleModel; // 2.0 compatible model extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked - extern int8_t tsKeepOriginalColumnName; // client @@ -78,66 +72,15 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// db parameters in client -extern int32_t tsCacheBlockSize; -extern int32_t tsBlocksPerVnode; -extern int32_t tsTableIncStepPerVnode; -extern int32_t tsMaxVgroupsPerDb; -extern int16_t tsDaysPerFile; -extern int32_t tsDaysToKeep; -extern int32_t tsMinRowsInFileBlock; -extern int32_t tsMaxRowsInFileBlock; -extern int16_t tsCommitTime; // seconds -extern int32_t tsTimePrecision; -extern int8_t tsCompression; -extern int8_t tsWAL; -extern int32_t tsFsyncPeriod; -extern int32_t tsReplications; -extern int16_t tsPartitons; -extern int32_t tsQuorum; -extern int8_t tsUpdate; -extern int8_t tsCacheLastRow; - -//tsdb -extern bool tsdbForceKeepFile; - // balance -extern int8_t tsEnableBalance; -extern int8_t tsAlternativeRole; -extern int32_t tsBalanceInterval; -extern int32_t tsOfflineThreshold; -extern int8_t tsEnableFlowCtrl; extern int8_t tsEnableSlaveQuery; -extern int8_t tsEnableAdjustMaster; -// restful -extern int32_t tsRestRowLimit; -extern int8_t tsTelegrafUseFieldNum; -// mqtt -extern int8_t tsEnableMqttModule; -extern char tsMqttHostName[]; -extern char tsMqttPort[]; -extern char tsMqttUser[]; -extern char tsMqttPass[]; -extern char tsMqttClientId[]; -extern char tsMqttTopic[]; - -// monitor -extern int8_t tsEnableMonitorModule; -extern char tsMonitorDbName[]; -extern char tsInternalPass[]; -extern int32_t tsMonitorInterval; - -// stream -extern int8_t tsEnableStream; - -// internal +// interna extern int8_t tsPrintAuth; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; -extern int32_t tsTopicBianryLen; // system info extern float tsTotalLogDirGB; diff --git a/include/util/tlog.h b/include/util/tlog.h index 9b6033e7fe..5e6604598d 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -32,7 +32,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t sdbDebugFlag; extern int32_t httpDebugFlag; extern int32_t mqttDebugFlag; extern int32_t monDebugFlag; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index dfbbe33b59..910d05e9a3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -30,19 +30,12 @@ // cluster char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; -char tsArbitrator[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second -int32_t tsNumOfMnodes = 1; -int8_t tsEnableVnodeBak = 1; int8_t tsEnableTelemetryReporting = 0; -int8_t tsArbOnline = 0; -int64_t tsArbOnlineTimestamp = TSDB_ARB_DUMMY_TIME; char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsDnodeId = 0; -int64_t tsDnodeStartTime = 0; // common int32_t tsRpcTimer = 300; @@ -129,59 +122,19 @@ int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name int8_t tsKeepOriginalColumnName = 0; -// db parameters -int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; -int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; -int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; -int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; -int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; -int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; -int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds -int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; -int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; -int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; -int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; -int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; -int32_t tsMaxVgroupsPerDb = 0; -int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; - // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; // balance -int8_t tsEnableBalance = 1; -int8_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; -// restful -int32_t tsRestRowLimit = 10240; -int8_t tsTelegrafUseFieldNum = 0; - -// mqtt -int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default -char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; -char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; -char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; -char tsMqttPass[TSDB_MQTT_PASS_LEN] = {0}; -char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; -char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor -int8_t tsEnableMonitorModule = 1; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; -int32_t tsMonitorInterval = 30; // seconds - -// stream -int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; @@ -191,7 +144,6 @@ char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; int32_t tsDiskCfgNum = 0; -int32_t tsTopicBianryLen = 16000; #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; @@ -245,14 +197,10 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { if (debugFlag != 0) { mDebugFlag = debugFlag; - sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; - httpDebugFlag = debugFlag; - mqttDebugFlag = debugFlag; - monDebugFlag = debugFlag; qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; @@ -459,16 +407,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "arbitrator"; - cfg.ptr = tsArbitrator; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // dnode configs cfg.option = "numOfThreadsPerCore"; cfg.ptr = &tsNumOfThreadsPerCore; @@ -509,26 +447,6 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - - cfg.option = "numOfMnodes"; - cfg.ptr = &tsNumOfMnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vnodeBak"; - cfg.ptr = &tsEnableVnodeBak; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -540,37 +458,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "balance"; - cfg.ptr = &tsEnableBalance; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "balanceInterval"; - cfg.ptr = &tsBalanceInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 30000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // 0-any; 1-mnode; 2-vnode - cfg.option = "role"; - cfg.ptr = &tsAlternativeRole; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 2; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // timer cfg.option = "maxTmrCtrl"; cfg.ptr = &tsMaxTmrCtrl; @@ -582,26 +469,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "monitorInterval"; - cfg.ptr = &tsMonitorInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 600; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "offlineThreshold"; - cfg.ptr = &tsOfflineThreshold; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 3; - cfg.maxValue = 86400 * 365; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -712,186 +579,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "maxVgroupsPerDb"; - cfg.ptr = &tsMaxVgroupsPerDb; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 8192; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cache"; - cfg.ptr = &tsCacheBlockSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; - cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MB; - taosAddConfigOption(cfg); - - cfg.option = "blocks"; - cfg.ptr = &tsBlocksPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; - cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "days"; - cfg.ptr = &tsDaysPerFile; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DAYS_PER_FILE; - cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "keep"; - cfg.ptr = &tsDaysToKeep; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_KEEP; - cfg.maxValue = TSDB_MAX_KEEP; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minRows"; - cfg.ptr = &tsMinRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; - cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxRows"; - cfg.ptr = &tsMaxRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; - cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "precision"; - cfg.ptr = &tsTimePrecision; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_PRECISION; - cfg.maxValue = TSDB_MAX_PRECISION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_COMP_LEVEL; - cfg.maxValue = TSDB_MAX_COMP_LEVEL; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "walLevel"; - cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_WAL_LEVEL; - cfg.maxValue = TSDB_MAX_WAL_LEVEL; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fsync"; - cfg.ptr = &tsFsyncPeriod; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_FSYNC_PERIOD; - cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "replica"; - cfg.ptr = &tsReplications; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION; - cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "quorum"; - cfg.ptr = &tsQuorum; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION; - cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "update"; - cfg.ptr = &tsUpdate; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_UPDATE; - cfg.maxValue = TSDB_MAX_DB_UPDATE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cachelast"; - cfg.ptr = &tsCacheLastRow; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_CACHE_LAST_ROW; - cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttHostName"; - cfg.ptr = tsMqttHostName; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttPort"; - cfg.ptr = tsMqttPort; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_PORT_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttTopic"; - cfg.ptr = tsMqttTopic; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1085,76 +772,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "adjustMaster"; - cfg.ptr = &tsEnableAdjustMaster; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqtt"; - cfg.ptr = &tsEnableMqttModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "monitor"; - cfg.ptr = &tsEnableMonitorModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "stream"; - cfg.ptr = &tsEnableStream; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "topicBianryLen"; - cfg.ptr = &tsTopicBianryLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 16; - cfg.maxValue = 16000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "telegrafUseFieldNum"; - cfg.ptr = &tsTelegrafUseFieldNum; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "restfulRowLimit"; - cfg.ptr = &tsRestRowLimit; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // debug flag cfg.option = "numOfLogLines"; cfg.ptr = &tsNumOfLogLines; @@ -1236,17 +853,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - - cfg.option = "sdbDebugFlag"; - cfg.ptr = &sdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1307,36 +913,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "httpDebugFlag"; - cfg.ptr = &httpDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttDebugFlag"; - cfg.ptr = &mqttDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "monDebugFlag"; - cfg.ptr = &monDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1456,16 +1032,6 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbMetaCompactRatio"; - cfg.ptr = &tsTsdbMetaCompactRatio; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); // enable kill long query @@ -1630,4 +1196,3 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d return true; } - \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index be1506bccf..f2010b5813 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -22,10 +22,6 @@ void initLog(const char* path) { cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; - sdbDebugFlag = 0; - httpDebugFlag = 0; - mqttDebugFlag = 0; - monDebugFlag = 0; uDebugFlag = 143; rpcDebugFlag = 0; odbcDebugFlag = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d06099416b..791b6f5d12 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) { void mndCleanupStb(SMnode *pMnode) {} static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); if (pRaw == NULL) return NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ca244fed9f..5d73f6083a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -71,7 +71,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 6972605afd..3623516624 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - add_subdirectory(test) + #add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0999c44fab..f37c7727f7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4831,8 +4831,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { STableBlockDist tableBlockDist = {0}; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; - if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++numRowSteps; } tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2d7179c0d3..23a923a315 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -81,15 +81,11 @@ int32_t writeInterval = DEFAULT_LOG_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; int32_t mDebugFlag = 131; -int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; -int32_t httpDebugFlag = 131; -int32_t mqttDebugFlag = 131; -int32_t monDebugFlag = 131; int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt deleted file mode 100644 index bffa415deb..0000000000 --- a/src/balance/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -ADD_LIBRARY(balance ${SRC}) diff --git a/src/balance/inc/bnInt.h b/src/balance/inc/bnInt.h deleted file mode 100644 index e924776ff1..0000000000 --- a/src/balance/inc/bnInt.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_INT_H -#define TDENGINE_BALANCE_INT_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "mnodeInt.h" -#include "mnodeDef.h" -#include "mnodeDnode.h" - -typedef struct { - int32_t size; - int32_t maxSize; - SDnodeObj **list; -} SBnDnodes; - -typedef struct { - void * timer; - bool stop; - pthread_mutex_t mutex; - pthread_cond_t cond; - pthread_t thread; -} SBnThread; - -typedef struct { - pthread_mutex_t mutex; -} SBnMgmt; - -int32_t bnInit(); -void bnCleanUp(); -bool bnStart(); -void bnCheckStatus(); -void bnCheckModules(); - -extern SBnDnodes tsBnDnodes; -extern void *tsMnodeTmr; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h deleted file mode 100644 index a28c4459dd..0000000000 --- a/src/balance/inc/bnScore.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_SCORE_H -#define TDENGINE_BALANCE_SCORE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "bnInt.h" - -void bnInitDnodes(); -void bnCleanupDnodes(); -void bnAccquireDnodes(); -void bnReleaseDnodes(); -float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h deleted file mode 100644 index 74a761299d..0000000000 --- a/src/balance/inc/bnThread.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_THREAD_H -#define TDENGINE_BALANCE_THREAD_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "bnInt.h" - -int32_t bnInitThread(); -void bnCleanupThread(); -void bnNotify(); -void bnStartTimer(int32_t mseconds); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c deleted file mode 100644 index 9997d44ca5..0000000000 --- a/src/balance/src/bnMain.c +++ /dev/null @@ -1,728 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tref.h" -#include "tsync.h" -#include "tglobal.h" -#include "dnode.h" -#include "bnInt.h" -#include "bnScore.h" -#include "bnThread.h" -#include "mnodeDb.h" -#include "mnodeMnode.h" -#include "mnodeSdb.h" -#include "mnodeShow.h" -#include "mnodeUser.h" -#include "mnodeVgroup.h" - -extern int64_t tsDnodeRid; -extern int32_t tsSdbRid; -static SBnMgmt tsBnMgmt; -static void bnMonitorDnodeModule(); - -static void bnLock() { - pthread_mutex_lock(&tsBnMgmt.mutex); -} - -static void bnUnLock() { - pthread_mutex_unlock(&tsBnMgmt.mutex); -} - -static bool bnCheckFree(SDnodeObj *pDnode) { - if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]); - return false; - } - - if (pDnode->openVnodes >= TSDB_MAX_VNODES) { - mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES); - return false; - } - - if (pDnode->diskAvailable <= tsMinimalDataDirGB) { - mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable); - return false; - } - - if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) { - mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId); - return false; - } - - return true; -} - -static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { - SVnodeGid tmp = *pVnodeGid1; - *pVnodeGid1 = *pVnodeGid2; - *pVnodeGid2 = tmp; -} - -static void bnAdjustVnodeIndex(SVgObj *pInVg) { - int32_t d0Id = pInVg->vnodeGid[0].dnodeId; - int32_t d1Id = pInVg->vnodeGid[1].dnodeId; - int32_t d2Id = pInVg->vnodeGid[2].dnodeId; - - int32_t vgId = pInVg->vgId; - int32_t d0Num = 0; - int32_t d1Num = 0; - int32_t d2Num = 0; - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - if (pVgroup->vgId != vgId) { - if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++; - if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++; - if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++; - } - - mnodeDecVgroupRef(pVgroup); - } - - if (pInVg->numOfVnodes == 1) { - } - - if (pInVg->numOfVnodes == 2) { - mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num); - if (d0Num > d1Num) { - mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); - } - } - - if (pInVg->numOfVnodes >= 3) { - mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num); - if (d0Num <= d1Num && d0Num <= d2Num) { - if (d1Num > d2Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } else if (d1Num <= d2Num && d1Num <= d0Num) { - mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); - if (d0Num > d2Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } else { - mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]); - if (d1Num > d0Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } - } - - for (int i = 0; i < pInVg->numOfVnodes; ++i) { - mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId); - } -} - -static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { - mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); - - SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId); - if (pDnode != NULL) { - atomic_sub_fetch_32(&pDnode->openVnodes, 1); - mnodeDecDnodeRef(pDnode); - } - - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */ - int32_t numOfVnodes = 0; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i; - if (pTmpVodeGid == pVnodeGid) { - continue; - } - vnodeGid[numOfVnodes] = *pTmpVodeGid; - ++numOfVnodes; - } - memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); - pVgroup->numOfVnodes = numOfVnodes; - - bnAdjustVnodeIndex(pVgroup); - mnodeUpdateVgroup(pVgroup); -} - -int32_t bnAllocVnodes(SVgObj *pVgroup) { - int32_t dnode = 0; - int32_t vnodes = 0; - - bnLock(); - bnAccquireDnodes(); - - mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes, - mnodeGetDnodesNum(), tsBnDnodes.size); - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - for (; dnode < tsBnDnodes.size; ++dnode) { - SDnodeObj *pDnode = tsBnDnodes.list[dnode]; - if (bnCheckFree(pDnode)) { - SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i; - pVnodeGid->dnodeId = pDnode->dnodeId; - pVnodeGid->pDnode = pDnode; - dnode++; - vnodes++; - mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i); - break; - } else { - mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, - dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); - } - } - } - - if (vnodes != pVgroup->numOfVnodes) { - bnReleaseDnodes(); - bnUnLock(); - - mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes); - - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status], - pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); - mnodeDecDnodeRef(pDnode); - } - - if (mnodeGetOnlineDnodesNum() == 0) { - return TSDB_CODE_MND_NOT_READY; - } else { - return TSDB_CODE_MND_NO_ENOUGH_DNODES; - } - } - - bnAdjustVnodeIndex(pVgroup); - bnReleaseDnodes(); - bnUnLock(); - return TSDB_CODE_SUCCESS; -} - -static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { - int32_t rmVnodeVer = 0; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = pVgroup->vnodeGid + i; - if (pVnode == pRmVnode) { - rmVnodeVer = mnodeGetVgidVer(pVnode->vver); - mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i, - pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer); - } - } - - bool isReady = false; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = pVgroup->vnodeGid + i; - SDnodeObj *pDnode = pVnode->pDnode; - if (pVnode == pRmVnode) continue; - int32_t vver = mnodeGetVgidVer(pVnode->vver); - - mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i, - pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); - if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue; - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue; - - if (rmVnodeVer == 0 || vver >= rmVnodeVer) { - mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", - pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); - isReady = true; - } - } - - return isReady; -} - -/** - * desc: remove one vnode from vgroup - * all vnodes in vgroup should in ready state, except the balancing one - **/ -static int32_t bnRemoveVnode(SVgObj *pVgroup) { - if (pVgroup->numOfVnodes <= 1) return -1; - - SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1]; - mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId); - - if (!bnCheckVgroupReady(pVgroup, pSelVnode)) { - mDebug("vgId:%d, is not ready", pVgroup->vgId); - return -1; - } else { - mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); - bnDiscardVnode(pVgroup, pSelVnode); - return TSDB_CODE_SUCCESS; - } -} - -static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pGid = &pVgroup->vnodeGid[i]; - if (pGid->dnodeId == 0) break; - if (pGid->dnodeId == pDnode->dnodeId) { - return true; - } - } - - return false; -} - -static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; - if (!bnCheckFree(pDnode)) continue; - - mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); - return pDnode; - } - - return NULL; -} - -static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { - if (pDestDnode == NULL || pSrcDnode == pDestDnode) { - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - SVnodeGid vnodeGids[TSDB_MAX_REPLICA]; - memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); - - int32_t numOfVnodes = pVgroup->numOfVnodes; - vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId; - vnodeGids[numOfVnodes].pDnode = pDestDnode; - numOfVnodes++; - - // move the src vnode to the end - for (int32_t v = 0; v < numOfVnodes; ++v) { - if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) { - bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]); - pVgroup->lbDnodeId = pSrcDnode->dnodeId; - break; - } - } - - // adjust the vgroup postion - if (pSrcDnode == NULL) { - bnAdjustVnodeIndex(pVgroup); - } - - memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); - pVgroup->numOfVnodes = numOfVnodes; - atomic_add_fetch_32(&pDestDnode->openVnodes, 1); - - mnodeUpdateVgroup(pVgroup); - - return TSDB_CODE_SUCCESS; -} - -static bool bnMonitorBalance() { - if (tsBnDnodes.size < 2) return false; - - mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size); - for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) { - SDnodeObj *pDnode = tsBnDnodes.list[src]; - mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId, - dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes); - } - - float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score; - if (scoresDiff < 0.01) { - mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff); - return false; - } - - for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) { - SDnodeObj *pSrcDnode = tsBnDnodes.list[src]; - float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1); - if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) { - continue; - } - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { - for (int32_t dest = 0; dest < src; dest++) { - SDnodeObj *pDestDnode = tsBnDnodes.list[dest]; - if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue; - if (taosGetTimestampMs() - pDestDnode->createdTime < 2000) continue; - - float destScore = bnTryCalcDnodeScore(pDestDnode, 1); - if (srcScore + 0.0001 < destScore) continue; - if (!bnCheckFree(pDestDnode)) continue; - - mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f", - pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score, - srcScore, pDestDnode->score, destScore); - bnAddVnode(pVgroup, pSrcDnode, pDestDnode); - mnodeDecVgroupRef(pVgroup); - mnodeCancelGetNextVgroup(pIter); - return true; - } - } - - mnodeDecVgroupRef(pVgroup); - } - } - - return false; -} - -// if mgmt changed to master -// 1. reset balanceAccessSquence to zero -// 2. reset state of dnodes to offline -// 3. reset lastAccess of dnodes to zero -void bnReset() { - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - - // while master change, should reset dnode to offline - mInfo("dnode:%d set access:%" PRId64 " to 0", pDnode->dnodeId, pDnode->lastAccess); - pDnode->lastAccess = 0; - if (pDnode->status != TAOS_DN_STATUS_DROPPING) { - pDnode->status = TAOS_DN_STATUS_OFFLINE; - pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; - } - - mnodeDecDnodeRef(pDnode); - } - - tsAccessSquence = 0; -} - -static bool bnMonitorVgroups() { - void * pIter = NULL; - SVgObj *pVgroup = NULL; - bool hasUpdatingVgroup = false; - - while (1) { - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL || pVgroup->pDb == NULL) break; - - int32_t dbReplica = pVgroup->pDb->cfg.replications; - int32_t vgReplica = pVgroup->numOfVnodes; - int32_t code = -1; - - if (vgReplica > dbReplica) { - mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica); - hasUpdatingVgroup = true; - code = bnRemoveVnode(pVgroup); - } else if (vgReplica < dbReplica) { - mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica); - hasUpdatingVgroup = true; - - SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup); - if (pAvailDnode == NULL) { - code = TSDB_CODE_MND_DNODE_NOT_EXIST; - } else { - code = bnAddVnode(pVgroup, NULL, pAvailDnode); - } - } - - mnodeDecVgroupRef(pVgroup); - if (code == TSDB_CODE_SUCCESS) { - mnodeCancelGetNextVgroup(pIter); - break; - } - } - - return hasUpdatingVgroup; -} - -static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) { - mDebug("dnode:%d, in dropping state", pDnode->dnodeId); - - void * pIter = NULL; - bool hasThisDnode = false; - while (1) { - SVgObj *pVgroup = NULL; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup); - mnodeDecVgroupRef(pVgroup); - - if (hasThisDnode) { - mnodeCancelGetNextVgroup(pIter); - break; - } - } - - if (!hasThisDnode) { - mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId); - mnodeDropDnode(pDnode, NULL); - return true; - } - - return false; -} - -static bool bnMontiorDropping() { - void *pIter = NULL; - SDnodeObj *pDnode = NULL; - - while (1) { - mnodeDecDnodeRef(pDnode); - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { - if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue; - if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; - if (mnodeGetDnodesNum() <= 1) continue; - - mLInfo("dnode:%d, set to removing state for it offline:%" PRId64 " seconds", pDnode->dnodeId, - tsAccessSquence - pDnode->lastAccess); - - pDnode->status = TAOS_DN_STATUS_DROPPING; - mnodeUpdateDnode(pDnode); - mnodeDecDnodeRef(pDnode); - mnodeCancelGetNextDnode(pIter); - return true; - } - - if (pDnode->status == TAOS_DN_STATUS_DROPPING) { - bool ret = bnMonitorDnodeDropping(pDnode); - mnodeDecDnodeRef(pDnode); - mnodeCancelGetNextDnode(pIter); - return ret; - } - } - - return false; -} - -bool bnStart() { - if (!sdbIsMaster()) return false; - - bnLock(); - bnAccquireDnodes(); - - bnMonitorDnodeModule(); - - bool updateSoon = bnMontiorDropping(); - - if (!updateSoon) { - updateSoon = bnMonitorVgroups(); - } - - if (!updateSoon) { - updateSoon = bnMonitorBalance(); - } - - bnReleaseDnodes(); - bnUnLock(); - - return updateSoon; -} - -static void bnSetVgroupOffline(SDnodeObj* pDnode) { - void *pIter = NULL; - while (1) { - SVgObj *pVgroup; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - if (pVgroup->vnodeGid[i].pDnode == pDnode) { - pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE; - } - } - mnodeDecVgroupRef(pVgroup); - } -} - -void bnCheckStatus() { - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - - void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid); - if (dnodeSdb == NULL) return; - - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - if (tsAccessSquence - pDnode->lastAccess > 3) { - if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { - pDnode->status = TAOS_DN_STATUS_OFFLINE; - pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; - mInfo("dnode:%d, set to offline state, access seq:%" PRId64 " last seq:%" PRId64 " laststat:%d", pDnode->dnodeId, - tsAccessSquence, pDnode->lastAccess, pDnode->status); - bnSetVgroupOffline(pDnode); - bnStartTimer(3000); - } - } - mnodeDecDnodeRef(pDnode); - } - - taosReleaseRef(tsSdbRid, tsDnodeRid); -} - -void bnCheckModules() { - if (sdbIsMaster()) { - bnLock(); - bnAccquireDnodes(); - bnMonitorDnodeModule(); - bnReleaseDnodes(); - bnUnLock(); - } -} - -int32_t bnInit() { - pthread_mutex_init(&tsBnMgmt.mutex, NULL); - bnInitDnodes(); - bnInitThread(); - bnReset(); - - return 0; -} - -void bnCleanUp() { - bnCleanupThread(); - bnCleanupDnodes(); - pthread_mutex_destroy(&tsBnMgmt.mutex); -} - -int32_t bnDropDnode(SDnodeObj *pDnode) { - int32_t totalFreeVnodes = 0; - void * pIter = NULL; - SDnodeObj *pTempDnode = NULL; - - while (1) { - pIter = mnodeGetNextDnode(pIter, &pTempDnode); - if (pTempDnode == NULL) break; - - if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) { - totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes); - } - - mnodeDecDnodeRef(pTempDnode); - } - - if (pDnode->openVnodes > totalFreeVnodes) { - mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes); - return TSDB_CODE_MND_NO_ENOUGH_DNODES; - } - - pDnode->status = TAOS_DN_STATUS_DROPPING; - mnodeUpdateDnode(pDnode); - - bnStartTimer(1100); - - return TSDB_CODE_SUCCESS; -} - -int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode) { - if (pDnode == NULL) - return 0; - - if (pDnode->isMgmt || pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE - || pDnode->status == TAOS_DN_STATUS_DROPPING - || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - return 0; - } else { - return 1; - } -} - -static void bnMonitorDnodeModule() { - int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes >= tsNumOfMnodes) return; - - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode == NULL) break; - - if (!bnDnodeCanCreateMnode(pDnode)) continue; - - mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes); - mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true); - -#if 0 - // Only create one mnode each time - return; -#else - numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes >= tsNumOfMnodes) return; -#endif - } -} - -int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) { - if (!sdbIsMaster()) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - if (tsEnableBalance != 0) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_BALANCE_ENABLED; - } - - SVgObj *pVgroup = mnodeGetVgroup(vnodeId); - if (pVgroup == NULL) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_VGROUP_NOT_EXIST; - } - - SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId); - if (pDestDnode == NULL) { - mnodeDecVgroupRef(pVgroup); - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - bnLock(); - bnAccquireDnodes(); - - int32_t code = TSDB_CODE_SUCCESS; - if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId, - dnodeId, pSrcDnode->dnodeId); - code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE; - } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId, - dnodeId, dnodeId); - code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE; - } else if (!bnCheckFree(pDestDnode)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId, - dnodeId); - code = TSDB_CODE_MND_DNODE_NOT_FREE; - } else { - code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode); - mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code)); - } - - bnReleaseDnodes(); - bnUnLock(); - - mnodeDecVgroupRef(pVgroup); - mnodeDecDnodeRef(pDestDnode); - - return code; -} diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c deleted file mode 100644 index 04a14357c9..0000000000 --- a/src/balance/src/bnScore.c +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "mnodeShow.h" -#include "mnodeUser.h" -#include "bnScore.h" - -SBnDnodes tsBnDnodes; - -static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn); - -static int32_t bnCalcCpuScore(SDnodeObj *pDnode) { - if (pDnode->cpuAvgUsage < 80) - return 0; - else if (pDnode->cpuAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) { - if (pDnode->memoryAvgUsage < 80) - return 0; - else if (pDnode->memoryAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcDiskScore(SDnodeObj *pDnode) { - if (pDnode->diskAvgUsage < 80) - return 0; - else if (pDnode->diskAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcBandScore(SDnodeObj *pDnode) { - if (pDnode->bandwidthUsage < 30) - return 0; - else if (pDnode->bandwidthUsage < 80) - return 10; - else - return 50; -} - -static float bnCalcModuleScore(SDnodeObj *pDnode) { - if (pDnode->numOfCores <= 0) return 0; - if (pDnode->isMgmt) { - return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores; - } - return 0; -} - -static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) { - if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000; - if (pDnode->numOfCores <= 0) return 0; - return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores; -} - -/** - * calc singe score, such as cpu/memory/disk/bandwitdh/vnode - * 1. get the score config - * 2. if the value is out of range, use border data - * 3. otherwise use interpolation method - **/ -static void bnCalcDnodeScore(SDnodeObj *pDnode) { - pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + - bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) + - pDnode->customScore; -} - -float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) { - int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + - bnCalcBandScore(pDnode); - float moduleScore = bnCalcModuleScore(pDnode); - float vnodeScore = bnCalcVnodeScore(pDnode, extra); - - float score = systemScore + moduleScore + vnodeScore + pDnode->customScore; - return score; -} - -void bnInitDnodes() { - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores); - mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); - - memset(&tsBnDnodes, 0, sizeof(SBnDnodes)); - tsBnDnodes.maxSize = 16; - tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *)); -} - -void bnCleanupDnodes() { - if (tsBnDnodes.list != NULL) { - free(tsBnDnodes.list); - tsBnDnodes.list = NULL; - } -} - -static void bnCheckDnodesSize(int32_t dnodesNum) { - if (tsBnDnodes.maxSize <= dnodesNum) { - int32_t maxSize = dnodesNum * 2; - SDnodeObj** list1 = NULL; - int32_t retry = 0; - - while(list1 == NULL && retry++ < 3) { - list1 = realloc(tsBnDnodes.list, maxSize * sizeof(SDnodeObj *)); - } - if(list1) { - tsBnDnodes.list = list1; - tsBnDnodes.maxSize = maxSize; - } - } -} - -void bnAccquireDnodes() { - int32_t dnodesNum = mnodeGetDnodesNum(); - bnCheckDnodesSize(dnodesNum); - - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - int32_t dnodeIndex = 0; - - while (1) { - if (dnodeIndex >= dnodesNum) { - mnodeCancelGetNextDnode(pIter); - break; - } - - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mnodeDecDnodeRef(pDnode); - continue; - } - - bnCalcDnodeScore(pDnode); - - int32_t orderIndex = dnodeIndex; - for (; orderIndex > 0; --orderIndex) { - if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) { - break; - } - tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1]; - } - tsBnDnodes.list[orderIndex] = pDnode; - dnodeIndex++; - } - - tsBnDnodes.size = dnodeIndex; -} - -void bnReleaseDnodes() { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode != NULL) { - mnodeDecDnodeRef(pDnode); - } - } -} - -static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SUserObj *pUser = mnodeGetUserFromConn(pConn); - if (pUser == NULL) return 0; - - if (strcmp(pUser->pAcct->user, "root") != 0) { - mnodeDecUserRef(pUser); - return TSDB_CODE_MND_NO_RIGHTS; - } - - int32_t cols = 0; - SSchema *pSchema = pMeta->schema; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "system scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "custom scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "module scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "vnode scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "total scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "open vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cpu cores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "balance state"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = mnodeGetDnodesNum(); - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pIter = NULL; - - mnodeDecUserRef(pUser); - - return 0; -} - -static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - int32_t numOfRows = 0; - SDnodeObj *pDnode = NULL; - char * pWrite; - int32_t cols = 0; - - while (numOfRows < rows) { - pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode); - if (pDnode == NULL) break; - - int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode); - float moduleScore = bnCalcModuleScore(pDnode); - float vnodeScore = bnCalcVnodeScore(pDnode, 0); - - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->dnodeId; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)systemScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)pDnode->customScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)moduleScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)vnodeScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->openVnodes; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->numOfCores; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]); - cols++; - - numOfRows++; - mnodeDecDnodeRef(pDnode); - } - - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); - pShow->numOfReads += numOfRows; - return numOfRows; -} diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c deleted file mode 100644 index 20da83ccba..0000000000 --- a/src/balance/src/bnThread.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ttimer.h" -#include "tglobal.h" -#include "mnodeSdb.h" -#include "bnThread.h" - -static SBnThread tsBnThread; - -static void *bnThreadFunc(void *arg) { - setThreadName("balance"); - - while (1) { - pthread_mutex_lock(&tsBnThread.mutex); - if (tsBnThread.stop) { - pthread_mutex_unlock(&tsBnThread.mutex); - break; - } - - pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); - mDebug("balance thread wakes up to work"); - bool updateSoon = bnStart(); - mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); - - bnStartTimer(updateSoon ? 1000 : -1); - pthread_mutex_unlock(&(tsBnThread.mutex)); - } - - mDebug("balance thread is stopped"); - return NULL; -} - -int32_t bnInitThread() { - memset(&tsBnThread, 0, sizeof(SBnThread)); - tsBnThread.stop = false; - pthread_mutex_init(&tsBnThread.mutex, NULL); - pthread_cond_init(&tsBnThread.cond, NULL); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL); - pthread_attr_destroy(&thattr); - - if (ret != 0) { - mError("failed to create balance thread since %s", strerror(ret)); - return -1; - } - - bnStartTimer(2000); - mDebug("balance thread is created"); - return 0; -} - -void bnCleanupThread() { - mDebug("balance thread will be cleanup"); - - if (tsBnThread.timer != NULL) { - taosTmrStopA(&tsBnThread.timer); - tsBnThread.timer = NULL; - mDebug("stop balance timer"); - } - - pthread_mutex_lock(&tsBnThread.mutex); - tsBnThread.stop = true; - pthread_cond_signal(&tsBnThread.cond); - pthread_mutex_unlock(&(tsBnThread.mutex)); - pthread_join(tsBnThread.thread, NULL); - - pthread_cond_destroy(&tsBnThread.cond); - pthread_mutex_destroy(&tsBnThread.mutex); -} - -static void bnPostSignal() { - if (tsBnThread.stop) return; - - pthread_mutex_lock(&tsBnThread.mutex); - pthread_cond_signal(&tsBnThread.cond); - pthread_mutex_unlock(&(tsBnThread.mutex)); -} - -/* - * once sdb work as mater, then tsAccessSquence reset to zero - * increase tsAccessSquence every balance interval - */ - -static void bnProcessTimer(void *handle, void *tmrId) { - if (!sdbIsMaster()) return; - if (tsBnThread.stop) return; - - tsBnThread.timer = NULL; - bnStartTimer(-1); - bnCheckStatus(); - - if (handle == NULL) { - ++tsAccessSquence; - - if (tsAccessSquence % tsBalanceInterval == 0) { - mDebug("balance function is scheduled by timer"); - bnPostSignal(); - } - } else { - int64_t mseconds = (int64_t)handle; - mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds); - bnPostSignal(); - } -} - -void bnStartTimer(int32_t mseconds) { - if (tsBnThread.stop) return; - - if (mseconds != -1) { - mTrace("balance function will be called after %d ms", mseconds); - taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); - } else { - taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); - } -} - -void bnNotify() { - bnStartTimer(500); -} diff --git a/src/inc/http.h b/src/inc/http.h deleted file mode 100644 index 0d4c386cbf..0000000000 --- a/src/inc/http.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_H -#define TDENGINE_HTTP_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -int32_t httpGetReqCount(); -int32_t httpInitSystem(); -int32_t httpStartSystem(); -void httpStopSystem(); -void httpCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/monitor.h b/src/inc/monitor.h deleted file mode 100644 index d2e5e06487..0000000000 --- a/src/inc/monitor.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MONITOR_H -#define TDENGINE_MONITOR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -typedef struct { - char * acctId; - int64_t currentPointsPerSecond; - int64_t maxPointsPerSecond; - int64_t totalTimeSeries; - int64_t maxTimeSeries; - int64_t totalStorage; - int64_t maxStorage; - int64_t totalQueryTime; - int64_t maxQueryTime; - int64_t totalInbound; - int64_t maxInbound; - int64_t totalOutbound; - int64_t maxOutbound; - int64_t totalDbs; - int64_t maxDbs; - int64_t totalUsers; - int64_t maxUsers; - int64_t totalStreams; - int64_t maxStreams; - int64_t totalConns; - int64_t maxConns; - int8_t accessState; -} SAcctMonitorObj; - -int32_t monInitSystem(); -int32_t monStartSystem(); -void monStopSystem(); -void monCleanupSystem(); -void monSaveAcctLog(SAcctMonitorObj *pMonObj); -void monSaveLog(int32_t level, const char *const format, ...); -void monExecuteSQL(char *sql); -typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); -void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tacct.h b/src/inc/tacct.h deleted file mode 100644 index 52215fac52..0000000000 --- a/src/inc/tacct.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_ACCT_H -#define TDENGINE_ACCT_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - ACCT_GRANT_USER, - ACCT_GRANT_DB, - ACCT_GRANT_TABLE -} EAcctGrantType; - -int32_t acctInit(); -void acctCleanUp(); -int32_t acctCheck(void *pAcct, EAcctGrantType type); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tadmin.h b/src/inc/tadmin.h deleted file mode 100644 index b7de33576a..0000000000 --- a/src/inc/tadmin.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_ADMIN_H -#define TDENGINE_ADMIN_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct HttpServer; - -void adminInitHandle(struct HttpServer* pServer); -void opInitHandle(struct HttpServer* pServer); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tbn.h b/src/inc/tbn.h deleted file mode 100644 index b35f90eb15..0000000000 --- a/src/inc/tbn.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_H -#define TDENGINE_BALANCE_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct SVgObj; -struct SDnodeObj; - -int32_t bnInit(); -void bnCleanUp(); -void bnNotify(); -void bnCheckModules(); -void bnReset(); -int32_t bnAllocVnodes(struct SVgObj *pVgroup); -int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); -int32_t bnDropDnode(struct SDnodeObj *pDnode); -int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tgrant.h b/src/inc/tgrant.h deleted file mode 100644 index f62a521b6c..0000000000 --- a/src/inc/tgrant.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_GTANT_H -#define TDENGINE_GTANT_H - -#ifdef __cplusplus -"C" { -#endif - -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantInit(); -void grantCleanUp(); -void grantParseParameter(); -int32_t grantCheck(EGrantType grant); -void grantReset(EGrantType grant, uint64_t value); -void grantAdd(EGrantType grant, uint64_t value); -void grantRestore(EGrantType grant, uint64_t value); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tmqtt.h b/src/inc/tmqtt.h deleted file mode 100644 index 256e61fbae..0000000000 --- a/src/inc/tmqtt.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MQTT_H -#define TDENGINE_MQTT_H - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mqttInitSystem(); -int32_t mqttStartSystem(); -void mqttStopSystem(); -void mqttCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tp.h b/src/inc/tp.h deleted file mode 100644 index 1d6570898e..0000000000 --- a/src/inc/tp.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TP -#define TDENGINE_TP - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t tpInit(); -void tpCleanUp(); -void tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tsync.h b/src/inc/tsync.h deleted file mode 100644 index d1b68e3f5a..0000000000 --- a/src/inc/tsync.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_H -#define TDENGINE_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TAOS_SYNC_MAX_REPLICA 5 -#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF - -typedef enum { - TAOS_SYNC_ROLE_OFFLINE = 0, - TAOS_SYNC_ROLE_UNSYNCED = 1, - TAOS_SYNC_ROLE_SYNCING = 2, - TAOS_SYNC_ROLE_SLAVE = 3, - TAOS_SYNC_ROLE_MASTER = 4 -} ESyncRole; - -typedef enum { - TAOS_SYNC_STATUS_INIT = 0, - TAOS_SYNC_STATUS_START = 1, - TAOS_SYNC_STATUS_FILE = 2, - TAOS_SYNC_STATUS_CACHE = 3 -} ESyncStatus; - -typedef struct { - uint32_t nodeId; // node ID assigned by TDengine - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN -} SNodeInfo; - -typedef struct { - int8_t quorum; // number of confirms required, >=1 - int8_t replica; // number of replications, >=1 - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; -} SSyncCfg; - -typedef struct { - int32_t selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int32_t role[TAOS_SYNC_MAX_REPLICA]; -} SNodesRole; - -// get the wal file from index or after -// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); - -// when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -// when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); - -// when role is changed, call this to notify app -typedef void (*FNotifyRole)(int32_t vgId, int8_t role); - -// if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); - -// when data file is synced successfully, notity app -typedef void (*FStartSyncFile)(int32_t vgId); -typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); - -// get file version -typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); - -typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); -typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); - -typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[TSDB_FILENAME_LEN]; // path to the file - void * pTsdb; - FGetWalInfo getWalInfoFp; - FWriteToCache writeToCacheFp; - FConfirmForward confirmForward; - FNotifyRole notifyRoleFp; - FNotifyFlowCtrl notifyFlowCtrlFp; - FStartSyncFile startSyncFileFp; - FStopSyncFile stopSyncFileFp; - FGetVersion getVersionFp; - FSendFile sendFileFp; - FRecvFile recvFileFp; -} SSyncInfo; - -typedef void *tsync_h; - -int32_t syncInit(); -void syncCleanUp(); - -int64_t syncStart(const SSyncInfo *); -void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); -void syncRecover(int64_t rid); // recover from other nodes: -int32_t syncGetNodesRole(int64_t rid, SNodesRole *); - -extern char *syncRole[]; - -//global configurable parameters -extern int32_t sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SYNC_H diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt deleted file mode 100644 index 8a05d63e14..0000000000 --- a/src/plugins/monitor/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -AUX_SOURCE_DIRECTORY(./src SRC) - -ADD_LIBRARY(monitor ${SRC}) - -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) -ELSE () - TARGET_LINK_LIBRARIES(monitor taos) -ENDIF () diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c deleted file mode 100644 index fea793fa86..0000000000 --- a/src/plugins/monitor/src/monMain.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tlog.h" -#include "ttimer.h" -#include "tutil.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "dnode.h" -#include "monitor.h" -#include "taoserror.h" - -#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} -#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} -#define monWarn(...) { if (monDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} -#define monInfo(...) { if (monDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} -#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} -#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} - -#define SQL_LENGTH 1030 -#define LOG_LEN_STR 512 -#define IP_LEN_STR TSDB_EP_LEN -#define CHECK_INTERVAL 1000 - -typedef enum { - MON_CMD_CREATE_DB, - MON_CMD_CREATE_TB_LOG, - MON_CMD_CREATE_MT_DN, - MON_CMD_CREATE_MT_ACCT, - MON_CMD_CREATE_TB_DN, - MON_CMD_CREATE_TB_ACCT_ROOT, - MON_CMD_CREATE_TB_SLOWQUERY, - MON_CMD_MAX -} EMonCmd; - -typedef enum { - MON_STATE_NOT_INIT, - MON_STATE_INITED -} EMonState; - -typedef struct { - pthread_t thread; - void * conn; - char ep[TSDB_EP_LEN]; - int8_t cmdIndex; - int8_t state; - int8_t start; // enable/disable by mnode - int8_t quiting; // taosd is quiting - char sql[SQL_LENGTH + 1]; -} SMonConn; - -static SMonConn tsMonitor = {0}; -static void monSaveSystemInfo(); -static void *monThreadFunc(void *param); -static void monBuildMonitorSql(char *sql, int32_t cmd); -extern int32_t (*monStartSystemFp)(); -extern void (*monStopSystemFp)(); -extern void (*monExecuteSQLFp)(char *sql); - -int32_t monInitSystem() { - if (tsMonitor.ep[0] == 0) { - strcpy(tsMonitor.ep, tsLocalEp); - } - - int32_t len = (int32_t)strlen(tsMonitor.ep); - for (int32_t i = 0; i < len; ++i) { - if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { - tsMonitor.ep[i] = '_'; - } - } - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) { - monError("failed to create thread to for monitor module, reason:%s", strerror(errno)); - return -1; - } - - pthread_attr_destroy(&thAttr); - monDebug("monitor thread is launched"); - - monStartSystemFp = monStartSystem; - monStopSystemFp = monStopSystem; - return 0; -} - -int32_t monStartSystem() { - if (taos_init()) { - return -1; - } - tsMonitor.start = 1; - monExecuteSQLFp = monExecuteSQL; - monInfo("monitor module start"); - return 0; -} - -static void *monThreadFunc(void *param) { - monDebug("starting to initialize monitor module ..."); - setThreadName("monitor"); - - while (1) { - static int32_t accessTimes = 0; - accessTimes++; - taosMsleep(1000); - - if (tsMonitor.quiting) { - tsMonitor.state = MON_STATE_NOT_INIT; - monInfo("monitor thread will quit, for taosd is quiting"); - break; - } else { - taosGetDisk(); - } - - if (tsMonitor.start == 0) { - continue; - } - - if (dnodeGetDnodeId() <= 0) { - monDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); - continue; - } - - if (tsMonitor.conn == NULL) { - tsMonitor.state = MON_STATE_NOT_INIT; - tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0); - if (tsMonitor.conn == NULL) { - monError("failed to connect to database, reason:%s", tstrerror(terrno)); - continue; - } else { - monDebug("connect to database success"); - } - } - - if (tsMonitor.state == MON_STATE_NOT_INIT) { - int32_t code = 0; - - for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { - monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); - void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - code = taos_errno(res); - taos_free_result(res); - - if (code != 0) { - monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); - break; - } else { - monDebug("successfully to exec sql:%s", tsMonitor.sql); - } - } - - if (tsMonitor.start && code == 0) { - tsMonitor.state = MON_STATE_INITED; - } - } - - if (tsMonitor.state == MON_STATE_INITED) { - if (accessTimes % tsMonitorInterval == 0) { - monSaveSystemInfo(); - } - } - } - - monInfo("monitor thread is stopped"); - return NULL; -} - -static void monBuildMonitorSql(char *sql, int32_t cmd) { - memset(sql, 0, SQL_LENGTH); - -#ifdef _STORAGE - char *keepValue = "30,30,30"; -#else - char *keepValue = "30"; -#endif - - if (cmd == MON_CMD_CREATE_DB) { - snprintf(sql, SQL_LENGTH, - "create database if not exists %s replica 1 days 10 keep %s cache %d " - "blocks %d precision 'us'", - tsMonitorDbName, keepValue, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MIN_TOTAL_BLOCKS); - } else if (cmd == MON_CMD_CREATE_MT_DN) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.dn(ts timestamp" - ", cpu_taosd float, cpu_system float, cpu_cores int" - ", mem_taosd float, mem_system float, mem_total int" - ", disk_used float, disk_total int" - ", band_speed float" - ", io_read float, io_write float" - ", req_http int, req_select int, req_insert int" - ") tags (dnodeid int, fqdn binary(%d))", - tsMonitorDbName, TSDB_FQDN_LEN); - } else if (cmd == MON_CMD_CREATE_TB_DN) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName, - dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp); - } else if (cmd == MON_CMD_CREATE_MT_ACCT) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.acct(ts timestamp " - ", currentPointsPerSecond bigint, maxPointsPerSecond bigint" - ", totalTimeSeries bigint, maxTimeSeries bigint" - ", totalStorage bigint, maxStorage bigint" - ", totalQueryTime bigint, maxQueryTime bigint" - ", totalInbound bigint, maxInbound bigint" - ", totalOutbound bigint, maxOutbound bigint" - ", totalDbs smallint, maxDbs smallint" - ", totalUsers smallint, maxUsers smallint" - ", totalStreams smallint, maxStreams smallint" - ", totalConns smallint, maxConns smallint" - ", accessState smallint" - ") tags (acctId binary(%d))", - tsMonitorDbName, TSDB_USER_LEN); - } else if (cmd == MON_CMD_CREATE_TB_ACCT_ROOT) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, TSDB_DEFAULT_USER, - tsMonitorDbName, TSDB_DEFAULT_USER); - } else if (cmd == MON_CMD_CREATE_TB_SLOWQUERY) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.slowquery(ts timestamp, username " - "binary(%d), created_time timestamp, time bigint, sql binary(%d))", - tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); - } else if (cmd == MON_CMD_CREATE_TB_LOG) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.log(ts timestamp, level tinyint, " - "content binary(%d), ipaddr binary(%d))", - tsMonitorDbName, LOG_LEN_STR, IP_LEN_STR); - } - - sql[SQL_LENGTH] = 0; -} - -void monStopSystem() { - tsMonitor.start = 0; - tsMonitor.state = MON_STATE_NOT_INIT; - monExecuteSQLFp = NULL; - monInfo("monitor module stopped"); -} - -void monCleanupSystem() { - tsMonitor.quiting = 1; - monStopSystem(); - if (taosCheckPthreadValid(tsMonitor.thread)) { - pthread_join(tsMonitor.thread, NULL); - } - - if (tsMonitor.conn != NULL) { - taos_close(tsMonitor.conn); - tsMonitor.conn = NULL; - } - monInfo("monitor module is cleaned up"); -} - -// unit is MB -static int32_t monBuildMemorySql(char *sql) { - float sysMemoryUsedMB = 0; - bool suc = taosGetSysMemory(&sysMemoryUsedMB); - if (!suc) { - monDebug("failed to get sys memory info"); - } - - float procMemoryUsedMB = 0; - suc = taosGetProcMemory(&procMemoryUsedMB); - if (!suc) { - monDebug("failed to get proc memory info"); - } - - return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); -} - -// unit is % -static int32_t monBuildCpuSql(char *sql) { - float sysCpuUsage = 0, procCpuUsage = 0; - bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); - if (!suc) { - monDebug("failed to get cpu usage"); - } - - if (sysCpuUsage <= procCpuUsage) { - sysCpuUsage = procCpuUsage + 0.1f; - } - - return sprintf(sql, ", %f, %f, %d", procCpuUsage, sysCpuUsage, tsNumOfCores); -} - -// unit is GB -static int32_t monBuildDiskSql(char *sql) { - return sprintf(sql, ", %f, %d", tsUsedDataDirGB, (int32_t)tsTotalDataDirGB); -} - -// unit is Kb -static int32_t monBuildBandSql(char *sql) { - float bandSpeedKb = 0; - bool suc = taosGetBandSpeed(&bandSpeedKb); - if (!suc) { - monDebug("failed to get bandwidth speed"); - } - - return sprintf(sql, ", %f", bandSpeedKb); -} - -static int32_t monBuildReqSql(char *sql) { - SStatisInfo info = dnodeGetStatisInfo(); - return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); -} - -static int32_t monBuildIoSql(char *sql) { - float readKB = 0, writeKB = 0; - bool suc = taosGetProcIO(&readKB, &writeKB); - if (!suc) { - monDebug("failed to get io info"); - } - - return sprintf(sql, ", %f, %f", readKB, writeKB); -} - -static void monSaveSystemInfo() { - int64_t ts = taosGetTimestampUs(); - char * sql = tsMonitor.sql; - int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts); - - pos += monBuildCpuSql(sql + pos); - pos += monBuildMemorySql(sql + pos); - pos += monBuildDiskSql(sql + pos); - pos += monBuildBandSql(sql + pos); - pos += monBuildIoSql(sql + pos); - pos += monBuildReqSql(sql + pos); - - void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int32_t code = taos_errno(res); - taos_free_result(res); - - if (code != 0) { - monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); - } else { - monDebug("successfully to save system info, sql:%s", tsMonitor.sql); - } -} - -static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) { - int32_t c = taos_errno(result); - if (c != TSDB_CODE_SUCCESS) { - monError("save %s failed, reason:%s", (char *)param, tstrerror(c)); - } else { - int32_t rows = taos_affected_rows(result); - monDebug("save %s succ, rows:%d", (char *)param, rows); - } - - taos_free_result(result); -} - -void monSaveAcctLog(SAcctMonitorObj *pMon) { - if (tsMonitor.state != MON_STATE_INITED) return; - - char sql[1024] = {0}; - sprintf(sql, - "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %d)", - tsMonitorDbName, pMon->acctId, tsMonitorDbName, pMon->acctId, - pMon->currentPointsPerSecond, pMon->maxPointsPerSecond, - pMon->totalTimeSeries, pMon->maxTimeSeries, - pMon->totalStorage, pMon->maxStorage, - pMon->totalQueryTime, pMon->maxQueryTime, - pMon->totalInbound, pMon->maxInbound, - pMon->totalOutbound, pMon->maxOutbound, - pMon->totalDbs, pMon->maxDbs, - pMon->totalUsers, pMon->maxUsers, - pMon->totalStreams, pMon->maxStreams, - pMon->totalConns, pMon->maxConns, - pMon->accessState); - - monDebug("save account info, sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info"); -} - -void monSaveLog(int32_t level, const char *const format, ...) { - if (tsMonitor.state != MON_STATE_INITED) return; - - va_list argpointer; - char sql[SQL_LENGTH] = {0}; - int32_t max_length = SQL_LENGTH - 30; - int32_t len = snprintf(sql, (size_t)max_length, "insert into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, - taosGetTimestampUs(), level); - - va_start(argpointer, format); - len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer); - va_end(argpointer); - if (len > max_length) len = max_length; - - len += sprintf(sql + len, "', '%s')", tsLocalEp); - sql[len++] = 0; - - monDebug("save log, sql: %s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log"); -} - -void monExecuteSQL(char *sql) { - if (tsMonitor.state != MON_STATE_INITED) return; - - monDebug("execute sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); -} - -void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { - if (tsMonitor.conn == NULL) { - callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); - return; - } - - monDebug("execute sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, callback, param); -} diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt deleted file mode 100644 index 0d9be42bd5..0000000000 --- a/src/wal/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) - -ADD_LIBRARY(twal ${SRC}) -TARGET_LINK_LIBRARIES(twal tutil common) -ADD_SUBDIRECTORY(test) diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt deleted file mode 100644 index c5bc4198f1..0000000000 --- a/src/wal/test/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(../inc) - - LIST(APPEND WALTEST_SRC ./waltest.c) - ADD_EXECUTABLE(waltest ${WALTEST_SRC}) - TARGET_LINK_LIBRARIES(waltest twal os tutil) - -ENDIF () - -IF (TD_DARWIN) - INCLUDE_DIRECTORIES(../inc) - - LIST(APPEND WALTEST_SRC ./waltest.c) - ADD_EXECUTABLE(waltest ${WALTEST_SRC}) - TARGET_LINK_LIBRARIES(waltest twal os tutil) - -ENDIF () - From c971df227c2ff363d22c07929155f957f1c45f6d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 18:02:16 +0800 Subject: [PATCH 03/11] TD-10431 add version to db --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 3 +++ source/dnode/mnode/impl/src/mndVgroup.c | 3 --- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 30c18d3e18..115d896da2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -214,6 +214,7 @@ typedef struct { int64_t createdTime; int64_t updateTime; int64_t uid; + int32_t version; SDbCfg cfg; } SDbObj; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1cf9423da8..9f2b2d17f3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -74,6 +74,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) SDB_SET_INT64(pRaw, dataPos, pDb->uid) + SDB_SET_INT32(pRaw, dataPos, pDb->version) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) @@ -117,6 +118,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) @@ -571,6 +573,7 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return code; } + dbObj.version++; code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); mndReleaseDb(pMnode, pDb); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 5d73f6083a..9a462c024b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -141,11 +141,8 @@ static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { mTrace("vgId:%d, perform update action", pOldVgroup->vgId); - pOldVgroup->vgId = pNewVgroup->vgId; - pOldVgroup->createdTime = pNewVgroup->createdTime; pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; - memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN); pOldVgroup->replica = pNewVgroup->replica; memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; From c7a7939f6896d04417b57339aae6fb2d56fa8988 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Dec 2021 19:55:50 +0800 Subject: [PATCH 04/11] add tablemeta cache --- include/common/taosmsg.h | 3 +- include/libs/catalog/catalog.h | 56 +--- include/libs/query/query.h | 46 +++ source/libs/catalog/inc/catalogInt.h | 19 +- source/libs/catalog/src/catalog.c | 421 ++++++++++++++++++++++----- source/libs/parser/src/parserUtil.c | 23 -- source/libs/query/src/querymsg.c | 130 ++++++++- 7 files changed, 557 insertions(+), 141 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index caf872689c..fd6c0620e8 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -839,7 +839,7 @@ typedef struct { int32_t tversion; uint64_t tuid; uint64_t suid; - SVgroupMsg vgroup; + int32_t vgId; SSchema pSchema[]; } STableMetaMsg; @@ -867,6 +867,7 @@ typedef struct { int32_t dbVgroupVersion; int32_t dbVgroupNum; int32_t dbHashRange; + int32_t dbHashType; SVgroupInfo vgroupInfo[]; //int32_t vgIdList[]; } SUseDbRspMsg; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f9d3b3c8c1..38965c6ba9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -45,42 +45,10 @@ typedef struct SMetaData { SEpSet *pEpSet; // qnode epset list } SMetaData; -typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema -} STableComInfo; - -/* - * ASSERT(sizeof(SCTableMeta) == 24) - * ASSERT(tableType == TSDB_CHILD_TABLE) - * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. - */ -typedef struct SCTableMeta { - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; -} SCTableMeta; - -/* - * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. - */ -typedef struct STableMeta { - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; - // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; -} STableMeta; - typedef struct SCatalogCfg { - + bool enableVgroupCache; + uint32_t maxTblCacheNum; + uint32_t maxDBCacheNum; } SCatalogCfg; int32_t catalogInit(SCatalogCfg *cfg); @@ -96,19 +64,25 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList); -int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); + +/** + * get cluster vgroup list. + * @pVgroupList - hash of vgroup list, key:vgId, value:SVgroupInfo + * @return + */ +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash); +int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta); -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta* pTableMeta); /** diff --git a/include/libs/query/query.h b/include/libs/query/query.h index 02ae708874..bfe2db6a61 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -24,10 +24,49 @@ extern "C" { typedef SVgroupListRspMsg SVgroupListInfo; +typedef struct STableComInfo { + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + int16_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema +} STableComInfo; + +/* + * ASSERT(sizeof(SCTableMeta) == 24) + * ASSERT(tableType == TSDB_CHILD_TABLE) + * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. + */ +typedef struct SCTableMeta { + int32_t vgId:24; + int8_t tableType; + uint64_t uid; + uint64_t suid; +} SCTableMeta; + +/* + * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. + */ +typedef struct STableMeta { + //BEGIN: KEEP THIS PART SAME WITH SCTableMeta + int32_t vgId:24; + int8_t tableType; + uint64_t uid; + uint64_t suid; + //END: KEEP THIS PART SAME WITH SCTableMeta + + // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info + int16_t sversion; + int16_t tversion; + STableComInfo tableInfo; + SSchema schema[]; +} STableMeta; + + typedef struct SDBVgroupInfo { int32_t vgroupVersion; SArray *vgId; int32_t hashRange; + int32_t hashType; } SDBVgroupInfo; typedef struct SUseDbOutput { @@ -36,6 +75,13 @@ typedef struct SUseDbOutput { SDBVgroupInfo *dbVgroup; } SUseDbOutput; +typedef struct STableMetaOutput { + int32_t metaNum; + char ctbFname[TSDB_TABLE_FNAME_LEN]; + char tbFname[TSDB_TABLE_FNAME_LEN]; + SCTableMeta ctbMeta; + STableMeta *tbMeta; +} STableMetaOutput; extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a08b64f9a9..455c82b1bc 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -24,16 +24,16 @@ extern "C" { #include "common.h" #include "tlog.h" -#define CTG_DEFAULT_CLUSTER_NUMBER 6 -#define CTG_DEFAULT_VGROUP_NUMBER 100 -#define CTG_DEFAULT_DB_NUMBER 20 +#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 +#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 +#define CTG_DEFAULT_CACHE_DB_NUMBER 20 +#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000 #define CTG_DEFAULT_INVALID_VERSION (-1) typedef struct SVgroupListCache { int32_t vgroupVersion; - SHashObj *cache; // key:vgId, value:SVgroupInfo* - SArray *arrayCache; // SVgroupInfo + SHashObj *cache; // key:vgId, value:SVgroupInfo } SVgroupListCache; typedef struct SDBVgroupCache { @@ -41,20 +41,23 @@ typedef struct SDBVgroupCache { } SDBVgroupCache; typedef struct STableMetaCache { - SHashObj *cache; //key:fulltablename, value:STableMeta + SHashObj *cache; //key:fulltablename, value:STableMeta + SHashObj *stableCache; //key:suid, value:STableMeta* } STableMetaCache; typedef struct SCatalog { SVgroupListCache vgroupCache; - SDBVgroupCache dbCache; - STableMetaCache tableCache; + SDBVgroupCache dbCache; + STableMetaCache tableCache; } SCatalog; typedef struct SCatalogMgmt { void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node + SCatalogCfg cfg; } SCatalogMgmt; +typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); extern int32_t ctgDebugFlag; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 92b6094529..9921a2696b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -47,14 +47,14 @@ int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSe return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) { - if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) { +int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { + if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) { *exist = 0; return TSDB_CODE_SUCCESS; } if (pVgroupList) { - *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + *pVgroupList = pCatalog->vgroupCache.cache; } *exist = 1; @@ -62,7 +62,6 @@ int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* return TSDB_CODE_SUCCESS; } - int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; @@ -92,6 +91,7 @@ int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgrou (*dbInfo)->vgroupVersion = info->vgroupVersion; (*dbInfo)->hashRange = info->hashRange; + (*dbInfo)->hashType = info->hashType; } *exist = 1; @@ -130,17 +130,328 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp } -int32_t catalogInit(SCatalogCfg *cfg) { - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == ctgMgmt.pCluster) { - CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); +int32_t ctgGetTableMetaFromCache(SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) { + if (NULL == pCatalog->tableCache.cache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName); + + STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); + + if (NULL == tbMeta) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if (tbMeta->tableType == TSDB_CHILD_TABLE) { + STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != tbMeta->suid) { + ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + *pTableMeta = calloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgError("calloc size[%d] failed", metaSize); + return TSDB_CODE_CTG_MEM_ERROR; + } + + memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + } else { + int32_t metaSize = sizeof(STableMeta) + (tbMeta->tableInfo.numOfTags + tbMeta->tableInfo.numOfColumns) * sizeof(SSchema); + *pTableMeta = calloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgError("calloc size[%d] failed", metaSize); + return TSDB_CODE_CTG_MEM_ERROR; + } + + memcpy(*pTableMeta, tbMeta, metaSize); + } + + *exist = 1; + + return TSDB_CODE_SUCCESS; +} + +void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { + epSet->inUse = 0; + epSet->numOfEps = vgroupInfo->numOfEps; + + for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) { + memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i])); + memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i])); + } +} + + +int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_TABLE_META, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + SEpSet epSet; + + ctgGenEpSet(&epSet, vgroupInfo); + + rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); + + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code); + return rpcRsp.code; + } + + code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; } return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { +int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { + switch (hashType) { + default: + *fp = MurmurHash3_32; + break; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetTableHashVgroup(SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + SDBVgroupInfo *dbInfo = NULL; + int32_t code = 0; + + CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); + + if (NULL == dbInfo) { + ctgWarn("db[%s] vgroup info not found", pDBName); + return TSDB_CODE_TSC_DB_NOT_SELECTED; + } + + if (dbInfo->vgroupVersion < 0 || NULL == dbInfo->vgId) { + ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgId:%p", pDBName, dbInfo->vgroupVersion, dbInfo->vgId); + CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + int32_t vgNum = taosArrayGetSize(dbInfo->vgId); + if (vgNum <= 0) { + ctgError("db[%s] vgroup cache invalid, vgroup number:%p", vgNum); + CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + + CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp)); + + char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); + uint32_t hashUnit = dbInfo->hashRange / vgNum; + uint32_t vgId = hashValue / hashUnit; + + SHashObj *vgroupHash = NULL; + + CTG_ERR_JRET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); + if (NULL == vgroupHash) { + ctgError("get empty vgroup cache"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + +_return: + if (dbInfo && dbInfo->vgId) { + taosArrayDestroy(dbInfo->vgId); + dbInfo->vgId = NULL; + } + + tfree(dbInfo); + + return code; +} + + + +STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +int32_t ctgGetTableMetaImpl(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { + if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t exist = 0; + + if (!forceUpdate) { + CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + } + + CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName)); + + CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); + + if (0 == exist) { + ctgError("get table meta from cache failed, but fetch succeed"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgUpdateTableMetaCache(SCatalog *pCatalog, STableMetaOutput *output) { + if (output->metaNum != 1 && output->metaNum != 2) { + ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == output->tbMeta) { + ctgError("no valid table meta got from meta rsp"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == pCatalog->tableCache.cache) { + pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.cache) { + ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + if (NULL == pCatalog->tableCache.cache) { + pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.cache) { + ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + + pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.stableCache) { + ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + if (output->metaNum == 2) { + if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { + ctgError("push ctable[%s] to table cache failed", output->ctbFname); + goto error_exit; + } + + if (TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE); + goto error_exit; + } + } + + if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, sizeof(*output->tbMeta)) != 0) { + ctgError("push table[%s] to table cache failed", output->tbFname); + goto error_exit; + } + + if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { + if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) { + ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid); + goto error_exit; + } + } + + return TSDB_CODE_SUCCESS; + +error_exit: + if (pCatalog->vgroupCache.cache) { + taosHashCleanup(pCatalog->vgroupCache.cache); + pCatalog->vgroupCache.cache = NULL; + } + + pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + + +int32_t catalogInit(SCatalogCfg *cfg) { + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == ctgMgmt.pCluster) { + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); + } + + if (cfg) { + memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); + } else { + ctgMgmt.cfg.enableVgroupCache = true; + ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetHandle(const char *clusterId, SCatalog** catalogHandle) { if (NULL == clusterId || NULL == catalogHandle) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -190,7 +501,7 @@ int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { -int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { +int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { if (NULL == pVgroup) { ctgError("no valid vgroup list info to update"); return TSDB_CODE_CTG_INTERNAL_ERROR; @@ -200,22 +511,11 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); return TSDB_CODE_CTG_INVALID_INPUT; } - - - if (NULL == pCatalog->vgroupCache.arrayCache) { - pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0])); - if (NULL == pCatalog->vgroupCache.arrayCache) { - ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum); - return TSDB_CODE_CTG_MEM_ERROR; - } - } else { - taosArrayClear(pCatalog->vgroupCache.arrayCache); - } if (NULL == pCatalog->vgroupCache.cache) { - pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_CACHE_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->vgroupCache.cache) { - ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER); + ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_CACHE_VGROUP_NUMBER); return TSDB_CODE_CTG_MEM_ERROR; } } else { @@ -224,13 +524,7 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) SVgroupInfo *vInfo = NULL; for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { - vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]); - if (NULL == vInfo) { - ctgError("push to vgroup array cache failed"); - goto error_exit; - } - - if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) { + if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &pVgroup->vgroupInfo[i], sizeof(pVgroup->vgroupInfo[i])) != 0) { ctgError("push to vgroup hash cache failed"); goto error_exit; } @@ -241,11 +535,6 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) return TSDB_CODE_SUCCESS; error_exit: - if (pCatalog->vgroupCache.arrayCache) { - taosArrayDestroy(pCatalog->vgroupCache.arrayCache); - pCatalog->vgroupCache.arrayCache = NULL; - } - if (pCatalog->vgroupCache.cache) { taosHashCleanup(pCatalog->vgroupCache.cache); pCatalog->vgroupCache.cache = NULL; @@ -256,15 +545,14 @@ error_exit: return TSDB_CODE_CTG_INTERNAL_ERROR; } - -int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) { +int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { return TSDB_CODE_CTG_INVALID_INPUT; } int32_t exist = 0; - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); if (exist) { return TSDB_CODE_SUCCESS; @@ -274,10 +562,10 @@ int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); - CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup)); + CTG_ERR_RET(catalogUpdateVgroupCache(pCatalog, pVgroup)); - if (pVgroupList) { - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + if (pVgroupHash) { + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); } if (0 == exist) { @@ -288,6 +576,7 @@ int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } + int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { if (NULL == pCatalog || NULL == dbName || NULL == version) { return TSDB_CODE_CTG_INVALID_INPUT; @@ -309,7 +598,7 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { +int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -324,9 +613,9 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB } if (NULL == pCatalog->dbCache.cache) { - pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { - ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_DB_NUMBER); + ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); return TSDB_CODE_CTG_MEM_ERROR; } } @@ -369,11 +658,11 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); if (DbOut.vgroupList) { - CTG_ERR_JRET(catalogUpdateVgroup(pCatalog, DbOut.vgroupList)); + CTG_ERR_JRET(catalogUpdateVgroupCache(pCatalog, DbOut.vgroupList)); } if (DbOut.dbVgroup) { - CTG_ERR_JRET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup)); + CTG_ERR_JRET(catalogUpdateDBVgroupCache(pCatalog, dbName, DbOut.dbVgroup)); } if (dbInfo) { @@ -388,37 +677,35 @@ _return: return code; } +int32_t catalogGetTableMeta(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta); +} - -int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) { + if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) { return TSDB_CODE_CTG_INVALID_INPUT; } - SBuildTableMetaInput bInput = {0}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; + SVgroupInfo vgroupInfo = {0}; + + CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); - if (code) { - return code; - } + STableMetaOutput output = {0}; + + CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); - SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output)); + tfree(output.tbMeta); + return TSDB_CODE_SUCCESS; } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta* pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); +} + +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList) { } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 28d01b9e66..7fcb7ea304 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1464,29 +1464,6 @@ int32_t copyTagData(STagData* dst, const STagData* src) { return 0; } -STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} uint32_t getTableMetaSize(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index 8f35fd9c3e..c5864fd41b 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -222,6 +222,13 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion; pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange); + pOut->dbVgroup->hashType = htonl(pRsp->dbHashType); + + if (pOut->dbVgroup->hashRange < 0) { + qError("invalid hashRange[%d] for db[%s]", pOut->dbVgroup->hashRange, pRsp->db); + code = TSDB_CODE_TSC_INVALID_INPUT; + goto _exit; + } for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) { *(vgIdList + i) = htonl(*(vgIdList + i)); @@ -244,13 +251,134 @@ _exit: return code; } +static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { + pMetaMsg->numOfTags = htonl(pMetaMsg->numOfTags); + pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); + pMetaMsg->sversion = htonl(pMetaMsg->sversion); + pMetaMsg->tversion = htonl(pMetaMsg->tversion); + pMetaMsg->tuid = htobe64(pMetaMsg->tuid); + pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->vgId = htonl(pMetaMsg->vgId); + + if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + qError("invalid numOfColumns[%d] in table meta rsp msg", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && pMetaMsg->tableType != TSDB_NORMAL_TABLE) { + qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->sversion < 0) { + qError("invalid sversion[%d] in table meta rsp msg", pMetaMsg->sversion); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tversion < 0) { + qError("invalid tversion[%d] in table meta rsp msg", pMetaMsg->tversion); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSchema* pSchema = pMetaMsg->pSchema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); + + pSchema++; + } + + if (pMetaMsg->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchema[0].colId); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STableMeta **pMeta) { + int32_t total = msg->numOfColumns + msg->numOfTags; + int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; + + STableMeta* pTableMeta = calloc(1, metaSize); + if (NULL == pTableMeta) { + qError("calloc size[%d] failed", metaSize); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType; + pTableMeta->uid = msg->suid; + pTableMeta->suid = msg->suid; + pTableMeta->sversion = msg->sversion; + pTableMeta->tversion = msg->tversion; + + pTableMeta->tableInfo.numOfTags = msg->numOfTags; + pTableMeta->tableInfo.precision = msg->precision; + pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; + + for(int32_t i = 0; i < msg->numOfColumns; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total); + + *pMeta = pTableMeta; + + return TSDB_CODE_SUCCESS; +} + + +int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)msg; + int32_t code = queryConvertTableMetaMsg(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaOutput *pOut = (STableMetaOutput *)output; + + if (!tIsValidSchema(pMetaMsg->pSchema, pMetaMsg->numOfColumns, pMetaMsg->numOfTags)) { + qError("validate table meta schema in rsp msg failed"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { + pOut->metaNum = 2; + + memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); + memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); + + pOut->ctbMeta.vgId = pMetaMsg->vgId; + pOut->ctbMeta.tableType = pMetaMsg->tableType; + pOut->ctbMeta.uid = pMetaMsg->tuid; + pOut->ctbMeta.suid = pMetaMsg->suid; + + code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta); + } else { + pOut->metaNum = 1; + + memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); + + code = queryCreateTableMetaFromMsg(pMetaMsg, false, &pOut->tbMeta); + } + + return code; +} + void msgInit() { queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; - //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; + queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; From 26a986022631f315cf39f3f7fc50d096aa547bab Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 15 Dec 2021 23:24:03 +0800 Subject: [PATCH 05/11] fix search bug --- source/libs/index/inc/index_fst.h | 4 ++ source/libs/index/src/index_fst.c | 4 ++ source/libs/index/test/indexTests.cpp | 55 +++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index a1d4962e8b..20037f829a 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -30,6 +30,7 @@ extern "C" { typedef struct Fst Fst; typedef struct FstNode FstNode; +typedef struct StreamWithState StreamWithState; typedef enum { Included, Excluded, Unbounded} FstBound; @@ -283,6 +284,9 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +// into stream to expand later +StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb); + bool fstVerify(Fst *fst); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 37bdcb0ecf..54058489c2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1094,6 +1094,10 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } +StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) { + if (sb == NULL) { return NULL; } + return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); +} FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 0cfb0fedc3..f582536817 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -59,9 +59,22 @@ class FstReadMemory { return ok; } // add later - bool Search(const std::string &key, std::vector &result) { + bool Search(AutomationCtx *ctx, std::vector &result) { + FstStreamBuilder *sb = fstSearch(_fst, ctx); + StreamWithState *st = streamBuilderIntoStream(sb); + StreamWithStateResult *rt = NULL; + + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + result.push_back((uint64_t)(rt->out.out)); + } return true; } + bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector &result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } ~FstReadMemory() { fstCountingWriterDestroy(_w); @@ -186,11 +199,43 @@ void checkFstPerf() { printf("success to init fst read"); } Performance_fstReadRecords(m); - delete m; } +void checkFstPrefixSearch() { + FstWriter *fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + std::string key("ab"); + + for (int i = 0; i < count; i++) { + key[1] = key[1] + i; + fw->Put(key, i); + } + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + delete fw; + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + + // prefix search + std::vector result; + AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX); + m->Search(ctx, result); + assert(result.size() == count); + for (int i = 0; i < result.size(); i++) { + assert(result[i] == i); // check result + } + + free(ctx); + delete m; +} void validateFst() { int val = 100; int count = 100; @@ -209,6 +254,8 @@ void validateFst() { FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init() == false) { std::cout << "init readMemory failed" << std::endl; + delete m; + return; } { @@ -230,10 +277,12 @@ void validateFst() { } } delete m; - } + + int main(int argc, char** argv) { checkFstPerf(); + //checkFstPrefixSearch(); return 1; } From 8f16679be418e947ed2f74f1a5cdcd611b205525 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 15 Dec 2021 23:52:23 +0800 Subject: [PATCH 06/11] fix search bug --- source/libs/index/src/index_fst.c | 2 +- source/libs/index/src/index_fst_automation.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 54058489c2..e40195e6e6 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1180,7 +1180,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { - return bound->type == Included ? true : false; + return bound->type == Excluded? false : true; } void fstBoundDestroy(FstBoundWithData *bound) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index d905147654..803c95d0d8 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -145,7 +145,8 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) { StartWithStateValue *sv = NULL; if (atype == AUTOMATION_PREFIX) { - sv = startWithStateValueCreate(Running, FST_INT, 0); + int val = 0; + sv = startWithStateValueCreate(Running, FST_INT, &val); ctx->stdata = (void *)sv; } else if (atype == AUTMMATION_MATCH) { From 4932afa8e2bd51fefc59cac73e9c8ce18941ee58 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Dec 2021 00:15:57 +0800 Subject: [PATCH 07/11] fix search bug --- source/libs/index/src/index_fst.c | 2 +- source/libs/index/src/index_fst_automation.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index e40195e6e6..9cb4ac6836 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1123,7 +1123,7 @@ CompiledAddr fstGetRootAddr(Fst *fst) { Output fstEmptyFinalOutput(Fst *fst, bool *null) { Output res = 0; - FstNode *node = fst->root; + FstNode *node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; res = FST_NODE_FINAL_OUTPUT(node); diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 803c95d0d8..f70b90041b 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -68,7 +68,7 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { // prefix query, impl later static void* prefixStart(AutomationCtx *ctx) { - StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); return startWithStateValueDump(data); }; static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { From cb1279a674deea53e2e3c35621d027b0bdb8907c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Dec 2021 08:13:05 +0800 Subject: [PATCH 08/11] change get table meta api --- source/libs/parser/src/insertParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index fa59bc6ca7..219dd42f78 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -186,7 +186,7 @@ static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalog static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { SCatalogReq req; CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, NULL, NULL)); //TODO pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); return TSDB_CODE_SUCCESS; } From 83bf82f077f960910179836c82c08bd1418019c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Dec 2021 09:54:03 +0800 Subject: [PATCH 09/11] support new apis --- include/libs/catalog/catalog.h | 8 +- source/common/inc/commonInt.h | 1 + source/libs/catalog/src/catalog.c | 182 +++++++++++++++++++++------ source/libs/parser/src/astValidate.c | 2 +- 4 files changed, 153 insertions(+), 40 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 38965c6ba9..6bbc4f9109 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -32,7 +32,7 @@ extern "C" { struct SCatalog; typedef struct SCatalogReq { - char clusterId[TSDB_CLUSTER_ID_LEN]; //???? + char dbName[TSDB_DB_NAME_LEN]; SArray *pTableName; // table full name SArray *pUdf; // udf name bool qNodeRequired; // valid qnode @@ -82,7 +82,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta* pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); /** @@ -91,7 +91,7 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const * @pVgroupList - array of SVgroupInfo * @return */ -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList); +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); /** @@ -103,7 +103,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe * @param pMetaData * @return */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index e7d2dba95c..5b71f83faf 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 9921a2696b..a670d9d639 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -16,6 +16,7 @@ #include "catalogInt.h" #include "trpc.h" #include "query.h" +#include "tname.h" SCatalogMgmt ctgMgmt = {0}; @@ -47,7 +48,7 @@ int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSe return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { +int32_t ctgGetVgroupFromCache(struct SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) { *exist = 0; return TSDB_CODE_SUCCESS; @@ -62,7 +63,7 @@ int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SHashObj** pVgroupList, int32_ return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; @@ -130,13 +131,13 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp } -int32_t ctgGetTableMetaFromCache(SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) { +int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) { if (NULL == pCatalog->tableCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + char tbFullName[TSDB_TABLE_FNAME_LEN]; snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName); @@ -200,7 +201,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE return TSDB_CODE_CTG_INVALID_INPUT; } - char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + char tbFullName[TSDB_TABLE_FNAME_LEN]; snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); @@ -251,7 +252,54 @@ int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableHashVgroup(SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { +int32_t ctgGetVgroupFromVgId(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, int32_t vgId, SVgroupInfo *pVgroup) { + SHashObj *vgroupHash = NULL; + + CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); + if (NULL == vgroupHash) { + ctgError("get empty vgroup cache"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetVgroupFromVgIdBatch(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SArray* vgIds, SArray* vgroupList) { + SHashObj *vgroupHash = NULL; + SVgroupInfo pVgroup = {0}; + int32_t vgIdNum = taosArrayGetSize(vgIds); + + CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); + if (NULL == vgroupHash) { + ctgError("get empty vgroup cache"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + for (int32_t i = 0; i < vgIdNum; ++i) { + int32_t *vgId = taosArrayGet(vgIds, i); + + if (NULL == taosHashGetClone(vgroupHash, vgId, sizeof(*vgId), &pVgroup)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == taosArrayPush(vgroupList, &pVgroup)) { + ctgError("push vgroup to array failed, idx:%d", i); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { SDBVgroupInfo *dbInfo = NULL; int32_t code = 0; @@ -269,7 +317,7 @@ int32_t ctgGetTableHashVgroup(SCatalog *pCatalog, void *pRpc, const SEpSet *pMgm int32_t vgNum = taosArrayGetSize(dbInfo->vgId); if (vgNum <= 0) { - ctgError("db[%s] vgroup cache invalid, vgroup number:%p", vgNum); + ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); } @@ -277,7 +325,7 @@ int32_t ctgGetTableHashVgroup(SCatalog *pCatalog, void *pRpc, const SEpSet *pMgm CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp)); - char tbFullName[TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + char tbFullName[TSDB_TABLE_FNAME_LEN]; snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); @@ -285,18 +333,7 @@ int32_t ctgGetTableHashVgroup(SCatalog *pCatalog, void *pRpc, const SEpSet *pMgm uint32_t hashUnit = dbInfo->hashRange / vgNum; uint32_t vgId = hashValue / hashUnit; - SHashObj *vgroupHash = NULL; - - CTG_ERR_JRET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); - if (NULL == vgroupHash) { - ctgError("get empty vgroup cache"); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { - ctgError("vgId[%d] not found in vgroup list", vgId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, vgId, pVgroup)); _return: if (dbInfo && dbInfo->vgId) { @@ -335,7 +372,7 @@ STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) { return pTableMeta; } -int32_t ctgGetTableMetaImpl(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { +int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -363,7 +400,7 @@ int32_t ctgGetTableMetaImpl(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtE } -int32_t ctgUpdateTableMetaCache(SCatalog *pCatalog, STableMetaOutput *output) { +int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { if (output->metaNum != 1 && output->metaNum != 2) { ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); return TSDB_CODE_CTG_INTERNAL_ERROR; @@ -433,7 +470,6 @@ error_exit: return TSDB_CODE_CTG_INTERNAL_ERROR; } - int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { @@ -451,7 +487,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(const char *clusterId, SCatalog** catalogHandle) { +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { if (NULL == clusterId || NULL == catalogHandle) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -545,7 +581,7 @@ error_exit: return TSDB_CODE_CTG_INTERNAL_ERROR; } -int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -677,7 +713,7 @@ _return: return code; } -int32_t catalogGetTableMeta(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta); } @@ -701,21 +737,97 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSe return TSDB_CODE_SUCCESS; } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta* pTableMeta) { +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); } -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList) { - -} - - -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { return TSDB_CODE_CTG_INVALID_INPUT; } - return 0; + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SDBVgroupInfo *dbVgroup = NULL; + + CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); + + if (tbMeta->tableType == TSDB_SUPER_TABLE) { + CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); + + CTG_ERR_JRET(ctgGetVgroupFromVgIdBatch(pCatalog, pRpc, pMgmtEps, dbVgroup->vgId, pVgroupList)); + } else { + CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, tbMeta->vgId, &vgroupInfo)); + + if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { + ctgError("push vgroupInfo to array failed"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + } + +_return: + tfree(tbMeta); + if (dbVgroup && dbVgroup->vgId) { + taosArrayDestroy(dbVgroup->vgId); + dbVgroup->vgId = NULL; + } + + tfree(dbVgroup); + + return code; +} + + +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t code = 0; + + if (pReq->pTableName) { + char dbName[TSDB_FULL_DB_NAME_LEN]; + int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); + if (tbNum > 0) { + pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); + if (NULL == pRsp->pTableMeta) { + ctgError("taosArrayInit num[%d] failed", tbNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName *name = taosArrayGet(pReq->pTableName, i); + STableMeta *pTableMeta = NULL; + + snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname); + + CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta)); + + if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { + ctgError("taosArrayPush failed, idx:%d", i); + tfree(pTableMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + } + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (pRsp->pTableMeta) { + int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); + for (int32_t i = 0; i < aSize; ++i) { + STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); + tfree(pMeta); + } + + taosArrayDestroy(pRsp->pTableMeta); + } + + return code; } void catalogDestroy(void) { diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index d2813886b3..9f1363a9b4 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4088,7 +4088,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } // load the meta data from catalog - code = catalogGetAllMeta(pCatalog, NULL, &req, &data); + code = catalogGetAllMeta(pCatalog, NULL, NULL, &req, &data); if (code != TSDB_CODE_SUCCESS) { return code; } From c84b6d09e497442c3a0e940fbb410915b7791f0b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Dec 2021 22:18:51 -0500 Subject: [PATCH 10/11] TD-12035 Scan subquery physical plan code. --- include/libs/parser/parser.h | 12 +- include/libs/planner/planner.h | 104 ++++++++++++----- include/libs/planner/plannerOp.h | 48 ++++++++ include/util/tarray.h | 2 +- source/libs/parser/src/insertParser.c | 52 +++++---- source/libs/parser/test/CMakeLists.txt | 11 +- source/libs/parser/test/insertTest.cpp | 21 ++++ source/libs/planner/inc/plannerInt.h | 51 +-------- source/libs/planner/src/physicalPlan.c | 150 +++++++++++++++++-------- source/libs/planner/src/planner.c | 3 +- source/libs/scheduler/CMakeLists.txt | 2 +- source/util/src/tarray.c | 2 +- 12 files changed, 295 insertions(+), 163 deletions(-) create mode 100644 include/libs/planner/plannerOp.h diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 2f152c3e2b..d65b5ab570 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -132,13 +132,15 @@ struct SInsertStmtInfo; bool qIsInsertSql(const char* pStr, size_t length); typedef struct SParseContext { + const char* pAcctId; + const char* pDbname; + void *pRpc; + const char* pClusterId; + const SEpSet* pEpSet; + int64_t id; // query id, generated by uuid generator + int8_t schemaAttached; // denote if submit block is built with table schema or not const char* pSql; // sql string size_t sqlLen; // length of the sql string - int64_t id; // operator id, generated by uuid generator - const char* pDbname; - const SEpSet* pEpSet; - int8_t schemaAttached; // denote if submit block is built with table schema or not - char* pMsg; // extended error message if exists to help avoid the problem in sql statement. int32_t msgLen; // max length of the msg } SParseContext; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index be00ed65f0..844757eeb5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -20,51 +20,92 @@ extern "C" { #endif +#include "taosmsg.h" + #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_SCAN 3 enum OPERATOR_TYPE_E { - OP_TableScan = 1, - OP_DataBlocksOptScan = 2, - OP_TableSeqScan = 3, - OP_TagScan = 4, - OP_TableBlockInfoScan= 5, - OP_Aggregate = 6, - OP_Project = 7, - OP_Groupby = 8, - OP_Limit = 9, - OP_SLimit = 10, - OP_TimeWindow = 11, - OP_SessionWindow = 12, - OP_StateWindow = 22, - OP_Fill = 13, - OP_MultiTableAggregate = 14, - OP_MultiTableTimeInterval = 15, -// OP_DummyInput = 16, //TODO remove it after fully refactor. -// OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. -// OP_GlobalAggregate = 18, // global merge for the multi-way data sources. - OP_Filter = 19, - OP_Distinct = 20, - OP_Join = 21, - OP_AllTimeWindow = 23, - OP_AllMultiTableTimeInterval = 24, - OP_Order = 25, - OP_Exchange = 26, + OP_Unknown, +#define INCLUDE_AS_ENUM +#include "plannerOp.h" +#undef INCLUDE_AS_ENUM + OP_TotalNum }; struct SEpSet; -struct SPhyNode; struct SQueryStmtInfo; +typedef SSchema SSlotSchema; + +typedef struct SDataBlockSchema { + SSlotSchema *pSchema; + int32_t numOfCols; // number of columns +} SDataBlockSchema; + +typedef struct SQueryNodeBasicInfo { + int32_t type; // operator type + const char *name; // operator name +} SQueryNodeBasicInfo; + +typedef struct SPhyNode { + SQueryNodeBasicInfo info; + SArray *pTargets; // target list to be computed or scanned at this node + SArray *pConditions; // implicitly-ANDed qual conditions + SDataBlockSchema targetSchema; + // children plan to generated result for current node to process + // in case of join, multiple plan nodes exist. + SArray *pChildren; + struct SPhyNode *pParent; +} SPhyNode; + +typedef struct SScanPhyNode { + SPhyNode node; + uint64_t uid; // unique id of the table + int8_t tableType; +} SScanPhyNode; + +typedef SScanPhyNode SSystemTableScanPhyNode; +typedef SScanPhyNode STagScanPhyNode; + +typedef struct STableScanPhyNode { + SScanPhyNode scan; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow window; + SArray *pTagsConditions; // implicitly-ANDed tag qual conditions +} STableScanPhyNode; + +typedef STableScanPhyNode STableSeqScanPhyNode; + +typedef struct SProjectPhyNode { + SPhyNode node; +} SProjectPhyNode; + +typedef struct SExchangePhyNode { + SPhyNode node; + uint64_t templateId; + SArray *pSourceEpSet; // SEpSet +} SExchangePhyNode; + +typedef struct SSubplanId { + uint64_t queryId; + uint64_t templateId; + uint64_t subplanId; +} SSubplanId; + typedef struct SSubplan { - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN - SArray *pDatasource; // the datasource subplan,from which to fetch the result - struct SPhyNode *pNode; // physical plan of current subplan + SSubplanId id; // unique id of the subplan + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t level; // the execution level of current subplan, starting from 0. + SEpSet execEpSet; // for the scan sub plan, the optional execution node + SArray *pChildern; // the datasource subplan,from which to fetch the result + SArray *pParents; // the data destination subplan, get data from current subplan + SPhyNode *pNode; // physical plan of current subplan } SSubplan; typedef struct SQueryDag { - SArray **pSubplans; + SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; /** @@ -74,6 +115,7 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); + /** * Convert to subplan to string for the scheduler to send to the executor */ diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h new file mode 100644 index 0000000000..27c7c534a2 --- /dev/null +++ b/include/libs/planner/plannerOp.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#if defined(INCLUDE_AS_ENUM) // enum define mode + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) OP_##op, +#elif defined(INCLUDE_AS_NAME) // comment define mode + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) #op, +#else + #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME +#endif + +OP_ENUM_MACRO(TableScan) +OP_ENUM_MACRO(DataBlocksOptScan) +OP_ENUM_MACRO(TableSeqScan) +OP_ENUM_MACRO(TagScan) +OP_ENUM_MACRO(TableBlockInfoScan) +OP_ENUM_MACRO(Aggregate) +OP_ENUM_MACRO(Project) +OP_ENUM_MACRO(Groupby) +OP_ENUM_MACRO(Limit) +OP_ENUM_MACRO(SLimit) +OP_ENUM_MACRO(TimeWindow) +OP_ENUM_MACRO(SessionWindow) +OP_ENUM_MACRO(StateWindow) +OP_ENUM_MACRO(Fill) +OP_ENUM_MACRO(MultiTableAggregate) +OP_ENUM_MACRO(MultiTableTimeInterval) +OP_ENUM_MACRO(Filter) +OP_ENUM_MACRO(Distinct) +OP_ENUM_MACRO(Join) +OP_ENUM_MACRO(AllTimeWindow) +OP_ENUM_MACRO(AllMultiTableTimeInterval) +OP_ENUM_MACRO(Order) +OP_ENUM_MACRO(Exchange) diff --git a/include/util/tarray.h b/include/util/tarray.h index e0f14dcd25..f7c72add01 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -41,7 +41,7 @@ typedef struct SArray { * @param elemSize * @return */ -void* taosArrayInit(size_t size, size_t elemSize); +SArray* taosArrayInit(size_t size, size_t elemSize); /** * diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index fa59bc6ca7..97102c5b00 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -71,8 +71,7 @@ typedef struct SInsertParseContext { const char* pSql; SMsgBuf msg; struct SCatalog* pCatalog; - SMetaData meta; // need release - const STableMeta* pTableMeta; + STableMeta tableMeta; SHashObj* pTableBlockHashObj; // data block for each table. need release int32_t totalNum; SInsertStmtInfo* pOutput; @@ -165,29 +164,29 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { +static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) { if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); } - SName name = {0}; - strcpy(name.dbname, pCxt->pComCxt->pDbname); - strncpy(name.tname, pStname->z, pStname->n); - taosArrayPush(tableNameList, &name); - + char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); + if (NULL != p) { // db.table + strcpy(fullDbName, pCxt->pComCxt->pAcctId); + fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0]; + strncpy(fullDbName, pStname->z, p - pStname->z); + strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); + } else { + snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname); + strncpy(tableName, pStname->z, pStname->n); + } return TSDB_CODE_SUCCESS; } -static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) { - pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); - return buildTableName(pCxt, pStname, pMetaReq->pTableName); -} - static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - SCatalogReq req; - CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO - pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); + char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, &pCxt->tableMeta)); return TSDB_CODE_SUCCESS; } @@ -646,13 +645,13 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); CHECK_CODE(getTableMeta(pCxt, &sToken)); - if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { + if (TSDB_SUPER_TABLE != pCxt->tableMeta.tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); + SSchema* pTagsSchema = getTableTagSchema(&pCxt->tableMeta); SParsedDataColInfo spd = {0}; - setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(&pCxt->tableMeta)); // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); @@ -669,7 +668,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(&pCxt->tableMeta).precision)); return TSDB_CODE_SUCCESS; } @@ -811,12 +810,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } STableDataBlocks *dataBuf = NULL; - CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->tableMeta.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(&pCxt->tableMeta).rowSize, &pCxt->tableMeta, &dataBuf, NULL)); if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(&pCxt->tableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -862,18 +861,17 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pCatalog = NULL, - .pTableMeta = NULL, + .tableMeta = {0}, .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, .pOutput = *pInfo }; - CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO - if (NULL == context.pTableBlockHashObj) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog)); CHECK_CODE(skipInsertInto(&context)); CHECK_CODE(parseInsertBody(&context)); diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 4b9e586be3..03b76152da 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -6,13 +6,16 @@ SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(parserTest ${SOURCE_LIST}) -TARGET_LINK_LIBRARIES( - parserTest - PUBLIC os util common parser catalog transport gtest function planner query -) TARGET_INCLUDE_DIRECTORIES( parserTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc" ) + +TARGET_LINK_LIBRARIES( + parserTest + PUBLIC os util common parser catalog transport gtest function planner query +) + +TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 9cf48da4eb..5877adf41c 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -27,6 +27,27 @@ namespace { } } +extern "C" { + +#include + +void *__real_malloc(size_t); + +void *__wrap_malloc(size_t c) { + // printf("My MALLOC called: %d\n", c); + // void *array[32]; + // int size = backtrace(array, 32); + // char **symbols = backtrace_symbols(array, size); + // for (int i = 0; i < size; ++i) { + // cout << symbols[i] << endl; + // } + // free(symbols); + + return __real_malloc(c); +} + +} + // syntax: // INSERT INTO // tb_name diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 6dcd19782c..6c65e4810d 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -23,6 +23,7 @@ extern "C" { #include "common.h" #include "tarray.h" #include "planner.h" +#include "parser.h" #include "taosmsg.h" #define QNODE_TAGSCAN 1 @@ -40,11 +41,6 @@ extern "C" { #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 -typedef struct SQueryNodeBasicInfo { - int32_t type; // operator type - char *name; // operator name -} SQueryNodeBasicInfo; - typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not int32_t phase; // merge|partial @@ -54,8 +50,9 @@ typedef struct SQueryDistPlanNodeInfo { } SQueryDistPlanNodeInfo; typedef struct SQueryTableInfo { - char *tableName; - uint64_t uid; + char *tableName; // to be deleted + uint64_t uid; // to be deleted + STableMetaInfo* pMeta; STimeWindow window; } SQueryTableInfo; @@ -72,46 +69,6 @@ typedef struct SQueryPlanNode { struct SQueryPlanNode *pParent; } SQueryPlanNode; -typedef SSchema SSlotSchema; - -typedef struct SDataBlockSchema { - int32_t index; - SSlotSchema *pSchema; - int32_t numOfCols; // number of columns -} SDataBlockSchema; - -typedef struct SPhyNode { - SQueryNodeBasicInfo info; - SArray *pTargets; // target list to be computed or scanned at this node - SArray *pConditions; // implicitly-ANDed qual conditions - SDataBlockSchema targetSchema; - // children plan to generated result for current node to process - // in case of join, multiple plan nodes exist. - SArray *pChildren; - struct SPhyNode *pParent; -} SPhyNode; - -typedef struct SScanPhyNode { - SPhyNode node; - STimeWindow window; - uint64_t uid; // unique id of the table -} SScanPhyNode; - -typedef SScanPhyNode STagScanPhyNode; - -typedef SScanPhyNode SSystemTableScanPhyNode; - -typedef struct SMultiTableScanPhyNode { - SScanPhyNode scan; - SArray *pTagsConditions; // implicitly-ANDed tag qual conditions -} SMultiTableScanPhyNode; - -typedef SMultiTableScanPhyNode SMultiTableSeqScanPhyNode; - -typedef struct SProjectPhyNode { - SPhyNode node; -} SProjectPhyNode; - /** * Optimize the query execution plan, currently not implement yet. * @param pQueryNode diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index c4564a9a09..e7acb12bc0 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,64 +14,107 @@ */ #include "plannerInt.h" +#include "parser.h" -// typedef struct SQueryPlanNode { -// void *pExtInfo; // additional information -// SArray *pPrevNodes; // children -// struct SQueryPlanNode *nextNode; // parent -// } SQueryPlanNode; +static const char* gOpName[] = { + "Unknown", +#define INCLUDE_AS_NAME +#include "plannerOp.h" +#undef INCLUDE_AS_NAME +}; -// typedef struct SSubplan { -// int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN -// SArray *pDatasource; // the datasource subplan,from which to fetch the result -// struct SPhyNode *pNode; // physical plan of current subplan -// } SSubplan; +typedef struct SPlanContext { + struct SCatalog* pCatalog; + struct SQueryDag* pDag; + SSubplan* pCurrentSubplan; + SSubplanId nextId; +} SPlanContext; -// typedef struct SQueryDag { -// SArray **pSubplans; -// } SQueryDag; - -// typedef struct SScanPhyNode { -// SPhyNode node; -// STimeWindow window; -// uint64_t uid; // unique id of the table -// } SScanPhyNode; - -// typedef SScanPhyNode STagScanPhyNode; - -void fillDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - dataBlockSchema->index = 0; // todo +static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); dataBlockSchema->numOfCols = pPlanNode->numOfCols; } -void fillPhyNode(SQueryPlanNode* pPlanNode, int32_t type, const char* name, SPhyNode* node) { +static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { + SPhyNode* node = (SPhyNode*)calloc(1, size); node->info.type = type; - node->info.name = name; + node->info.name = gOpName[type]; SWAP(node->pTargets, pPlanNode->pExpr, SArray*); - fillDataBlockSchema(pPlanNode, &(node->targetSchema)); + toDataBlockSchema(pPlanNode, &(node->targetSchema)); } -SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { - STagScanPhyNode* node = calloc(1, sizeof(STagScanPhyNode)); - fillPhyNode(pPlanNode, OP_TagScan, "TagScan", (SPhyNode*)node); - return (SPhyNode*)node; +static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { + return initPhyNode(pPlanNode, OP_TagScan, sizeof(STagScanPhyNode)); } -SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) { - STagScanPhyNode* node = calloc(1, sizeof(STagScanPhyNode)); - fillPhyNode(pPlanNode, OP_TableScan, "SingleTableScan", (SPhyNode*)node); - return (SPhyNode*)node; +static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { + SSubplan* subplan = calloc(1, sizeof(SSubplan)); + subplan->id = pCxt->nextId; + ++(pCxt->nextId.subplanId); + subplan->type = type; + subplan->level = 0; + if (NULL != pCxt->pCurrentSubplan) { + subplan->level = pCxt->pCurrentSubplan->level + 1; + if (NULL == pCxt->pCurrentSubplan->pChildern) { + pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + } + taosArrayPush(pCxt->pCurrentSubplan->pChildern, subplan); + subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(subplan->pParents, pCxt->pCurrentSubplan); + } + pCxt->pCurrentSubplan = subplan; + return subplan; } -SPhyNode* createPhyNode(SQueryPlanNode* pPlanNode) { +static uint8_t getScanFlag(SQueryPlanNode* pPlanNode) { + // todo + return MASTER_SCAN; +} + +static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + STableScanPhyNode* node = (STableScanPhyNode*)initPhyNode(pPlanNode, OP_TableScan, sizeof(STableScanPhyNode)); + node->scan.uid = pTable->pMeta->pTableMeta->uid; + node->scan.tableType = pTable->pMeta->pTableMeta->tableType; + node->scanFlag = getScanFlag(pPlanNode); + node->window = pTable->window; + // todo tag cond +} + +static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { + // todo +} + +static void splitSubplanBySTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList; + for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); + vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); + subplan->pNode = createTableScanNode(pCxt, pPlanNode, pTable); + // todo reset pCxt->pCurrentSubplan + } +} + +static SPhyNode* createExchangeNode() { + +} + +static SPhyNode* createScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; + if (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType) { + splitSubplanBySTable(pCxt, pPlanNode, pTable); + return createExchangeNode(pCxt, pTable); + } + return createTableScanNode(pCxt, pPlanNode, pTable); +} + +static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SPhyNode* node = NULL; switch (pPlanNode->info.type) { case QNODE_TAGSCAN: node = createTagScanNode(pPlanNode); break; case QNODE_TABLESCAN: - node = createScanNode(pPlanNode); + node = createScanNode(pCxt, pPlanNode); break; default: assert(false); @@ -80,7 +123,7 @@ SPhyNode* createPhyNode(SQueryPlanNode* pPlanNode) { node->pChildren = taosArrayInit(4, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { - SPhyNode* child = createPhyNode(taosArrayGet(pPlanNode->pChildren, i)); + SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); child->pParent = node; taosArrayPush(node->pChildren, &child); } @@ -88,13 +131,30 @@ SPhyNode* createPhyNode(SQueryPlanNode* pPlanNode) { return node; } -SSubplan* createSubplan(SQueryPlanNode* pSubquery) { - SSubplan* subplan = calloc(1, sizeof(SSubplan)); - subplan->pNode = createPhyNode(pSubquery); - // todo - return subplan; +static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + subplan->pNode = createPhyNode(pCxt, pRoot); + SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(l0, &subplan); + taosArrayPush(pCxt->pDag->pSubplans, &l0); + // todo deal subquery } -int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { - return 0; +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) { + SPlanContext context = { + .pCatalog = pCatalog, + .pDag = calloc(1, sizeof(SQueryDag)), + .pCurrentSubplan = NULL + }; + if (NULL == context.pDag) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + createSubplanByLevel(&context, pQueryNode); + *pDag = context.pDag; + return TSDB_CODE_SUCCESS; +} + +int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) { + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index b18dd29257..19aac36e78 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -95,6 +95,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla } switch(type) { + case QNODE_TAGSCAN: case QNODE_TABLESCAN: { SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); @@ -162,7 +163,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, NULL); + SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); if (pQueryInfo->info.distinct) { pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 770a6b02c2..fd00085381 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner + PRIVATE os util planner common ) \ No newline at end of file diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 581a797343..cc8d6646b6 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -17,7 +17,7 @@ #include "tarray.h" #include "talgo.h" -void* taosArrayInit(size_t size, size_t elemSize) { +SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); if (size < TARRAY_MIN_SIZE) { From 3b8725461a436bc1be525f481911a8dcfc2cae3d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Dec 2021 14:01:10 +0800 Subject: [PATCH 11/11] change use db msg and remove vgroup cache --- include/common/taosmsg.h | 23 +- include/libs/catalog/catalog.h | 16 +- include/libs/query/query.h | 13 +- source/libs/catalog/CMakeLists.txt | 2 + source/libs/catalog/src/catalog.c | 324 +++++----------------- source/libs/catalog/test/CMakeLists.txt | 18 ++ source/libs/catalog/test/catalogTests.cpp | 152 ++++++++++ source/libs/query/src/querymsg.c | 174 +++--------- 8 files changed, 296 insertions(+), 426 deletions(-) create mode 100644 source/libs/catalog/test/CMakeLists.txt diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 1fe930ba7a..8abfe0ffed 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -221,8 +221,7 @@ typedef struct SBuildTableMetaInput { typedef struct SBuildUseDBInput { char db[TSDB_TABLE_FNAME_LEN]; - int32_t vgroupVersion; - int32_t dbGroupVersion; + int32_t vgVersion; } SBuildUseDBInput; @@ -627,8 +626,7 @@ typedef struct { typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int8_t ignoreNotExists; - int32_t vgroupVersion; - int32_t dbGroupVersion; + int32_t vgVersion; int32_t reserve[8]; } SUseDbMsg; @@ -808,6 +806,9 @@ typedef struct SSTableVgroupMsg { typedef struct SVgroupInfo { int32_t vgId; + int32_t hashBegin; + int32_t hashEnd; + int8_t inUse; int8_t numOfEps; SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; @@ -863,16 +864,12 @@ typedef struct { } STagData; typedef struct { - int32_t vgroupNum; - int32_t vgroupVersion; - char db[TSDB_TABLE_FNAME_LEN]; - int32_t dbVgroupVersion; - int32_t dbVgroupNum; - int32_t dbHashRange; - int32_t dbHashType; + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; SVgroupInfo vgroupInfo[]; -//int32_t vgIdList[]; -} SUseDbRspMsg; +} SUseDbRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6bbc4f9109..1f2452291b 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -61,22 +61,8 @@ int32_t catalogInit(SCatalogCfg *cfg); */ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); - - -int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); - -/** - * get cluster vgroup list. - * @pVgroupList - hash of vgroup list, key:vgId, value:SVgroupInfo - * @return - */ -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash); -int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); - - - int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); diff --git a/include/libs/query/query.h b/include/libs/query/query.h index bfe2db6a61..8720fd085c 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tarray.h" +#include "thash.h" typedef SVgroupListRspMsg SVgroupListInfo; @@ -63,16 +64,14 @@ typedef struct STableMeta { typedef struct SDBVgroupInfo { - int32_t vgroupVersion; - SArray *vgId; - int32_t hashRange; - int32_t hashType; + int32_t vgVersion; + int8_t hashMethod; + SHashObj *vgInfo; //key:vgId, value:SVgroupInfo } SDBVgroupInfo; typedef struct SUseDbOutput { - SVgroupListInfo *vgroupList; - char db[TSDB_TABLE_FNAME_LEN]; - SDBVgroupInfo *dbVgroup; + char db[TSDB_FULL_DB_NAME_LEN]; + SDBVgroupInfo dbVgroup; } SUseDbOutput; typedef struct STableMetaOutput { diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index 25c80d502a..e6311152d6 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( catalog PRIVATE os util common transport query ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a670d9d639..9fdac36060 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,50 +20,7 @@ SCatalogMgmt ctgMgmt = {0}; -int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) { - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; - - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); - if (code) { - return code; - } - - SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_VGROUP_LIST, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - - code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetVgroupFromCache(struct SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { - if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) { - *exist = 0; - return TSDB_CODE_SUCCESS; - } - - if (pVgroupList) { - *pVgroupList = pCatalog->vgroupCache.cache; - } - - *exist = 1; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; @@ -71,28 +28,13 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (NULL == info || info->vgroupVersion < pCatalog->vgroupCache.vgroupVersion) { + if (NULL == info) { *exist = 0; return TSDB_CODE_SUCCESS; } if (dbInfo) { - *dbInfo = calloc(1, sizeof(**dbInfo)); - if (NULL == *dbInfo) { - ctgError("calloc size[%d] failed", (int32_t)sizeof(**dbInfo)); - return TSDB_CODE_CTG_MEM_ERROR; - } - - (*dbInfo)->vgId = taosArrayDup(info->vgId); - if (NULL == (*dbInfo)->vgId) { - ctgError("taos array duplicate failed"); - tfree(*dbInfo); - return TSDB_CODE_CTG_MEM_ERROR; - } - - (*dbInfo)->vgroupVersion = info->vgroupVersion; - (*dbInfo)->hashRange = info->hashRange; - (*dbInfo)->hashType = info->hashType; + *dbInfo = *info; } *exist = 1; @@ -242,8 +184,8 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE } -int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { - switch (hashType) { +int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { + switch (hashMethod) { default: *fp = MurmurHash3_32; break; @@ -252,96 +194,79 @@ int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgroupFromVgId(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, int32_t vgId, SVgroupInfo *pVgroup) { +int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) { SHashObj *vgroupHash = NULL; - - CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); - if (NULL == vgroupHash) { - ctgError("get empty vgroup cache"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + SVgroupInfo *vgInfo = NULL; - if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { - ctgError("vgId[%d] not found in vgroup list", vgId); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + while (pIter) { + vgInfo = pIter; - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetVgroupFromVgIdBatch(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SArray* vgIds, SArray* vgroupList) { - SHashObj *vgroupHash = NULL; - SVgroupInfo pVgroup = {0}; - int32_t vgIdNum = taosArrayGetSize(vgIds); - - CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); - if (NULL == vgroupHash) { - ctgError("get empty vgroup cache"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - for (int32_t i = 0; i < vgIdNum; ++i) { - int32_t *vgId = taosArrayGet(vgIds, i); + if (NULL == taosArrayPush(vgroupList, vgInfo)) { + ctgError("taosArrayPush failed"); + break; + } - if (NULL == taosHashGetClone(vgroupHash, vgId, sizeof(*vgId), &pVgroup)) { - ctgError("vgId[%d] not found in vgroup list", vgId); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - if (NULL == taosArrayPush(vgroupList, &pVgroup)) { - ctgError("push vgroup to array failed, idx:%d", i); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + pIter = taosHashIterate(dbInfo->vgInfo, pIter); + vgInfo = NULL; } return TSDB_CODE_SUCCESS; } - - -int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo *dbInfo = NULL; - int32_t code = 0; - - CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); - - if (NULL == dbInfo) { - ctgWarn("db[%s] vgroup info not found", pDBName); +int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + if (vgNum <= 0) { + ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); return TSDB_CODE_TSC_DB_NOT_SELECTED; } - if (dbInfo->vgroupVersion < 0 || NULL == dbInfo->vgId) { - ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgId:%p", pDBName, dbInfo->vgroupVersion, dbInfo->vgId); - CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - int32_t vgNum = taosArrayGetSize(dbInfo->vgId); - if (vgNum <= 0) { - ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); - CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp)); + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - uint32_t hashUnit = dbInfo->hashRange / vgNum; - uint32_t vgId = hashValue / hashUnit; - CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, vgId, pVgroup)); - -_return: - if (dbInfo && dbInfo->vgId) { - taosArrayDestroy(dbInfo->vgId); - dbInfo->vgId = NULL; + void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + break; + } + + pIter = taosHashIterate(dbInfo->vgInfo, pIter); + vgInfo = NULL; } - - tfree(dbInfo); + + if (NULL == vgInfo) { + ctgError("no hash range found for hashvalue[%u]", hashValue); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *pVgroup = *vgInfo; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + SDBVgroupInfo dbInfo = {0}; + int32_t code = 0; + int32_t vgId = 0; + + CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); + + if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { + ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); + return TSDB_CODE_TSC_DB_NOT_SELECTED; + } + + CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); return code; } @@ -524,95 +449,6 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) return TSDB_CODE_SUCCESS; } - -int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { - if (NULL == pCatalog || NULL == version) { - return TSDB_CODE_CTG_INVALID_INPUT; - } - - *version = pCatalog->vgroupCache.vgroupVersion; - - return TSDB_CODE_SUCCESS; -} - - - -int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { - if (NULL == pVgroup) { - ctgError("no valid vgroup list info to update"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - if (pVgroup->vgroupVersion < 0) { - ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); - return TSDB_CODE_CTG_INVALID_INPUT; - } - - if (NULL == pCatalog->vgroupCache.cache) { - pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_CACHE_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->vgroupCache.cache) { - ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_CACHE_VGROUP_NUMBER); - return TSDB_CODE_CTG_MEM_ERROR; - } - } else { - taosHashClear(pCatalog->vgroupCache.cache); - } - - SVgroupInfo *vInfo = NULL; - for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { - if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &pVgroup->vgroupInfo[i], sizeof(pVgroup->vgroupInfo[i])) != 0) { - ctgError("push to vgroup hash cache failed"); - goto error_exit; - } - } - - pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion; - - return TSDB_CODE_SUCCESS; - -error_exit: - if (pCatalog->vgroupCache.cache) { - taosHashCleanup(pCatalog->vgroupCache.cache); - pCatalog->vgroupCache.cache = NULL; - } - - pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { - return TSDB_CODE_CTG_INVALID_INPUT; - } - - int32_t exist = 0; - - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); - - if (exist) { - return TSDB_CODE_SUCCESS; - } - - SVgroupListInfo *pVgroup = NULL; - - CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); - - CTG_ERR_RET(catalogUpdateVgroupCache(pCatalog, pVgroup)); - - if (pVgroupHash) { - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); - } - - if (0 == exist) { - ctgError("catalog fetched but get from cache failed"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - - int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { if (NULL == pCatalog || NULL == dbName || NULL == version) { return TSDB_CODE_CTG_INVALID_INPUT; @@ -629,7 +465,7 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } - *version = dbInfo->vgroupVersion; + *version = dbInfo->vgVersion; return TSDB_CODE_SUCCESS; } @@ -639,7 +475,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName return TSDB_CODE_CTG_INVALID_INPUT; } - if (dbInfo->vgroupVersion < 0) { + if (dbInfo->vgVersion < 0) { if (pCatalog->dbCache.cache) { taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)); } @@ -654,6 +490,12 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); return TSDB_CODE_CTG_MEM_ERROR; } + } else { + SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (oldInfo && oldInfo->vgInfo) { + taosHashCleanup(oldInfo->vgInfo); + oldInfo->vgInfo = NULL; + } } if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) { @@ -667,7 +509,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) { +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -688,28 +530,16 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* strncpy(input.db, dbName, sizeof(input.db)); input.db[sizeof(input.db) - 1] = 0; - input.vgroupVersion = pCatalog->vgroupCache.vgroupVersion; - input.dbGroupVersion = CTG_DEFAULT_INVALID_VERSION; + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - if (DbOut.vgroupList) { - CTG_ERR_JRET(catalogUpdateVgroupCache(pCatalog, DbOut.vgroupList)); - } - - if (DbOut.dbVgroup) { - CTG_ERR_JRET(catalogUpdateDBVgroupCache(pCatalog, dbName, DbOut.dbVgroup)); - } + CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup)); if (dbInfo) { *dbInfo = DbOut.dbVgroup; - DbOut.dbVgroup = NULL; } -_return: - tfree(DbOut.dbVgroup); - tfree(DbOut.vgroupList); - return code; } @@ -749,16 +579,20 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgroupInfo dbVgroup = {0}; CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); - if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); + CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); - CTG_ERR_JRET(ctgGetVgroupFromVgIdBatch(pCatalog, pRpc, pMgmtEps, dbVgroup->vgId, pVgroupList)); + if (tbMeta->tableType == TSDB_SUPER_TABLE) { + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList)); } else { - CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, tbMeta->vgId, &vgroupInfo)); + int32_t vgId = tbMeta->vgId; + if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { ctgError("push vgroupInfo to array failed"); @@ -768,12 +602,6 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe _return: tfree(tbMeta); - if (dbVgroup && dbVgroup->vgId) { - taosArrayDestroy(dbVgroup->vgId); - dbVgroup->vgId = NULL; - } - - tfree(dbVgroup); return code; } diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt new file mode 100644 index 0000000000..527156f176 --- /dev/null +++ b/source/libs/catalog/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build catalog unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + catalogTest + PUBLIC os util common catalog transport gtest query +) + +TARGET_INCLUDE_DIRECTORIES( + catalogTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc" +) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e69de29bb2..f495451091 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" + +namespace { + + +} + +TEST(testCase, normalCase) { + char *clusterId = "cluster1"; + struct SCatalog* pCtg = NULL; + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(clusterId, &pCtg); + ASSERT_EQ(code, 0); + + +} + +/* +TEST(testCase, normalCase) { + SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SCatalogReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + + int32_t num = tsCompatibleModel? 2:1; + ASSERT_EQ(taosArrayGetSize(pExprList), num); + + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 1); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.interBytes, 16); + + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STREQ(p1->pExpr->_function.functionName, "top"); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + + ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); + + struct SQueryPlanNode* n = nullptr; + code = createQueryPlan(pQueryInfo, &n); + + char* str = NULL; + queryPlanToString(n, &str); + printf("%s\n", str); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, displayPlan) { + generateLogicplan("select count(*) from `t.1abc`"); + generateLogicplan("select count(*)+ 22 from `t.1abc`"); + generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30"); + generateLogicplan("select count(*) from `t.1abc` group by a"); + generateLogicplan("select count(A+B) from `t.1abc` group by a"); + generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); + generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); + generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s) order by 1 desc "); + generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); + generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); + generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); + generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20"); + generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); + generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); + + // order by + group by column + limit offset + generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1"); + + // fill + generateLogicplan("select min(a) from `t.1abc` where ts>now and tsdb, bInput->db, sizeof(bMsg->db)); bMsg->db[sizeof(bMsg->db) - 1] = 0; - bMsg->vgroupVersion = bInput->vgroupVersion; - bMsg->dbGroupVersion = bInput->dbGroupVersion; + bMsg->vgVersion = bInput->vgVersion; *msgLen = (int32_t)sizeof(*bMsg); @@ -90,58 +78,12 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms } - -int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { - if (NULL == output || NULL == msg || msgSize <= 0) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - - SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg; - - pRsp->vgroupNum = htonl(pRsp->vgroupNum); - pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); - - if (pRsp->vgroupNum < 0) { - qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - if (pRsp->vgroupVersion < 0) { - qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { - qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - // keep SVgroupListInfo/SVgroupListRspMsg the same - *(SVgroupListInfo **)output = (SVgroupListInfo *)msg; - - if (pRsp->vgroupNum == 0) { - return TSDB_CODE_SUCCESS; - } - - for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { - pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); - } - } - - return TSDB_CODE_SUCCESS; -} - - - - int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { return TSDB_CODE_TSC_INVALID_INPUT; } - SUseDbRspMsg *pRsp = (SUseDbRspMsg *)msg; + SUseDbRsp *pRsp = (SUseDbRsp *)msg; SUseDbOutput *pOut = (SUseDbOutput *)output; int32_t code = 0; @@ -150,104 +92,52 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); - pRsp->dbVgroupVersion = htonl(pRsp->dbVgroupVersion); + pRsp->vgVersion = htonl(pRsp->vgVersion); + pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->vgroupNum = htonl(pRsp->vgroupNum); - pRsp->dbVgroupNum = htonl(pRsp->dbVgroupNum); - - if (pRsp->vgroupNum < 0) { - qError("invalid vgroup number[%d]", pRsp->vgroupNum); + if (pRsp->vgNum < 0) { + qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); return TSDB_CODE_TSC_INVALID_VALUE; } - if (pRsp->dbVgroupNum < 0) { - qError("invalid db vgroup number[%d]", pRsp->dbVgroupNum); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t expectSize = pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + pRsp->dbVgroupNum * sizeof(int32_t) + sizeof(*pRsp); + int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp); if (msgSize != expectSize) { - qError("vgroup list msg size mis-match, msgSize:%d, expected:%d, vgroup number:%d, db vgroup number:%d", msgSize, expectSize, pRsp->vgroupNum, pRsp->dbVgroupNum); + qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - if (pRsp->vgroupVersion < 0) { - qInfo("no new vgroup list info"); - if (pRsp->vgroupNum != 0) { - qError("invalid vgroup number[%d] for no new vgroup list case", pRsp->vgroupNum); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } else { - int32_t s = sizeof(*pOut->vgroupList) + sizeof(pOut->vgroupList->vgroupInfo[0]) * pRsp->vgroupNum; - pOut->vgroupList = calloc(1, s); - if (NULL == pOut->vgroupList) { - qError("calloc size[%d] failed", s); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pOut->vgroupList->vgroupNum = pRsp->vgroupNum; - pOut->vgroupList->vgroupVersion = pRsp->vgroupVersion; - - for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { - pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); - } - - memcpy(&pOut->vgroupList->vgroupInfo[i], &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i])); - } + pOut->dbVgroup.vgVersion = pRsp->vgVersion; + pOut->dbVgroup.hashMethod = pRsp->hashMethod; + pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup.vgInfo) { + qError("hash init[%d] failed", pRsp->vgNum); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int32_t *vgIdList = (int32_t *)((char *)pRsp->vgroupInfo + sizeof(pRsp->vgroupInfo[0]) * pRsp->vgroupNum); + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + pRsp->vgroupInfo[i].hashBegin = htonl(pRsp->vgroupInfo[i].hashBegin); + pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd); + + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + } + + if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + qError("hash push failed"); + goto _return; + } + } memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); - - if (pRsp->dbVgroupVersion < 0) { - qInfo("no new vgroup info for db[%s]", pRsp->db); - } else { - pOut->dbVgroup = calloc(1, sizeof(*pOut->dbVgroup)); - if (NULL == pOut->dbVgroup) { - qError("calloc size[%d] failed", (int32_t)sizeof(*pOut->dbVgroup)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _exit; - } - - pOut->dbVgroup->vgId = taosArrayInit(pRsp->dbVgroupNum, sizeof(int32_t)); - if (NULL == pOut->dbVgroup->vgId) { - qError("taosArrayInit size[%d] failed", pRsp->dbVgroupNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _exit; - } - - pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion; - pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange); - pOut->dbVgroup->hashType = htonl(pRsp->dbHashType); - - if (pOut->dbVgroup->hashRange < 0) { - qError("invalid hashRange[%d] for db[%s]", pOut->dbVgroup->hashRange, pRsp->db); - code = TSDB_CODE_TSC_INVALID_INPUT; - goto _exit; - } - - for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) { - *(vgIdList + i) = htonl(*(vgIdList + i)); - - taosArrayPush(pOut->dbVgroup->vgId, vgIdList + i) ; - } - } return code; -_exit: - if (pOut->dbVgroup && pOut->dbVgroup->vgId) { - taosArrayDestroy(pOut->dbVgroup->vgId); - pOut->dbVgroup->vgId = NULL; +_return: + if (pOut) { + tfree(pOut->dbVgroup.vgInfo); } - tfree(pOut->dbVgroup); - tfree(pOut->vgroupList); - return code; } @@ -375,11 +265,9 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { void msgInit() { queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; - queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; /*