From 762086c22be6ef2996977c8d05e6e954b2a438c6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 26 Mar 2022 18:10:26 +0800 Subject: [PATCH 01/13] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 17 +- source/libs/catalog/src/catalog.c | 187 +----------------- source/libs/catalog/src/catalogDbg.c | 222 ++++++++++++++++++++++ source/libs/catalog/test/catalogTests.cpp | 82 ++++---- 4 files changed, 284 insertions(+), 224 deletions(-) create mode 100644 source/libs/catalog/src/catalogDbg.c diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9bd5c1c016..a5887e8807 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -112,7 +112,13 @@ typedef struct SCtgRuntimeStat { } SCtgRuntimeStat; typedef struct SCtgCacheStat { - + uint64_t clusterNum; + uint64_t dbNum; + uint64_t tblNum; + uint64_t vgHitNum; + uint64_t vgMissNum; + uint64_t tblHitNum; + uint64_t tblMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -186,7 +192,7 @@ typedef struct SCatalogMgmt { bool exit; SRWLatch lock; SCtgQueue queue; - TdThread updateThread; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; @@ -206,6 +212,10 @@ typedef struct SCtgAction { #define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) #define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) +#define CTG_STAT_GET(n) atomic_load_64(&(n)) + +#define CTG_RUNTIME_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.runtime._item)) +#define CTG_CACHE_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.cache._item)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -291,6 +301,9 @@ typedef struct SCtgAction { #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +extern void ctgdShowClusterCache(SCatalog* pCtg); +extern int32_t ctgdShowCacheInfo(void); #ifdef __cplusplus } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6eab0d280b..73b363ec5c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; -SCtgDebug gCTGDebug = {0}; SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_UPDATE_VG, "update vgInfo", @@ -53,181 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ } }; -int32_t ctgDbgEnableDebug(char *option) { - if (0 == strcasecmp(option, "lock")) { - gCTGDebug.lockEnable = true; - qDebug("lock debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "cache")) { - gCTGDebug.cacheEnable = true; - qDebug("cache debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "api")) { - gCTGDebug.apiEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "meta")) { - gCTGDebug.metaEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - qError("invalid debug option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetStatNum(char *option, void *res) { - if (0 == strcasecmp(option, "runtime.qDoneNum")) { - *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); - return TSDB_CODE_SUCCESS; - } - - qError("invalid stat option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; -} - -int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; -} - -int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { - int32_t num = 0; - for (uint16_t i = 0; i < rent->slotNum; ++i) { - SCtgRentSlot *slot = &rent->slots[i]; - if (NULL == slot->meta) { - continue; - } - - num += taosArrayGetSize(slot->meta); - } - - return num; -} - -int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { - if (NULL == pCtg || NULL == pCtg->dbCache) { - return 0; - } - - switch (type) { - case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCtg->dbCache); - case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->dbRent); - case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->stbRent); - default: - break; - } - - SCtgDBCache *dbCache = NULL; - int32_t num = 0; - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - dbCache = (SCtgDBCache *)pIter; - switch (type) { - case CTG_DBG_META_NUM: - num += ctgDbgGetTbMetaNum(dbCache); - break; - case CTG_DBG_STB_NUM: - num += ctgDbgGetStbNum(dbCache); - break; - default: - ctgError("invalid type:%d", type); - break; - } - pIter = taosHashIterate(pCtg->dbCache, pIter); - } - - return num; -} - -void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { - if (!gCTGDebug.metaEnable) { - return; - } - - STableComInfo *c = &p->tableInfo; - - if (TSDB_CHILD_TABLE == p->tableType) { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); - return; - } else { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", - tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); - } - - int32_t colNum = c->numOfColumns + c->numOfTags; - for (int32_t i = 0; i < colNum; ++i) { - SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); - } -} - -void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { - if (NULL == dbHash || !gCTGDebug.cacheEnable) { - return; - } - - int32_t i = 0; - SCtgDBCache *dbCache = NULL; - void *pIter = taosHashIterate(dbHash, NULL); - while (pIter) { - char *dbFName = NULL; - size_t len = 0; - - dbCache = (SCtgDBCache *)pIter; - - dbFName = taosHashGetKey(pIter, &len); - - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; - int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; - int32_t hashMethod = -1; - int32_t vgNum = 0; - - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); - } - } - - ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); - - pIter = taosHashIterate(dbHash, pIter); - } -} - - - - -void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (!gCTGDebug.cacheEnable || NULL == pCtg) { - return; - } - - ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - - ctgDbgShowDBCache(pCtg, pCtg->dbCache); -} - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { @@ -361,7 +186,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + CTG_RUNTIME_STAT_ADD(qNum); tsem_post(&gCtgMgmt.queue.reqSem); @@ -1595,7 +1420,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - ctgDbgShowTableMeta(pCtg, tbName, meta); + ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); @@ -1962,7 +1787,7 @@ _return: if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); - ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta); + ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta); } CTG_RET(code); @@ -2161,9 +1986,9 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + CTG_RUNTIME_STAT_ADD(qDoneNum); - ctgDbgShowClusterCache(pCtg); + ctgdShowClusterCache(pCtg); } CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/catalogDbg.c b/source/libs/catalog/src/catalogDbg.c new file mode 100644 index 0000000000..1d4ad0082c --- /dev/null +++ b/source/libs/catalog/src/catalogDbg.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" + +extern SCatalogMgmt gCtgMgmt; +SCtgDebug gCTGDebug = {0}; + +int32_t ctgdEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockEnable = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheEnable = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "meta")) { + gCTGDebug.metaEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; +} + +int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; +} + +int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { + int32_t num = 0; + for (uint16_t i = 0; i < rent->slotNum; ++i) { + SCtgRentSlot *slot = &rent->slots[i]; + if (NULL == slot->meta) { + continue; + } + + num += taosArrayGetSize(slot->meta); + } + + return num; +} + +int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { + return 0; + } + + switch (type) { + case CTG_DBG_DB_NUM: + return (int32_t)taosHashGetSize(pCtg->dbCache); + case CTG_DBG_DB_RENT_NUM: + return ctgdGetRentNum(&pCtg->dbRent); + case CTG_DBG_STB_RENT_NUM: + return ctgdGetRentNum(&pCtg->stbRent); + default: + break; + } + + SCtgDBCache *dbCache = NULL; + int32_t num = 0; + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + dbCache = (SCtgDBCache *)pIter; + switch (type) { + case CTG_DBG_META_NUM: + num += ctgdGetTbMetaNum(dbCache); + break; + case CTG_DBG_STB_NUM: + num += ctgdGetStbNum(dbCache); + break; + default: + ctgError("invalid type:%d", type); + break; + } + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + return num; +} + +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { + if (!gCTGDebug.metaEnable) { + return; + } + + STableComInfo *c = &p->tableInfo; + + if (TSDB_CHILD_TABLE == p->tableType) { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + return; + } else { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); + } + + int32_t colNum = c->numOfColumns + c->numOfTags; + for (int32_t i = 0; i < colNum; ++i) { + SSchema *s = &p->schema[i]; + ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + } +} + +void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { + if (NULL == dbHash || !gCTGDebug.cacheEnable) { + return; + } + + int32_t i = 0; + SCtgDBCache *dbCache = NULL; + void *pIter = taosHashIterate(dbHash, NULL); + while (pIter) { + char *dbFName = NULL; + size_t len = 0; + + dbCache = (SCtgDBCache *)pIter; + + dbFName = taosHashGetKey(pIter, &len); + + int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; + int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; + int32_t hashMethod = -1; + int32_t vgNum = 0; + + if (dbCache->vgInfo) { + vgVersion = dbCache->vgInfo->vgVersion; + hashMethod = dbCache->vgInfo->hashMethod; + if (dbCache->vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + } + } + + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + + pIter = taosHashIterate(dbHash, pIter); + } +} + + + + +void ctgdShowClusterCache(SCatalog* pCtg) { + if (!gCTGDebug.cacheEnable || NULL == pCtg) { + return; + } + + ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg); + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); + + ctgdShowDBCache(pCtg, pCtg->dbCache); + + ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg); +} + +int32_t ctgdShowCacheInfo(void) { + if (!gCTGDebug.cacheEnable) { + return TSDB_CODE_CTG_OUT_OF_SERVICE; + } + + CTG_API_ENTER(); + + SCatalog *pCtg = NULL; + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + ctgdShowClusterCache(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e62819b078..27d08f6428 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -39,10 +39,10 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId); -extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); -extern "C" int32_t ctgDbgEnableDebug(char *option); -extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); +extern "C" int32_t ctgdEnableDebug(char *option); +extern "C" int32_t ctgdGetStatNum(char *option, void *res); void ctgTestSetRspTableMeta(); void ctgTestSetRspCTableMeta(); @@ -140,9 +140,9 @@ void ctgTestInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, "/var/log/taos"); - ctgDbgEnableDebug("api"); - ctgDbgEnableDebug("meta"); - ctgDbgEnableDebug("cache"); + ctgdEnableDebug("api"); + ctgdEnableDebug("meta"); + ctgdEnableDebug("cache"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epSet.numOfEps, 3); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { taosMsleep(50); } @@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { taosMsleep(50); } else { @@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(code, 0); while (true) { - int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); - int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { taosMsleep(50); } else { @@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) { } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) { } } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { taosMsleep(50); } From 9e3f03ad745602fef4f0f07e6636a05d2225e5cd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Mar 2022 13:49:21 +0800 Subject: [PATCH 02/13] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 12 +- source/libs/catalog/src/catalog.c | 145 ++++++++++++++-------- source/libs/catalog/test/catalogTests.cpp | 8 +- 3 files changed, 105 insertions(+), 60 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a5887e8807..09f51dc03e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -115,6 +115,7 @@ typedef struct SCtgCacheStat { uint64_t clusterNum; uint64_t dbNum; uint64_t tblNum; + uint64_t stblNum; uint64_t vgHitNum; uint64_t vgMissNum; uint64_t tblHitNum; @@ -210,12 +211,13 @@ typedef struct SCtgAction { #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) -#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) -#define CTG_STAT_GET(n) atomic_load_64(&(n)) +#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_RUNTIME_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.runtime._item)) -#define CTG_CACHE_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.cache._item)) +#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n)) +#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n)) +#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 73b363ec5c..04f93c31f3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -74,15 +74,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { + int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; + CTG_CACHE_STAT_SUB(stblNum, stblNum); } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); CTG_LOCK(CTG_WRITE, &cache->metaLock); if (cache->metaCache) { + int32_t tblNum = taosHashGetSize(cache->metaCache); taosHashCleanup(cache->metaCache); cache->metaCache = NULL; + CTG_CACHE_STAT_SUB(tblNum, tblNum); } CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } @@ -118,6 +122,8 @@ void ctgFreeHandle(SCatalog* pCtg) { ctgFreeMetaRent(&pCtg->stbRent); if (pCtg->dbCache) { + int32_t dbNum = taosHashGetSize(pCtg->dbCache); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; @@ -130,6 +136,8 @@ void ctgFreeHandle(SCatalog* pCtg) { } taosHashCleanup(pCtg->dbCache); + + CTG_CACHE_STAT_SUB(dbNum, dbNum); } taosMemoryFree(pCtg); @@ -186,7 +194,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); - CTG_RUNTIME_STAT_ADD(qNum); + CTG_RUNTIME_STAT_ADD(qNum, 1); tsem_post(&gCtgMgmt.queue.reqSem); @@ -445,34 +453,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + SCtgDBCache *dbCache = NULL; + if (NULL == pCtg->dbCache) { - *pCache = NULL; - *inCache = false; - ctgWarn("empty db cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + ctgDebug("empty db cache, dbFName:%s", dbFName); + goto _return; } - SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *pCache = NULL; - *inCache = false; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; } ctgAcquireVgInfo(pCtg, dbCache, inCache); if (!(*inCache)) { - ctgReleaseDBCache(pCtg, dbCache); - - *pCache = NULL; - return TSDB_CODE_SUCCESS; + ctgDebug("vgInfo of db %s not in cache", dbFName); + goto _return; } *pCache = dbCache; *inCache = true; + CTG_CACHE_STAT_ADD(vgHitNum, 1); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + +_return: + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + *inCache = false; + + CTG_CACHE_STAT_ADD(vgMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -588,11 +607,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + goto _return; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; @@ -607,8 +625,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", pTableName->tname); + goto _return; } int32_t sz = 0; @@ -617,13 +635,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { - *exist = 0; ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - return TSDB_CODE_SUCCESS; + goto _return; } - *exist = 1; if (dbId) { *dbId = dbCache->dbId; } @@ -633,6 +649,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + return TSDB_CODE_SUCCESS; } @@ -644,8 +664,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); taosMemoryFreeClear(*pTableMeta); - *exist = 0; - return TSDB_CODE_SUCCESS; + goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { @@ -671,8 +690,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(tblMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -1202,6 +1231,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(dbNum, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); @@ -1261,6 +1292,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + CTG_CACHE_STAT_SUB(dbNum, 1); + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1393,6 +1426,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_LOCK(CTG_WRITE, &tbCache->stbLock); if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1419,6 +1454,10 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + if (NULL == orig) { + CTG_CACHE_STAT_ADD(tblNum, 1); + } + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); ctgdShowTableMeta(pCtg, tbName, meta); @@ -1440,6 +1479,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(stblNum, 1); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1699,7 +1740,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t exist = 0; + bool inCache = false; int32_t code = 0; uint64_t dbId = 0; uint64_t suid = 0; @@ -1709,11 +1750,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_FLAG_SET_INF_DB(flag); } - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId)); int32_t tbType = 0; - if (exist) { + if (inCache) { if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) { goto _return; } @@ -1755,8 +1796,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL)); - if (0 == exist) { + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); + if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; } @@ -1768,7 +1809,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons _return: - if (CTG_TABLE_NOT_EXIST(code) && exist) { + if (CTG_TABLE_NOT_EXIST(code) && inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); @@ -1900,12 +1941,16 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - } + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); @@ -1944,6 +1989,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -1986,7 +2033,7 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_RUNTIME_STAT_ADD(qDoneNum); + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); ctgdShowClusterCache(pCtg); } @@ -2208,6 +2255,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { } *catalogHandle = clusterCtg; + + CTG_CACHE_STAT_ADD(clusterNum, 1); return TSDB_CODE_SUCCESS; @@ -2228,10 +2277,12 @@ void catalogFreeHandle(SCatalog* pCtg) { return; } + CTG_CACHE_STAT_SUB(clusterNum, 1); + uint64_t clusterId = pCtg->clusterId; ctgFreeHandle(pCtg); - + ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } @@ -2242,24 +2293,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCtg->dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); - if (!inCache) { - ctgReleaseDBCache(pCtg, dbCache); + int32_t code = 0; + CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + if (!inCache) { *version = CTG_DEFAULT_INVALID_VERSION; CTG_API_LEAVE(TSDB_CODE_SUCCESS); } @@ -2274,6 +2313,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); + +_return: + + CTG_API_LEAVE(code); } int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { @@ -2374,11 +2417,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { } STableMeta *tblMeta = NULL; - int32_t exist = 0; + bool inCache = false; uint64_t dbId = 0; - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); - if (0 == exist) { + if (!inCache) { ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); goto _return; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 27d08f6428..73d0dc2011 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist, int32_t flag, uint64_t *dbId); + bool *inCache, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgdEnableDebug(char *option); @@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) { int32_t code = 0; int32_t n = 0; STableMeta *tbMeta = NULL; - int32_t exist = 0; + bool inCache = false; SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL); - if (code || 0 == exist) { + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); + if (code || !inCache) { assert(0); } From f867538446cd02cf60417efe06cea70d8e21eccf Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Mar 2022 19:48:47 +0800 Subject: [PATCH 03/13] feature/qnode --- include/libs/qcom/query.h | 6 + source/libs/qcom/inc/queryInt.h | 19 +++ source/libs/qcom/src/queryExplain.c | 199 ++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 source/libs/qcom/src/queryExplain.c diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f487e5ae22..c3870665a6 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,6 +53,12 @@ typedef struct SIndexMeta { } SIndexMeta; +typedef struct SPhysiNodeExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SPhysiNodeExplainResNode; + /* * ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(tableType == TSDB_CHILD_TABLE) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 75c1e134cd..86a1e04ae5 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -20,6 +20,25 @@ extern "C" { #endif +#define QUERY_EXPLAIN_MAX_RES_LEN 1024 + +#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s" +#define EXPLAIN_ORDER_FORMAT "Order: " + +typedef struct SQueryExplainRowInfo { + int32_t level; + int32_t len; + char *buf; +} SQueryExplainRowInfo; + + +#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) + + +#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c new file mode 100644 index 0000000000..d547e67170 --- /dev/null +++ b/source/libs/qcom/src/queryExplain.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "queryInt.h" +#include "query.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-truncation" + +void qFreeExplainRes(SPhysiNodeExplainRes *res) { + +} + +int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { + +} + +int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SPhysiNodeExplainResNode **pRes) { + if (NULL == pNode) { + *pRes = NULL; + qError("physical node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + SPhysiNodeExplainResNode *res = calloc(1, sizeof(SPhysiNodeExplainResNode)); + if (NULL == res) { + qError("calloc SPhysiNodeExplainRes failed"); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + int32_t code = 0; + res->pNode = pNode; + res->pExecInfo = pExecInfo; + QRY_ERR_JRET(qMakeExplainResChildrenInfo(pNode, pExecInfo, &res->pChildren)); + + *pRes = res; + + return TSDB_CODE_SUCCESS; + +_return: + + qFreeExplainRes(res); + + QRY_RET(code); +} + +int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNodeExplainResNode **pRes) { + char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + if (NULL == tbuf) { + qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *pExecInfo = NULL; // TODO + int32_t code = qMakeExplainResNode(plan->pNode, pExecInfo, pRes); + + taosMemoryFree(tbuf); + + QRY_RET(code); +} + +int32_t qExplainResNodeAppendExecInfo(void *pExecInfo, char *tbuf) { + +} + +int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t level) { + SQueryExplainRowInfo row = {0}; + row.buf = strdup(tbuf); + if (NULL == row.buf) { + qError("strdup %s failed", tbuf); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + row.level = level; + row.len = len; + + if (taosArrayPush(pRows, &row)) { + qError("taosArrayPush row to explain res rows failed"); + taosMemoryFree(row.buf); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { + int32_t tlen = 0; + SPhysiNode* pNode = pResNode->pNode; + if (NULL == pNode) { + qError("pyhsical node in explain res node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainResNodeAppendExecInfo(pResNode->pExecInfo, tbuf)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + case QUERY_NODE_PHYSICAL_PLAN_AGG: + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + case QUERY_NODE_PHYSICAL_PLAN_INSERT: + default: + qError("not supported physical node type %d", pNode->type); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { + if (NULL == pResNode) { + qError("explain res node is NULL"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t code = 0; + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, pRsp, tbuf, level)); + + SNode* pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { + QRY_ERR_RET(qExplainResNodeToRows((SPhysiNodeExplainResNode *)pNode, pRsp, tbuf, level + 1)); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { + if (NULL == pResNode) { + qError("explain res node is NULL"); + QRY_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t code = 0; + char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + if (NULL == tbuf) { + qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); + + int32_t rspSize = sizeof(SRetrieveTableRsp) + ; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qExplainRowsToRsp(rows, rsp)); + + *pRsp = rsp; + rsp = NULL; + +_return: + + taosMemoryFree(tbuf); + taosMemoryFree(rsp); + taosArrayDestroy(rows); + + QRY_RET(code); +} + + +#pragma GCC diagnostic pop From d1102c5f96a2d69fd759627d6790c6fbbf62733f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 29 Mar 2022 08:55:22 +0800 Subject: [PATCH 04/13] feature/qnode --- source/libs/qcom/inc/queryInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 86a1e04ae5..2de0c89a13 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -32,7 +32,7 @@ typedef struct SQueryExplainRowInfo { } SQueryExplainRowInfo; -#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, _format, __VA_ARGS__) #define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) From c8e132052a02fc787205a76f67273b469cea7d3d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 29 Mar 2022 19:51:11 +0800 Subject: [PATCH 05/13] feature/qnode --- include/libs/qcom/query.h | 4 +- source/libs/nodes/src/nodesToSQLFuncs.c | 134 +++++++++ source/libs/nodes/src/nodesTraverseFuncs.c | 3 +- source/libs/nodes/src/nodesUtilFuncs.c | 63 ++++ source/libs/qcom/inc/queryInt.h | 26 +- source/libs/qcom/src/queryExplain.c | 321 ++++++++++++++++++--- source/libs/scheduler/src/scheduler.c | 15 +- 7 files changed, 519 insertions(+), 47 deletions(-) create mode 100644 source/libs/nodes/src/nodesToSQLFuncs.c diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c3870665a6..c0fc8c630e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,11 +53,11 @@ typedef struct SIndexMeta { } SIndexMeta; -typedef struct SPhysiNodeExplainResNode { +typedef struct SExplainResNode { SNodeList* pChildren; SPhysiNode* pNode; void* pExecInfo; -} SPhysiNodeExplainResNode; +} SExplainResNode; /* * ASSERT(sizeof(SCTableMeta) == 24) diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c new file mode 100644 index 0000000000..6e4cd0f04e --- /dev/null +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "cmdnodes.h" +#include "nodesUtil.h" +#include "plannodes.h" +#include "querynodes.h" +#include "taos.h" +#include "taoserror.h" +#include "thash.h" + +char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>", + "IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL", + "IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"}; +char *gLogicConditionStr[] = {"AND", "OR", "NOT"}; + +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { + switch (pNode->type) { + case QUERY_NODE_COLUMN: { + SColumnNode *colNode = (SColumnNode *)pNode; + *len = 0; + if (colNode->dbName[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->dbName); + } + + if (colNode->tableAlias[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableAlias); + } else if (colNode->tableName[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableName); + } + + *len += snprintf(buf, bufSize - *len, "`%s`", colNode->colName); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_VALUE:{ + SValueNode *colNode = (SValueNode *)pNode; + char *t = nodesGetStrValueFromNode(colNode); + if (NULL == t) { + nodesError("fail to get str value from valueNode"); + return TSDB_CODE_QRY_APP_ERROR; + } + + *len += snprintf(buf, bufSize - *len, "%s", t); + taosMemoryFree(t); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + *len += snprintf(buf, bufSize - *len, "("); + if (pOpNode->pLeft) { + QRY_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); + } + + if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { + nodesError("unknown operation type:%d", pOpNode->opType); + return TSDB_CODE_QRY_APP_ERROR; + } + + *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + + if (pOpNode->pRight) { + QRY_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_LOGIC_CONDITION:{ + SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "("); + + FOREACH(node, pLogicNode->pParameterList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + } + QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_FUNCTION:{ + SFunctionNode* pFuncNode = (SFunctionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "%s(", pFuncNode->functionName); + + FOREACH(node, pFuncNode->pParameterList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, ", "); + } + QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_NODE_LIST:{ + SNodeListNode* pListNode = (SNodeListNode *)pNode; + + //TODO + + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); + return TSDB_CODE_QRY_APP_ERROR; +} diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 7eaa049946..bbd0473edd 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -17,7 +17,8 @@ typedef enum ETraversalOrder { TRAVERSAL_PREORDER = 1, - TRAVERSAL_POSTORDER + TRAVERSAL_INORDER, + TRAVERSAL_POSTORDER, } ETraversalOrder; static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 035a2f1caa..7b2be5e355 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -467,6 +467,69 @@ void* nodesGetValueFromNode(SValueNode *pNode) { return NULL; } +char* nodesGetStrValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); + return buf; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRId64, pNode->datum.i); + return buf; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRIu64, pNode->datum.u); + return buf; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%e", pNode->datum.d); + return buf; + } + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + void *buf = taosMemoryMalloc(varDataLen(pNode->datum.p) + 1); + if (NULL == buf) { + return NULL; + } + + strncpy(buf, varDataVal(pNode->datum.p), varDataLen(pNode->datum.p) + 1); + return buf; + } + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 2de0c89a13..7b6b7d44ef 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -22,8 +22,24 @@ extern "C" { #define QUERY_EXPLAIN_MAX_RES_LEN 1024 -#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s" -#define EXPLAIN_ORDER_FORMAT "Order: " +#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" +#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" +#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%d%c offset=%d%c sliding=%d%c width=%d" +#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" + +#define EXPLAIN_ORDER_FORMAT "Order: %s" +#define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_ON_CONDITIONS_FORMAT "ON Conditions: " +#define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" +#define EXPLAIN_LOOPS_FORMAT "loops %d" +#define EXPLAIN_REVERSE_FORMAT "reverse %d" typedef struct SQueryExplainRowInfo { int32_t level; @@ -31,9 +47,11 @@ typedef struct SQueryExplainRowInfo { char *buf; } SQueryExplainRowInfo; +#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") +#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, _format, __VA_ARGS__) -#define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_NEWLINE(...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, __VA_ARGS__) +#define QUERY_EXPLAIN_APPEND(...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d547e67170..d6da9ab0e2 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -19,22 +19,124 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation" -void qFreeExplainRes(SPhysiNodeExplainRes *res) { +void qFreeExplainRes(SExplainResNode *res) { } +char *qFillModeString(EFillMode mode) { + switch (mode) { + case FILL_MODE_NONE: + return "none"; + case FILL_MODE_VALUE: + return "value"; + case FILL_MODE_PREV: + return "prev"; + case FILL_MODE_NULL: + return "null"; + case FILL_MODE_LINEAR: + return "linear"; + case FILL_MODE_NEXT: + return "next"; + default: + return "unknown"; + } +} + +char *qGetNameFromColumnNode(SNode *pNode) { + if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) { + return "NULL"; + } + + return ((SColumnNode *)pNode)->colName; +} + int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { + int32_t tlen = 0; + SNodeList *pPhysiChildren = NULL; + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + pPhysiChildren = pTagScanNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + pPhysiChildren = pTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + pPhysiChildren = pSTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + pPhysiChildren = pPrjNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + pPhysiChildren = pJoinNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + pPhysiChildren = pAggNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + pPhysiChildren = pExchNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + pPhysiChildren = pSortNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + pPhysiChildren = pIntNode->window.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; + pPhysiChildren = pSessNode->window.node.pChildren; + break; + } + default: + qError("not supported physical node type %d", pNode->type); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + if (pPhysiChildren) { + *pChildren = nodesMakeList(); + if (NULL == *pChildren) { + qError("nodesMakeList failed"); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + SNode* node = NULL; + SExplainResNode *pResNode = NULL; + FOREACH(node, pPhysiChildren) { + QRY_ERR_RET(qMakeExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); + } + + return TSDB_CODE_SUCCESS; } -int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SPhysiNodeExplainResNode **pRes) { +int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { if (NULL == pNode) { *pRes = NULL; qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - SPhysiNodeExplainResNode *res = calloc(1, sizeof(SPhysiNodeExplainResNode)); + SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == res) { qError("calloc SPhysiNodeExplainRes failed"); return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -56,7 +158,7 @@ _return: QRY_RET(code); } -int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNodeExplainResNode **pRes) { +int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); if (NULL == tbuf) { qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); @@ -71,7 +173,7 @@ int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNo QRY_RET(code); } -int32_t qExplainResNodeAppendExecInfo(void *pExecInfo, char *tbuf) { +int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t tlen) { } @@ -96,7 +198,7 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev } -int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { int32_t tlen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { @@ -107,25 +209,177 @@ int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pR switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainResNodeAppendExecInfo(pResNode->pExecInfo, tbuf)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); + if (pTagScanNode->reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); } QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - case QUERY_NODE_PHYSICAL_PLAN_JOIN: - case QUERY_NODE_PHYSICAL_PLAN_AGG: - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - case QUERY_NODE_PHYSICAL_PLAN_INSERT: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); + if (pTblScanNode->scan.reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + + if (pTblScanNode->pScanConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); + if (pSTblScanNode->scan.reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pPrjNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pJoinNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pAggNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pExchNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pSortNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, + pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pIntNode->pFill) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + + if (pIntNode->window.node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pIntNode->window.node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_QRY_APP_ERROR; @@ -135,7 +389,7 @@ int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pR } -int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -146,14 +400,22 @@ int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SPhysiNodeExplainResNode *)pNode, pRsp, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, pRsp, tbuf, level + 1)); } return TSDB_CODE_SUCCESS; } +int32_t qExplainRowsToRsp(SArray *rows, SRetrieveTableRsp **pRsp) { + int32_t rspSize = sizeof(SRetrieveTableRsp) + ; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } +} -int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { +int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_RET(TSDB_CODE_QRY_APP_ERROR); @@ -174,22 +436,11 @@ int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTab QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); - int32_t rspSize = sizeof(SRetrieveTableRsp) + ; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); - if (NULL == rsp) { - qError("malloc SRetrieveTableRsp failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - QRY_ERR_JRET(qExplainRowsToRsp(rows, rsp)); - - *pRsp = rsp; - rsp = NULL; + QRY_ERR_JRET(qExplainRowsToRsp(rows, pRsp)); _return: taosMemoryFree(tbuf); - taosMemoryFree(rsp); taosArrayDestroy(rows); QRY_RET(code); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5af13d97ca..fdb0b6498c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2126,19 +2126,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD tsem_init(&pJob->rspSem, 0, 0); - pJob->refId = taosAddRef(schMgmt.jobRef, pJob); - if (pJob->refId < 0) { - SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno)); + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_ERR_JRET(terrno); } + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); - schAcquireJob(pJob->refId); - *job = pJob->refId; if (syncSchedule) { From 6982f2b7bda6660bdfc1d04b8ebd688b211a741f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:09:30 +0800 Subject: [PATCH 06/13] feature/qnode --- include/libs/nodes/nodes.h | 2 + include/libs/nodes/plannodes.h | 12 + include/libs/nodes/querynodes.h | 1 + include/libs/qcom/query.h | 9 +- include/util/tdef.h | 4 + source/libs/nodes/inc/nodesUtil.h | 16 +- source/libs/nodes/src/nodesToSQLFuncs.c | 30 +- source/libs/nodes/src/nodesUtilFuncs.c | 3 +- source/libs/qcom/CMakeLists.txt | 2 +- source/libs/qcom/inc/queryInt.h | 33 ++- source/libs/qcom/src/queryExplain.c | 335 +++++++++++++++-------- source/libs/scalar/inc/filterInt.h | 2 - source/libs/scheduler/inc/schedulerInt.h | 7 +- source/libs/scheduler/src/scheduler.c | 136 ++++++++- 14 files changed, 447 insertions(+), 145 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9b8739a4f3..031445ac79 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -210,6 +210,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 08792b6f8f..fb4def4440 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -283,11 +283,23 @@ typedef struct SSubplan { SDataSinkNode* pDataSink; // data of the subplan flow into the datasink } SSubplan; +typedef enum EExplainMode { + EXPLAIN_MODE_DISABLE = 1, + EXPLAIN_MODE_STATIC, + EXPLAIN_MODE_ANALYZE +} EExplainMode; + +typedef struct SExplainInfo { + EExplainMode mode; + bool verbose; +} SExplainInfo; + typedef struct SQueryPlan { ENodeType type; uint64_t queryId; int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. + SExplainInfo explainInfo; } SQueryPlan; #ifdef __cplusplus diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 66f60bde98..2f4f938652 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -300,6 +300,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); void* nodesGetValueFromNode(SValueNode *pNode); +char* nodesGetStrValueFromNode(SValueNode *pNode); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c0fc8c630e..2a9dd28819 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,11 +53,6 @@ typedef struct SIndexMeta { } SIndexMeta; -typedef struct SExplainResNode { - SNodeList* pChildren; - SPhysiNode* pNode; - void* pExecInfo; -} SExplainResNode; /* * ASSERT(sizeof(SCTableMeta) == 24) @@ -179,6 +174,10 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); +int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level); +int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); +void qFreeExplainRowCtx(void *ctx); + SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index 193be4a3e6..7db06bbafb 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -224,6 +224,7 @@ typedef enum ELogicConditionType { #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_STB_COMMENT_LEN 1024 + /** * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. @@ -471,6 +472,9 @@ enum { #define QND_VGID 1 #define VND_VGID 0 +#define MAX_NUM_STR_SIZE 40 + + #ifdef __cplusplus } #endif diff --git a/source/libs/nodes/inc/nodesUtil.h b/source/libs/nodes/inc/nodesUtil.h index de00b6bca4..976044c16f 100644 --- a/source/libs/nodes/inc/nodesUtil.h +++ b/source/libs/nodes/inc/nodesUtil.h @@ -20,12 +20,16 @@ extern "C" { #endif -#define nodesFatal(param, ...) qFatal("NODES: " param, __VA_ARGS__) -#define nodesError(param, ...) qError("NODES: " param, __VA_ARGS__) -#define nodesWarn(param, ...) qWarn("NODES: " param, __VA_ARGS__) -#define nodesInfo(param, ...) qInfo("NODES: " param, __VA_ARGS__) -#define nodesDebug(param, ...) qDebug("NODES: " param, __VA_ARGS__) -#define nodesTrace(param, ...) qTrace("NODES: " param, __VA_ARGS__) +#define nodesFatal(...) qFatal("NODES: " __VA_ARGS__) +#define nodesError(...) qError("NODES: " __VA_ARGS__) +#define nodesWarn(...) qWarn("NODES: " __VA_ARGS__) +#define nodesInfo(...) qInfo("NODES: " __VA_ARGS__) +#define nodesDebug(...) qDebug("NODES: " __VA_ARGS__) +#define nodesTrace(...) qTrace("NODES: " __VA_ARGS__) + +#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 6e4cd0f04e..fa941cfe9c 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -50,7 +50,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { char *t = nodesGetStrValueFromNode(colNode); if (NULL == t) { nodesError("fail to get str value from valueNode"); - return TSDB_CODE_QRY_APP_ERROR; + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *len += snprintf(buf, bufSize - *len, "%s", t); @@ -62,18 +62,18 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SOperatorNode* pOpNode = (SOperatorNode*)pNode; *len += snprintf(buf, bufSize - *len, "("); if (pOpNode->pLeft) { - QRY_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { nodesError("unknown operation type:%d", pOpNode->opType); - return TSDB_CODE_QRY_APP_ERROR; + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); if (pOpNode->pRight) { - QRY_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); } *len += snprintf(buf, bufSize - *len, ")"); @@ -91,7 +91,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { if (!first) { *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); } - QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } @@ -110,7 +110,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { if (!first) { *len += snprintf(buf, bufSize - *len, ", "); } - QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } @@ -120,8 +120,20 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } case QUERY_NODE_NODE_LIST:{ SNodeListNode* pListNode = (SNodeListNode *)pNode; - - //TODO + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "("); + + FOREACH(node, pListNode->pNodeList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, ", "); + } + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -130,5 +142,5 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); - return TSDB_CODE_QRY_APP_ERROR; + NODES_RET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7b2be5e355..2725ad85dd 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -469,7 +469,7 @@ void* nodesGetValueFromNode(SValueNode *pNode) { char* nodesGetStrValueFromNode(SValueNode *pNode) { switch (pNode->node.resType.type) { - case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BOOL: { void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); if (NULL == buf) { return NULL; @@ -477,6 +477,7 @@ char* nodesGetStrValueFromNode(SValueNode *pNode) { sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); return buf; + } case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index a9bf0f5594..d50047e592 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( qcom - PRIVATE os util transport + PRIVATE os util transport nodes ) if(${BUILD_TEST}) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 7b6b7d44ef..6fc15aacfc 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -19,9 +19,12 @@ #ifdef __cplusplus extern "C" { #endif +#include "nodes.h" +#include "plannodes.h" #define QUERY_EXPLAIN_MAX_RES_LEN 1024 +//newline area #define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" #define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" #define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" @@ -30,29 +33,47 @@ extern "C" { #define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" #define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" #define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%d%c offset=%d%c sliding=%d%c width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" #define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" - #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " #define EXPLAIN_FILL_FORMAT "Fill: %s" -#define EXPLAIN_ON_CONDITIONS_FORMAT "ON Conditions: " +#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" + +//append area #define EXPLAIN_LOOPS_FORMAT "loops %d" #define EXPLAIN_REVERSE_FORMAT "reverse %d" + +typedef struct SExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SExplainResNode; + typedef struct SQueryExplainRowInfo { int32_t level; int32_t len; char *buf; } SQueryExplainRowInfo; +typedef struct SExplainRowCtx { + int32_t totalSize; + SArray *rows; +} SExplainRowCtx; + #define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define QUERY_EXPLAIN_NEWLINE(...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, __VA_ARGS__) -#define QUERY_EXPLAIN_APPEND(...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) - +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, QUERY_EXPLAIN_MAX_RES_LEN, "%*s", level, ""); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__); \ + } while (0) + +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d6da9ab0e2..d352a29e3c 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -15,12 +15,41 @@ #include "queryInt.h" #include "query.h" +#include "plannodes.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-truncation" +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); -void qFreeExplainRes(SExplainResNode *res) { +void qFreeExplainResTree(SExplainResNode *res) { + if (NULL == res) { + return; + } + + taosMemoryFreeClear(res->pExecInfo); + + SNode* node = NULL; + FOREACH(node, res->pChildren) { + qFreeExplainResTree((SExplainResNode *)node); + } + nodesDestroyList(res->pChildren); + + taosMemoryFreeClear(res); +} + +void qFreeExplainRowCtx(void *ctx) { + if (NULL == ctx) { + return; + } + + SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + int32_t rowSize = taosArrayGetSize(pCtx->rows); + for (int32_t i = 0; i < rowSize; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + taosMemoryFreeClear(row->buf); + } + + taosArrayDestroy(pCtx->rows); + taosMemoryFree(pCtx); } char *qFillModeString(EFillMode mode) { @@ -50,7 +79,7 @@ char *qGetNameFromColumnNode(SNode *pNode) { return ((SColumnNode *)pNode)->colName; } -int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { +int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; @@ -122,14 +151,14 @@ int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeLis SNode* node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { - QRY_ERR_RET(qMakeExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); } return TSDB_CODE_SUCCESS; } -int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { if (NULL == pNode) { *pRes = NULL; qError("physical node is NULL"); @@ -145,7 +174,7 @@ int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode int32_t code = 0; res->pNode = pNode; res->pExecInfo = pExecInfo; - QRY_ERR_JRET(qMakeExplainResChildrenInfo(pNode, pExecInfo, &res->pChildren)); + QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren)); *pRes = res; @@ -153,31 +182,27 @@ int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode _return: - qFreeExplainRes(res); + qFreeExplainResTree(res); QRY_RET(code); } -int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { - char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); - if (NULL == tbuf) { - qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - +int32_t qGenerateExplainResNodeTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { void *pExecInfo = NULL; // TODO - int32_t code = qMakeExplainResNode(plan->pNode, pExecInfo, pRes); - - taosMemoryFree(tbuf); - - QRY_RET(code); + QRY_RET(qGenerateExplainResNode(plan->pNode, pExecInfo, pRes)); } -int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t tlen) { - +int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { + int32_t tlen = *len; + + EXPLAIN_ROW_APPEND("(exec info here)"); + + *len = tlen; + + return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t level) { +int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = strdup(tbuf); if (NULL == row.buf) { @@ -187,8 +212,9 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev row.level = level; row.len = len; + ctx->totalSize += len; - if (taosArrayPush(pRows, &row)) { + if (taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -198,7 +224,7 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { int32_t tlen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { @@ -209,174 +235,198 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); if (pTagScanNode->reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); if (pTblScanNode->scan.reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pTblScanNode->pScanConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); if (pSTblScanNode->scan.reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pPrjNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pJoinNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ON_CONDITIONS_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pAggNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pExchNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pSortNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pIntNode->pFill) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } if (pIntNode->window.node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pIntNode->window.node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -389,33 +439,24 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t code = 0; - QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, pRsp, tbuf, level)); + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, tbuf, level)); SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, pRsp, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, tbuf, level + 1)); } return TSDB_CODE_SUCCESS; } -int32_t qExplainRowsToRsp(SArray *rows, SRetrieveTableRsp **pRsp) { - int32_t rspSize = sizeof(SRetrieveTableRsp) + ; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); - if (NULL == rsp) { - qError("malloc SRetrieveTableRsp failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } -} - -int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { +int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_RET(TSDB_CODE_QRY_APP_ERROR); @@ -427,24 +468,92 @@ int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **p qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - - SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); - if (NULL == rows) { - qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); - - QRY_ERR_JRET(qExplainRowsToRsp(rows, pRsp)); + QRY_ERR_JRET(qExplainResNodeToRows(pResNode, ctx, tbuf, level)); _return: taosMemoryFree(tbuf); - taosArrayDestroy(rows); QRY_RET(code); } +int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level) { + SExplainResNode *node = NULL; + int32_t code = 0; + + QRY_ERR_RET(qGenerateExplainResNodeTree(plan, pExecTree, &node)); + + if (NULL == *pRowCtx) { + *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); + if (NULL == *pRowCtx) { + qError("calloc SExplainRowCtx failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + (*pRowCtx)->rows = rows; + } + + QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, *pRowCtx, level)); + +_return: + + qFreeExplainResTree(node); + + if (code) { + taosArrayDestroy((*pRowCtx)->rows); + taosMemoryFree(*pRowCtx); + *pRowCtx = NULL; + } + + QRY_RET(code); +} + + +int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { + SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + int32_t rowNum = taosArrayGetSize(pCtx->rows); + if (rowNum <= 0) { + qError("empty explain res rows"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t colNum = 1; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + rsp->completed = 1; + rsp->numOfRows = rowNum; + + *(int32_t *)rsp->data = pCtx->totalSize; + + int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + char *data = (char *)(offset + rowNum); + int32_t tOffset = 0; + + for (int32_t i = 0; i < rowNum; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + *offset = tOffset; + tOffset += row->len; + + memcpy(data, row->buf, row->len); + + ++offset; + data += row->len; + } + + return TSDB_CODE_SUCCESS; +} + + -#pragma GCC diagnostic pop diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 834a722bd8..3e04e7b30a 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -36,8 +36,6 @@ extern "C" { #define FILTER_DUMMY_EMPTY_OPTR 127 -#define MAX_NUM_STR_SIZE 40 - #define FILTER_RM_UNIT_MIN_ROWS 100 enum { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 518da6e2b8..579a995056 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,6 +38,11 @@ enum { SCH_WRITE, }; +typedef struct SSchExplainGroup { + int32_t nodeNum; + SSubplan *plan; +} SSchExplainGroup; + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -142,7 +147,7 @@ typedef struct SSchTask { } SSchTask; typedef struct SSchJobAttr { - bool needFetch; + bool analyzeExplain; bool syncSchedule; bool queryJob; bool needFlowCtrl; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fdb0b6498c..376febcdf3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -476,6 +476,90 @@ _return: SCH_RET(code); } +int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { + int32_t code = 0; + pJob->queryId = pDag->queryId; + + if (pDag->numOfSubplans <= 0) { + SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); + if (levelNum <= 0) { + SCH_JOB_ELOG("invalid level num:%d", levelNum); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *groupHash = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == groupHash) { + SCH_JOB_ELOG("groupHash %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->subPlans = pDag->pSubplans; + + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SSchExplainGroup *pGroup = NULL; + void *rowCtx = NULL; + + for (int32_t i = 0; i < levelNum; ++i) { + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); + if (NULL == plans) { + SCH_JOB_ELOG("empty level plan, level:%d", i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); + if (taskNum <= 0) { + SCH_JOB_ELOG("invalid level plan number:%d, level:%d", taskNum, i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + for (int32_t n = 0; n < taskNum; ++n) { + SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); + if (pGroup) { + ++pGroup->nodeNum; + continue; + } + + SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { + SCH_TASK_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + void *pIter = taosHashIterate(groupHash, NULL); + while (pIter) { + pGroup = (SSchExplainGroup *)pIter; + + SCH_ERR_JRET(qAppendTaskExplainResRows(&rowCtx, pGroup->plan, NULL, i)); + + pIter = taosHashIterate(groupHash, pIter); + } + + taosHashClear(groupHash); + + SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); + } + + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qGetExplainRspFromRowCtx(rowCtx, &pRsp)); + + pJob->resData = pRsp; + +_return: + + taosHashCleanup(groupHash); + qFreeExplainRowCtx(rowCtx); + + SCH_RET(code); +} + + int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -2093,6 +2177,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + pJob->attr.analyzeExplain = (EXPLAIN_MODE_ANALYZE == pDag->explainInfo.mode); pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; pJob->sql = sql; @@ -2163,6 +2248,51 @@ _return: SCH_RET(code); } +int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, + bool syncSchedule) { + qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); + + int32_t code = 0; + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); + if (NULL == pJob) { + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->sql = sql; + pJob->attr.queryJob = true; + + SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); + + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); + } + + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + + pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + *job = pJob->refId; + SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + + schReleaseJob(pJob->refId); + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeJobImpl(pJob); + SCH_RET(code); +} + int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobRef) { qError("scheduler already initialized"); @@ -2211,7 +2341,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { + SCH_ERR_RET(schStaticExplain(transport, nodeList, pDag, pJob, sql, true)); + } else { + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + } SSchJob *job = schAcquireJob(*pJob); From 100d3230a01fb3587347904756361afb3dc0f7e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:21:47 +0800 Subject: [PATCH 07/13] feature/qnode --- include/libs/qcom/query.h | 2 +- source/libs/qcom/src/queryExplain.c | 17 ++++++++++------- source/libs/scheduler/src/scheduler.c | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 2a9dd28819..dd2dccc02d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,7 +174,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level); +int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level); int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); void qFreeExplainRowCtx(void *ctx); diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d352a29e3c..e1bdf96331 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -478,11 +478,13 @@ _return: QRY_RET(code); } -int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level) { +int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level) { SExplainResNode *node = NULL; int32_t code = 0; + struct SSubplan *pPlan = (struct SSubplan *)plan; + SExplainRowCtx *ctx = (SExplainRowCtx *)*pRowCtx; - QRY_ERR_RET(qGenerateExplainResNodeTree(plan, pExecTree, &node)); + QRY_ERR_RET(qGenerateExplainResNodeTree(pPlan, pExecTree, &node)); if (NULL == *pRowCtx) { *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); @@ -491,24 +493,25 @@ int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *p QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + ctx = (SExplainRowCtx *)*pRowCtx; + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - (*pRowCtx)->rows = rows; + ctx->rows = rows; } - - QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, *pRowCtx, level)); + + QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, ctx, level)); _return: qFreeExplainResTree(node); if (code) { - taosArrayDestroy((*pRowCtx)->rows); - taosMemoryFree(*pRowCtx); + qFreeExplainRowCtx(*pRowCtx); *pRowCtx = NULL; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 376febcdf3..d11dafb19b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -527,7 +527,7 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { - SCH_TASK_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } From 08bd344d2ff0bfdca8fa18233076a16a9920d976 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:59:05 +0800 Subject: [PATCH 08/13] feature/qnode --- source/libs/qcom/src/queryExplain.c | 22 +++++++++++----------- source/libs/scheduler/inc/schedulerInt.h | 8 ++++---- source/libs/scheduler/src/scheduler.c | 12 ++++++++---- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index e1bdf96331..91ca7f54b0 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -31,7 +31,7 @@ void qFreeExplainResTree(SExplainResNode *res) { FOREACH(node, res->pChildren) { qFreeExplainResTree((SExplainResNode *)node); } - nodesDestroyList(res->pChildren); + nodesClearList(res->pChildren); taosMemoryFreeClear(res); } @@ -214,7 +214,7 @@ int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32 row.len = len; ctx->totalSize += len; - if (taosArrayPush(ctx->rows, &row)) { + if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -273,9 +273,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pTblScanNode->pScanConditions) { + if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -301,7 +301,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -318,7 +318,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -340,7 +340,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -357,7 +357,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -374,7 +374,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -392,7 +392,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, - pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -415,7 +415,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 579a995056..1b2c7d54e2 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -147,10 +147,10 @@ typedef struct SSchTask { } SSchTask; typedef struct SSchJobAttr { - bool analyzeExplain; - bool syncSchedule; - bool queryJob; - bool needFlowCtrl; + EExplainMode explainMode; + bool syncSchedule; + bool queryJob; + bool needFlowCtrl; } SSchJobAttr; typedef struct SSchJob { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d11dafb19b..a26b5f32f3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2177,7 +2177,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->attr.analyzeExplain = (EXPLAIN_MODE_ANALYZE == pDag->explainInfo.mode); + pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; pJob->sql = sql; @@ -2528,11 +2528,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(pJob)); + if (!pJob->attr.explainMode == EXPLAIN_MODE_STATIC) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); + tsem_wait(&pJob->rspSem); + } + } else { + SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - tsem_wait(&pJob->rspSem); - status = SCH_GET_JOB_STATUS(pJob); if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { From 35f8a337af018d3afcfdebe9258f14349ac3fc77 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 19:37:21 +0800 Subject: [PATCH 09/13] feature/qnode --- include/util/tdef.h | 1 + source/libs/parser/src/parTranslater.c | 1 + source/libs/qcom/inc/queryInt.h | 12 ++++------ source/libs/qcom/src/queryExplain.c | 33 ++++++++++++++------------ source/libs/scheduler/src/scheduler.c | 3 ++- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4b53cde6f5..38c0000ccc 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -389,6 +389,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 +#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d453193d0f..ee22529f1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1944,6 +1944,7 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; + strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 6fc15aacfc..510ba92a86 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -22,8 +22,6 @@ extern "C" { #include "nodes.h" #include "plannodes.h" -#define QUERY_EXPLAIN_MAX_RES_LEN 1024 - //newline area #define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" #define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" @@ -42,8 +40,8 @@ extern "C" { #define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" //append area -#define EXPLAIN_LOOPS_FORMAT "loops %d" -#define EXPLAIN_REVERSE_FORMAT "reverse %d" +#define EXPLAIN_LOOPS_FORMAT "loops=%d" +#define EXPLAIN_REVERSE_FORMAT "reverse=%d" typedef struct SExplainResNode { @@ -68,11 +66,11 @@ typedef struct SExplainRowCtx { #define EXPLAIN_ROW_NEW(level, ...) \ do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, QUERY_EXPLAIN_MAX_RES_LEN, "%*s", level, ""); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__); \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2, ""); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) -#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) #define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index 91ca7f54b0..b2b061cee0 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -204,12 +204,13 @@ int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; - row.buf = strdup(tbuf); + row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { - qError("strdup %s failed", tbuf); + qError("taosMemoryMalloc %d failed", len); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + memcpy(row.buf, tbuf, len); row.level = level; row.len = len; ctx->totalSize += len; @@ -275,7 +276,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -310,7 +311,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -327,13 +328,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; @@ -349,7 +350,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -366,7 +367,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -383,7 +384,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -407,7 +408,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -424,7 +425,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -463,9 +464,9 @@ int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCt } int32_t code = 0; - char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); if (NULL == tbuf) { - qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -536,9 +537,9 @@ int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { } rsp->completed = 1; - rsp->numOfRows = rowNum; + rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = pCtx->totalSize; + *(int32_t *)rsp->data = htonl(pCtx->totalSize); int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); char *data = (char *)(offset + rowNum); @@ -555,6 +556,8 @@ int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { data += row->len; } + *pRsp = rsp; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a26b5f32f3..4a84536d00 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2261,6 +2261,7 @@ int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, i pJob->sql = sql; pJob->attr.queryJob = true; + pJob->attr.explainMode = pDag->explainInfo.mode; SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); @@ -2528,7 +2529,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - if (!pJob->attr.explainMode == EXPLAIN_MODE_STATIC) { + if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); tsem_wait(&pJob->rspSem); } From 236b8adad3d7e460b1dca075b057fb082e2ce5eb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 31 Mar 2022 16:00:47 +0800 Subject: [PATCH 10/13] feature/qnode --- include/common/ttime.h | 1 + include/libs/qcom/query.h | 7 +- include/os/osTime.h | 8 + source/common/src/ttime.c | 29 ++- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/qcom/inc/queryInt.h | 52 ++-- source/libs/qcom/src/queryExplain.c | 290 ++++++++++++--------- source/libs/scheduler/inc/schedulerInt.h | 6 +- source/libs/scheduler/src/scheduler.c | 42 +-- 9 files changed, 263 insertions(+), 174 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 57af24e635..2209cc998f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -62,6 +62,7 @@ int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t t void deltaToUtcInitOnce(); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index dd2dccc02d..5908b0b875 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,9 +174,10 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level); -int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); -void qFreeExplainRowCtx(void *ctx); +int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose); +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); +int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp); +void qFreeExplainCtx(void *ctx); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); diff --git a/include/os/osTime.h b/include/os/osTime.h index 9e426455dc..766fec0fbd 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -46,6 +46,14 @@ extern "C" { #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) #define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define NANOSECOND_PER_USEC (1000L) +#define NANOSECOND_PER_MSEC (1000000L) +#define NANOSECOND_PER_SEC (1000000000L) +#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60) +#define NANOSECOND_PER_HOUR (NANOSECOND_PER_MINUTE * 60) +#define NANOSECOND_PER_DAY (NANOSECOND_PER_HOUR * 24) +#define NANOSECOND_PER_WEEK (NANOSECOND_PER_DAY * 7) + int32_t taosGetTimeOfDay(struct timeval *tv); //@return timestamp in second diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 510a2809e7..23b19b55e7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -370,6 +370,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || + fromPrecision == TSDB_TIME_PRECISION_NANO); + static double factors[3] = {1000000., 1000., 1.}; + switch (toUnit) { + case 's': + return time * factors[fromPrecision] / NANOSECOND_PER_SEC; + case 'm': + return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE; + case 'h': + return time * factors[fromPrecision] / NANOSECOND_PER_HOUR; + case 'd': + return time * factors[fromPrecision] / NANOSECOND_PER_DAY; + case 'w': + return time * factors[fromPrecision] / NANOSECOND_PER_WEEK; + case 'a': + return time * factors[fromPrecision] / NANOSECOND_PER_MSEC; + case 'u': + return time * factors[fromPrecision] / NANOSECOND_PER_USEC; + case 'b': + return time * factors[fromPrecision]; + default: { + return -1; + } + } +} + static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { switch (unit) { case 's': @@ -688,4 +715,4 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); tstrncpy(buf, ts, bufLen); -} \ No newline at end of file +} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 7f6d8826e6..f57f476d51 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // set the output if (TSDB_CODE_SUCCESS == code) { pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); - if (NULL == pJoin->pOnConditions) { + if (NULL == pJoin->node.pTargets) { code = TSDB_CODE_OUT_OF_MEMORY; } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 510ba92a86..06d5b07dc7 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -23,25 +23,34 @@ extern "C" { #include "plannodes.h" //newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" #define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" #define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" -#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" -#define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" -#define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" #define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " #define EXPLAIN_FILL_FORMAT "Fill: %s" #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " -#define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" +#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" //append area -#define EXPLAIN_LOOPS_FORMAT "loops=%d" -#define EXPLAIN_REVERSE_FORMAT "reverse=%d" +#define EXPLAIN_GROUPS_FORMAT " groups=%d" +#define EXPLAIN_WIDTH_FORMAT " width=%d" +#define EXPLAIN_LOOPS_FORMAT " loops=%d" +#define EXPLAIN_REVERSE_FORMAT " reverse=%d" + +//TODO MOVE TO LIB +typedef struct SExplainGroup { + int32_t nodeNum; + SSubplan *plan; + void *execInfo; //TODO +} SExplainGroup; typedef struct SExplainResNode { @@ -56,22 +65,27 @@ typedef struct SQueryExplainRowInfo { char *buf; } SQueryExplainRowInfo; -typedef struct SExplainRowCtx { - int32_t totalSize; - SArray *rows; -} SExplainRowCtx; +typedef struct SExplainCtx { + int32_t totalSize; + bool verbose; + char *tbuf; + SArray *rows; + SHashObj *groupHash; +} SExplainCtx; #define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2, ""); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) + +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) #define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) -#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index b2b061cee0..af092e842e 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -36,22 +36,62 @@ void qFreeExplainResTree(SExplainResNode *res) { taosMemoryFreeClear(res); } -void qFreeExplainRowCtx(void *ctx) { +void qFreeExplainCtx(void *ctx) { if (NULL == ctx) { return; } - SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowSize = taosArrayGetSize(pCtx->rows); for (int32_t i = 0; i < rowSize; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); taosMemoryFreeClear(row->buf); } + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } +int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) { + int32_t code = 0; + SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); + if (NULL == ctx) { + qError("calloc SExplainCtx failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); + if (NULL == tbuf) { + qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ctx->verbose = verbose; + ctx->tbuf = tbuf; + ctx->rows = rows; + ctx->groupHash = groupHash; + + *pCtx = ctx; + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroy(rows); + taosHashCleanup(groupHash); + taosMemoryFree(ctx); + + QRY_RET(code); +} + + char *qFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: @@ -187,11 +227,6 @@ _return: QRY_RET(code); } -int32_t qGenerateExplainResNodeTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { - void *pExecInfo = NULL; // TODO - QRY_RET(qGenerateExplainResNode(plan->pNode, pExecInfo, pRes)); -} - int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = *len; @@ -202,7 +237,7 @@ int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { +int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { @@ -225,8 +260,11 @@ int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32 } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -246,10 +284,12 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: @@ -265,20 +305,22 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pTblScanNode->scan.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -294,10 +336,12 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ @@ -309,11 +353,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pPrjNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pPrjNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -326,51 +372,70 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pJoinNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + if (verbose) { + if (pJoinNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length); + if (pAggNode->pGroupKeys) { + EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pAggNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pAggNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->outputRowSize); + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + if (NULL == group) { + qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pExchNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pExchNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } + + QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ @@ -382,35 +447,42 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pSortNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pSortNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, - pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->offset, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, + pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + if (verbose) { + if (pIntNode->pFill) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } - if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -423,11 +495,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -440,88 +514,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t code = 0; - QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, tbuf, level)); + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); } return TSDB_CODE_SUCCESS; } -int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCtx *ctx, int32_t level) { - if (NULL == pResNode) { - qError("explain res node is NULL"); - QRY_RET(TSDB_CODE_QRY_APP_ERROR); - } - - int32_t code = 0; - char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); - if (NULL == tbuf) { - qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); - QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - QRY_ERR_JRET(qExplainResNodeToRows(pResNode, ctx, tbuf, level)); - -_return: - - taosMemoryFree(tbuf); - - QRY_RET(code); -} - -int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level) { +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; int32_t code = 0; - struct SSubplan *pPlan = (struct SSubplan *)plan; - SExplainRowCtx *ctx = (SExplainRowCtx *)*pRowCtx; - - QRY_ERR_RET(qGenerateExplainResNodeTree(pPlan, pExecTree, &node)); + SExplainCtx *ctx = (SExplainCtx *)pCtx; - if (NULL == *pRowCtx) { - *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); - if (NULL == *pRowCtx) { - qError("calloc SExplainRowCtx failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ctx = (SExplainRowCtx *)*pRowCtx; - - SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); - if (NULL == rows) { - qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ctx->rows = rows; + SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); + if (NULL == group) { + qError("group %d not in groupHash", groupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } + + QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node)); - QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, ctx, level)); + QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); _return: qFreeExplainResTree(node); - - if (code) { - qFreeExplainRowCtx(*pRowCtx); - *pRowCtx = NULL; - } QRY_RET(code); } -int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { - SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; +int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { + SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 1b2c7d54e2..f19e822974 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,10 +38,12 @@ enum { SCH_WRITE, }; -typedef struct SSchExplainGroup { +//TODO MOVE TO LIB +typedef struct SExplainGroup { int32_t nodeNum; SSubplan *plan; -} SSchExplainGroup; + void *execInfo; //TODO +} SExplainGroup; typedef struct SSchTrans { void *transInst; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4a84536d00..940aaed7c1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -478,7 +478,14 @@ _return: int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SExplainGroup *pGroup = NULL; + void *pCtx = NULL; + int32_t rootGroupId = 0; + pJob->queryId = pDag->queryId; + pJob->subPlans = pDag->pSubplans; if (pDag->numOfSubplans <= 0) { SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); @@ -497,12 +504,7 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->subPlans = pDag->pSubplans; - - SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SSchExplainGroup *pGroup = NULL; - void *rowCtx = NULL; + SCH_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -517,44 +519,44 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + SSubplan *plan = NULL; for (int32_t n = 0; n < taskNum; ++n) { - SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); if (pGroup) { ++pGroup->nodeNum; continue; } - SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; + SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } - void *pIter = taosHashIterate(groupHash, NULL); - while (pIter) { - pGroup = (SSchExplainGroup *)pIter; + if (0 == i) { + if (taskNum > 1) { + SCH_JOB_ELOG("invalid taskNum %d for level 0", taskNum); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } - SCH_ERR_JRET(qAppendTaskExplainResRows(&rowCtx, pGroup->plan, NULL, i)); - - pIter = taosHashIterate(groupHash, pIter); + rootGroupId = plan->id.groupId; } - taosHashClear(groupHash); - - SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); + SCH_JOB_DLOG("level %d group handled, taskNum:%d", i, taskNum); } + SCH_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); + SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qGetExplainRspFromRowCtx(rowCtx, &pRsp)); + SCH_ERR_JRET(qGetExplainRspFromCtx(pCtx, &pRsp)); pJob->resData = pRsp; _return: - taosHashCleanup(groupHash); - qFreeExplainRowCtx(rowCtx); + qFreeExplainCtx(pCtx); SCH_RET(code); } From 0608824186c025e294ee0aa3384ddbc4f66c44c8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 31 Mar 2022 18:02:58 +0800 Subject: [PATCH 11/13] feature/qnode --- include/common/ttime.h | 1 + include/libs/command/command.h | 5 + include/libs/nodes/plannodes.h | 2 +- include/libs/qcom/query.h | 10 +- source/common/src/ttime.c | 12 ++ source/libs/command/inc/commandInt.h | 95 +++++++++++++++ .../queryExplain.c => command/src/explain.c} | 112 ++++++++++++++++-- source/libs/qcom/inc/queryInt.h | 71 ----------- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/scheduler/inc/schedulerInt.h | 7 -- source/libs/scheduler/src/scheduler.c | 95 +-------------- 11 files changed, 227 insertions(+), 185 deletions(-) create mode 100644 source/libs/command/inc/commandInt.h rename source/libs/{qcom/src/queryExplain.c => command/src/explain.c} (85%) diff --git a/include/common/ttime.h b/include/common/ttime.h index 2209cc998f..306f54bedb 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -60,6 +60,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); +char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 699c2f9792..7e58d39692 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -15,5 +15,10 @@ #include "cmdnodes.h" #include "tmsg.h" +#include "plannodes.h" int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp); + +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); + + diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f41e049196..37163f60dd 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - uint8_t precision; + uint8_t precision; SFillNode* pFill; } SIntervalPhysiNode; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 5908b0b875..bb550e75e8 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,11 +174,6 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose); -int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); -int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp); -void qFreeExplainCtx(void *ctx); - SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); @@ -241,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t } \ } while (0) +#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + #ifdef __cplusplus } #endif diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 23b19b55e7..a65352f2b9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { return 0; } +char getPrecisionUnit(int32_t precision) { + static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: + return units[precision]; + default: + return 0; + } +} + int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h new file mode 100644 index 0000000000..5915aa7f36 --- /dev/null +++ b/source/libs/command/inc/commandInt.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "nodes.h" +#include "plannodes.h" +#include "ttime.h" + +#define EXPLAIN_MAX_GROUP_NUM 100 + +//newline area +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" +#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" +#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" +#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" +#define EXPLAIN_ORDER_FORMAT "Order: %s" +#define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " +#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" + +//append area +#define EXPLAIN_GROUPS_FORMAT " groups=%d" +#define EXPLAIN_WIDTH_FORMAT " width=%d" +#define EXPLAIN_LOOPS_FORMAT " loops=%d" +#define EXPLAIN_REVERSE_FORMAT " reverse=%d" + +typedef struct SExplainGroup { + int32_t nodeNum; + SSubplan *plan; + void *execInfo; //TODO +} SExplainGroup; + +typedef struct SExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SExplainResNode; + +typedef struct SQueryExplainRowInfo { + int32_t level; + int32_t len; + char *buf; +} SQueryExplainRowInfo; + +typedef struct SExplainCtx { + int32_t totalSize; + bool verbose; + char *tbuf; + SArray *rows; + SHashObj *groupHash; +} SExplainCtx; + +#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") +#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") + +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) + +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ + } while (0) + +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/command/src/explain.c similarity index 85% rename from source/libs/qcom/src/queryExplain.c rename to source/libs/command/src/explain.c index af092e842e..5fa12f5ebe 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/command/src/explain.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ -#include "queryInt.h" #include "query.h" #include "plannodes.h" +#include "commandInt.h" int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); void qFreeExplainResTree(SExplainResNode *res) { @@ -265,6 +266,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i bool isVerboseLine = false; char *tbuf = ctx->tbuf; bool verbose = ctx->verbose; + int32_t filterLen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -317,7 +319,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -356,7 +359,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -375,13 +379,15 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -404,7 +410,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -429,7 +436,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -450,7 +458,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -461,7 +470,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->offset, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + pIntNode->offset, getPrecisionUnit(pIntNode->precision), INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { @@ -479,7 +488,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -498,7 +508,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -595,5 +606,84 @@ int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { + int32_t code = 0; + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SExplainGroup *pGroup = NULL; + void *pCtx = NULL; + int32_t rootGroupId = 0; + + if (pDag->numOfSubplans <= 0) { + qError("invalid subplan num:%d", pDag->numOfSubplans); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); + if (levelNum <= 0) { + qError("invalid level num:%d", levelNum); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == groupHash) { + qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); + + for (int32_t i = 0; i < levelNum; ++i) { + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); + if (NULL == plans) { + qError("empty level plan, level:%d", i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); + if (taskNum <= 0) { + qError("invalid level plan number:%d, level:%d", taskNum, i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SSubplan *plan = NULL; + for (int32_t n = 0; n < taskNum; ++n) { + plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); + if (pGroup) { + ++pGroup->nodeNum; + continue; + } + + SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { + qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (0 == i) { + if (taskNum > 1) { + qError("invalid taskNum %d for level 0", taskNum); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + rootGroupId = plan->id.groupId; + } + + qDebug("level %d group handled, taskNum:%d", i, taskNum); + } + + QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); + + QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp)); + +_return: + + qFreeExplainCtx(pCtx); + + QRY_RET(code); +} + diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 06d5b07dc7..f120bf26ce 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -19,77 +19,6 @@ #ifdef __cplusplus extern "C" { #endif -#include "nodes.h" -#include "plannodes.h" - -//newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" -#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" -#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" -#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" -#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" -#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" -#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" -#define EXPLAIN_ORDER_FORMAT "Order: %s" -#define EXPLAIN_FILTER_FORMAT "Filter: " -#define EXPLAIN_FILL_FORMAT "Fill: %s" -#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " -#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" - -//append area -#define EXPLAIN_GROUPS_FORMAT " groups=%d" -#define EXPLAIN_WIDTH_FORMAT " width=%d" -#define EXPLAIN_LOOPS_FORMAT " loops=%d" -#define EXPLAIN_REVERSE_FORMAT " reverse=%d" - -//TODO MOVE TO LIB -typedef struct SExplainGroup { - int32_t nodeNum; - SSubplan *plan; - void *execInfo; //TODO -} SExplainGroup; - - -typedef struct SExplainResNode { - SNodeList* pChildren; - SPhysiNode* pNode; - void* pExecInfo; -} SExplainResNode; - -typedef struct SQueryExplainRowInfo { - int32_t level; - int32_t len; - char *buf; -} SQueryExplainRowInfo; - -typedef struct SExplainCtx { - int32_t totalSize; - bool verbose; - char *tbuf; - SArray *rows; - SHashObj *groupHash; -} SExplainCtx; - -#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") -#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") - -#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) - -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ - } while (0) - -#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) -#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) - -#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) -#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index a4a299317c..1a62c7d89d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport + PUBLIC os util nodes planner qcom common catalog transport command ) if(${BUILD_TEST}) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index f19e822974..22bd039219 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,13 +38,6 @@ enum { SCH_WRITE, }; -//TODO MOVE TO LIB -typedef struct SExplainGroup { - int32_t nodeNum; - SSubplan *plan; - void *execInfo; //TODO -} SExplainGroup; - typedef struct SSchTrans { void *transInst; void *transHandle; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 940aaed7c1..7c24a25f65 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -19,6 +19,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "command.h" SSchedulerMgmt schMgmt = {0}; @@ -476,92 +477,6 @@ _return: SCH_RET(code); } -int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { - int32_t code = 0; - SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SExplainGroup *pGroup = NULL; - void *pCtx = NULL; - int32_t rootGroupId = 0; - - pJob->queryId = pDag->queryId; - pJob->subPlans = pDag->pSubplans; - - if (pDag->numOfSubplans <= 0) { - SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); - if (levelNum <= 0) { - SCH_JOB_ELOG("invalid level num:%d", levelNum); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SHashObj *groupHash = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (NULL == groupHash) { - SCH_JOB_ELOG("groupHash %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SCH_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); - - for (int32_t i = 0; i < levelNum; ++i) { - plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); - if (NULL == plans) { - SCH_JOB_ELOG("empty level plan, level:%d", i); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); - if (taskNum <= 0) { - SCH_JOB_ELOG("invalid level plan number:%d, level:%d", taskNum, i); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SSubplan *plan = NULL; - for (int32_t n = 0; n < taskNum; ++n) { - plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); - pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); - if (pGroup) { - ++pGroup->nodeNum; - continue; - } - - SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; - if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { - SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - if (0 == i) { - if (taskNum > 1) { - SCH_JOB_ELOG("invalid taskNum %d for level 0", taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - rootGroupId = plan->id.groupId; - } - - SCH_JOB_DLOG("level %d group handled, taskNum:%d", i, taskNum); - } - - SCH_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); - - SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qGetExplainRspFromCtx(pCtx, &pRsp)); - - pJob->resData = pRsp; - -_return: - - qFreeExplainCtx(pCtx); - - SCH_RET(code); -} - - int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -2250,7 +2165,7 @@ _return: SCH_RET(code); } -int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, +int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); @@ -2264,8 +2179,10 @@ int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, i pJob->sql = sql; pJob->attr.queryJob = true; pJob->attr.explainMode = pDag->explainInfo.mode; + pJob->queryId = pDag->queryId; + pJob->subPlans = pDag->pSubplans; - SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); + SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { @@ -2345,7 +2262,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in } if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { - SCH_ERR_RET(schStaticExplain(transport, nodeList, pDag, pJob, sql, true)); + SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); } From b449369265894a3fdcb8070e8bab040db50bdcf9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Apr 2022 10:29:20 +0800 Subject: [PATCH 12/13] feature/qnode --- source/libs/catalog/src/catalog.c | 28 ++++++++++++---- source/libs/command/inc/commandInt.h | 6 ++-- source/libs/command/src/explain.c | 44 ++++++++++++------------- source/libs/nodes/src/nodesToSQLFuncs.c | 35 ++++++++++---------- source/libs/nodes/src/nodesUtilFuncs.c | 5 +-- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index db772000b5..e1f5332899 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2011,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); while (true) { - tsem_wait(&gCtgMgmt.queue.reqSem); + if (tsem_wait(&gCtgMgmt.queue.reqSem)) { + qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); @@ -2175,10 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_RET(ctgStartUpdateThread()); - - tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); - tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); + if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } + + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { @@ -2187,6 +2194,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + CTG_ERR_RET(ctgStartUpdateThread()); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; @@ -2718,8 +2727,13 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); - tsem_post(&gCtgMgmt.queue.reqSem); - tsem_post(&gCtgMgmt.queue.rspSem); + if (tsem_post(&gCtgMgmt.queue.reqSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } + + if (tsem_post(&gCtgMgmt.queue.rspSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 5915aa7f36..cc3e1d8b55 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -26,9 +26,9 @@ extern "C" { #define EXPLAIN_MAX_GROUP_NUM 100 //newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d" #define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" #define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" #define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 5fa12f5ebe..847a863e76 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -266,7 +266,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i bool isVerboseLine = false; char *tbuf = ctx->tbuf; bool verbose = ctx->verbose; - int32_t filterLen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -276,7 +275,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -297,7 +296,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -319,8 +318,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -329,7 +327,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -343,7 +341,15 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pSTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } break; } @@ -359,8 +365,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -379,15 +384,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -410,8 +413,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -436,8 +438,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -458,8 +459,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -488,8 +488,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -508,8 +507,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index fa941cfe9c..68f38bb6a7 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -30,18 +30,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { switch (pNode->type) { case QUERY_NODE_COLUMN: { SColumnNode *colNode = (SColumnNode *)pNode; - *len = 0; if (colNode->dbName[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->dbName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName); } if (colNode->tableAlias[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableAlias); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias); } else if (colNode->tableName[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName); } - *len += snprintf(buf, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); return TSDB_CODE_SUCCESS; } @@ -53,14 +52,14 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - *len += snprintf(buf, bufSize - *len, "%s", t); + *len += snprintf(buf + *len, bufSize - *len, "%s", t); taosMemoryFree(t); return TSDB_CODE_SUCCESS; } case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); if (pOpNode->pLeft) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } @@ -70,13 +69,13 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); if (pOpNode->pRight) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -85,17 +84,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pLogicNode->pParameterList) { if (!first) { - *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -104,17 +103,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "%s(", pFuncNode->functionName); + *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName); FOREACH(node, pFuncNode->pParameterList) { if (!first) { - *len += snprintf(buf, bufSize - *len, ", "); + *len += snprintf(buf + *len, bufSize - *len, ", "); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -123,17 +122,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pListNode->pNodeList) { if (!first) { - *len += snprintf(buf, bufSize - *len, ", "); + *len += snprintf(buf + *len, bufSize - *len, ", "); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index de46c5a48b..732d9acfbe 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -839,12 +839,13 @@ char* nodesGetStrValueFromNode(SValueNode *pNode) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - void *buf = taosMemoryMalloc(varDataLen(pNode->datum.p) + 1); + int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1; + void *buf = taosMemoryMalloc(bufSize); if (NULL == buf) { return NULL; } - strncpy(buf, varDataVal(pNode->datum.p), varDataLen(pNode->datum.p) + 1); + snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p)); return buf; } default: From 1bf4b37591d22fa2014f69fec9da7c6f39330eb5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Apr 2022 11:19:21 +0800 Subject: [PATCH 13/13] feature/qnode --- source/libs/command/inc/commandInt.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index cc3e1d8b55..771baca2ab 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -79,10 +79,14 @@ typedef struct SExplainCtx { #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + if (isVerboseLine) { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, ""); \ + } else { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> "); \ + } \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) #define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)