diff --git a/include/common/ttime.h b/include/common/ttime.h index 57af24e635..306f54bedb 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); +char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 699c2f9792..7e58d39692 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -15,5 +15,10 @@ #include "cmdnodes.h" #include "tmsg.h" +#include "plannodes.h" int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp); + +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); + + diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 5f86c75787..9849dfca39 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -215,6 +215,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f41e049196..37163f60dd 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - uint8_t precision; + uint8_t precision; SFillNode* pFill; } SIntervalPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 23a63c1a0b..ddcbaa0bee 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -314,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); void* nodesGetValueFromNode(SValueNode *pNode); +char* nodesGetStrValueFromNode(SValueNode *pNode); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f487e5ae22..bb550e75e8 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,6 +53,7 @@ typedef struct SIndexMeta { } SIndexMeta; + /* * ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(tableType == TSDB_CHILD_TABLE) @@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t } \ } while (0) +#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + #ifdef __cplusplus } #endif diff --git a/include/os/osTime.h b/include/os/osTime.h index 9e426455dc..766fec0fbd 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -46,6 +46,14 @@ extern "C" { #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) #define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define NANOSECOND_PER_USEC (1000L) +#define NANOSECOND_PER_MSEC (1000000L) +#define NANOSECOND_PER_SEC (1000000000L) +#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60) +#define NANOSECOND_PER_HOUR (NANOSECOND_PER_MINUTE * 60) +#define NANOSECOND_PER_DAY (NANOSECOND_PER_HOUR * 24) +#define NANOSECOND_PER_WEEK (NANOSECOND_PER_DAY * 7) + int32_t taosGetTimeOfDay(struct timeval *tv); //@return timestamp in second diff --git a/include/util/tdef.h b/include/util/tdef.h index 43981adea2..38c0000ccc 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -224,6 +224,7 @@ typedef enum ELogicConditionType { #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_STB_COMMENT_LEN 1024 + /** * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. @@ -388,6 +389,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 +#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 @@ -479,6 +481,9 @@ enum { #define QND_VGID 1 #define VND_VGID 0 +#define MAX_NUM_STR_SIZE 40 + + #ifdef __cplusplus } #endif diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 510a2809e7..a65352f2b9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { return 0; } +char getPrecisionUnit(int32_t precision) { + static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: + return units[precision]; + default: + return 0; + } +} + int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); @@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || + fromPrecision == TSDB_TIME_PRECISION_NANO); + static double factors[3] = {1000000., 1000., 1.}; + switch (toUnit) { + case 's': + return time * factors[fromPrecision] / NANOSECOND_PER_SEC; + case 'm': + return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE; + case 'h': + return time * factors[fromPrecision] / NANOSECOND_PER_HOUR; + case 'd': + return time * factors[fromPrecision] / NANOSECOND_PER_DAY; + case 'w': + return time * factors[fromPrecision] / NANOSECOND_PER_WEEK; + case 'a': + return time * factors[fromPrecision] / NANOSECOND_PER_MSEC; + case 'u': + return time * factors[fromPrecision] / NANOSECOND_PER_USEC; + case 'b': + return time * factors[fromPrecision]; + default: { + return -1; + } + } +} + static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { switch (unit) { case 's': @@ -688,4 +727,4 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); tstrncpy(buf, ts, bufLen); -} \ No newline at end of file +} diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9bd5c1c016..09f51dc03e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -112,7 +112,14 @@ typedef struct SCtgRuntimeStat { } SCtgRuntimeStat; typedef struct SCtgCacheStat { - + uint64_t clusterNum; + uint64_t dbNum; + uint64_t tblNum; + uint64_t stblNum; + uint64_t vgHitNum; + uint64_t vgMissNum; + uint64_t tblHitNum; + uint64_t tblMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -186,7 +193,7 @@ typedef struct SCatalogMgmt { bool exit; SRWLatch lock; SCtgQueue queue; - TdThread updateThread; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; @@ -204,8 +211,13 @@ typedef struct SCtgAction { #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) -#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) +#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) + +#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n)) +#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n)) +#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -291,6 +303,9 @@ typedef struct SCtgAction { #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +extern void ctgdShowClusterCache(SCatalog* pCtg); +extern int32_t ctgdShowCacheInfo(void); #ifdef __cplusplus } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a3019d8d56..e1f5332899 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; -SCtgDebug gCTGDebug = {0}; SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_UPDATE_VG, "update vgInfo", @@ -53,182 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ } }; -int32_t ctgDbgEnableDebug(char *option) { - if (0 == strcasecmp(option, "lock")) { - gCTGDebug.lockEnable = true; - qDebug("lock debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "cache")) { - gCTGDebug.cacheEnable = true; - qDebug("cache debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "api")) { - gCTGDebug.apiEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "meta")) { - gCTGDebug.metaEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - qError("invalid debug option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetStatNum(char *option, void *res) { - if (0 == strcasecmp(option, "runtime.qDoneNum")) { - *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); - return TSDB_CODE_SUCCESS; - } - - qError("invalid stat option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; -} - -int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; -} - -int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { - int32_t num = 0; - for (uint16_t i = 0; i < rent->slotNum; ++i) { - SCtgRentSlot *slot = &rent->slots[i]; - if (NULL == slot->meta) { - continue; - } - - num += taosArrayGetSize(slot->meta); - } - - return num; -} - -int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { - if (NULL == pCtg || NULL == pCtg->dbCache) { - return 0; - } - - switch (type) { - case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCtg->dbCache); - case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->dbRent); - case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->stbRent); - default: - break; - } - - SCtgDBCache *dbCache = NULL; - int32_t num = 0; - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - dbCache = (SCtgDBCache *)pIter; - switch (type) { - case CTG_DBG_META_NUM: - num += ctgDbgGetTbMetaNum(dbCache); - break; - case CTG_DBG_STB_NUM: - num += ctgDbgGetStbNum(dbCache); - break; - default: - ctgError("invalid type:%d", type); - break; - } - pIter = taosHashIterate(pCtg->dbCache, pIter); - } - - return num; -} - -void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { - if (!gCTGDebug.metaEnable) { - return; - } - - STableComInfo *c = &p->tableInfo; - - if (TSDB_CHILD_TABLE == p->tableType) { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); - return; - } else { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", - tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); - } - - int32_t colNum = c->numOfColumns + c->numOfTags; - for (int32_t i = 0; i < colNum; ++i) { - SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes); - } -} - -void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { - if (NULL == dbHash || !gCTGDebug.cacheEnable) { - return; - } - - int32_t i = 0; - SCtgDBCache *dbCache = NULL; - void *pIter = taosHashIterate(dbHash, NULL); - while (pIter) { - char *dbFName = NULL; - size_t len = 0; - - dbCache = (SCtgDBCache *)pIter; - - dbFName = taosHashGetKey(pIter, &len); - - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; - int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; - int32_t hashMethod = -1; - int32_t vgNum = 0; - - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); - } - } - - ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); - - pIter = taosHashIterate(dbHash, pIter); - } -} - - - - -void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (!gCTGDebug.cacheEnable || NULL == pCtg) { - return; - } - - ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - - ctgDbgShowDBCache(pCtg, pCtg->dbCache); -} - - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { return; @@ -249,15 +73,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { + int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; + CTG_CACHE_STAT_SUB(stblNum, stblNum); } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); CTG_LOCK(CTG_WRITE, &cache->metaLock); if (cache->metaCache) { + int32_t tblNum = taosHashGetSize(cache->metaCache); taosHashCleanup(cache->metaCache); cache->metaCache = NULL; + CTG_CACHE_STAT_SUB(tblNum, tblNum); } CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } @@ -293,6 +121,8 @@ void ctgFreeHandle(SCatalog* pCtg) { ctgFreeMetaRent(&pCtg->stbRent); if (pCtg->dbCache) { + int32_t dbNum = taosHashGetSize(pCtg->dbCache); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; @@ -305,6 +135,8 @@ void ctgFreeHandle(SCatalog* pCtg) { } taosHashCleanup(pCtg->dbCache); + + CTG_CACHE_STAT_SUB(dbNum, dbNum); } taosMemoryFree(pCtg); @@ -361,7 +193,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + CTG_RUNTIME_STAT_ADD(qNum, 1); tsem_post(&gCtgMgmt.queue.reqSem); @@ -620,34 +452,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + SCtgDBCache *dbCache = NULL; + if (NULL == pCtg->dbCache) { - *pCache = NULL; - *inCache = false; - ctgWarn("empty db cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + ctgDebug("empty db cache, dbFName:%s", dbFName); + goto _return; } - SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *pCache = NULL; - *inCache = false; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; } ctgAcquireVgInfo(pCtg, dbCache, inCache); if (!(*inCache)) { - ctgReleaseDBCache(pCtg, dbCache); - - *pCache = NULL; - return TSDB_CODE_SUCCESS; + ctgDebug("vgInfo of db %s not in cache", dbFName); + goto _return; } *pCache = dbCache; *inCache = true; + CTG_CACHE_STAT_ADD(vgHitNum, 1); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + +_return: + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + *inCache = false; + + CTG_CACHE_STAT_ADD(vgMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -763,11 +606,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + goto _return; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; @@ -782,8 +624,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", pTableName->tname); + goto _return; } int32_t sz = 0; @@ -792,13 +634,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { - *exist = 0; ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - return TSDB_CODE_SUCCESS; + goto _return; } - *exist = 1; if (dbId) { *dbId = dbCache->dbId; } @@ -808,6 +648,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + return TSDB_CODE_SUCCESS; } @@ -819,8 +663,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); taosMemoryFreeClear(*pTableMeta); - *exist = 0; - return TSDB_CODE_SUCCESS; + goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { @@ -846,8 +689,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(tblMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -1377,6 +1230,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(dbNum, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); @@ -1436,6 +1291,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + CTG_CACHE_STAT_SUB(dbNum, 1); + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1568,6 +1425,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_LOCK(CTG_WRITE, &tbCache->stbLock); if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1594,8 +1453,12 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + if (NULL == orig) { + CTG_CACHE_STAT_ADD(tblNum, 1); + } + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - ctgDbgShowTableMeta(pCtg, tbName, meta); + ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); @@ -1615,6 +1478,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(stblNum, 1); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1874,7 +1739,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t exist = 0; + bool inCache = false; int32_t code = 0; uint64_t dbId = 0; uint64_t suid = 0; @@ -1884,11 +1749,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_FLAG_SET_INF_DB(flag); } - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId)); int32_t tbType = 0; - if (exist) { + if (inCache) { if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) { goto _return; } @@ -1930,8 +1795,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL)); - if (0 == exist) { + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); + if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; } @@ -1943,7 +1808,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons _return: - if (CTG_TABLE_NOT_EXIST(code) && exist) { + if (CTG_TABLE_NOT_EXIST(code) && inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); @@ -1962,7 +1827,7 @@ _return: if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); - ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta); + ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta); } CTG_RET(code); @@ -2075,12 +1940,16 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - } + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); @@ -2119,6 +1988,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -2140,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); while (true) { - tsem_wait(&gCtgMgmt.queue.reqSem); + if (tsem_wait(&gCtgMgmt.queue.reqSem)) { + qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); @@ -2161,9 +2034,9 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); - ctgDbgShowClusterCache(pCtg); + ctgdShowClusterCache(pCtg); } CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); @@ -2304,8 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); - tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); + if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } + + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { @@ -2383,6 +2263,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { } *catalogHandle = clusterCtg; + + CTG_CACHE_STAT_ADD(clusterNum, 1); return TSDB_CODE_SUCCESS; @@ -2403,10 +2285,12 @@ void catalogFreeHandle(SCatalog* pCtg) { return; } + CTG_CACHE_STAT_SUB(clusterNum, 1); + uint64_t clusterId = pCtg->clusterId; ctgFreeHandle(pCtg); - + ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } @@ -2417,24 +2301,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCtg->dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); - if (!inCache) { - ctgReleaseDBCache(pCtg, dbCache); + int32_t code = 0; + CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + if (!inCache) { *version = CTG_DEFAULT_INVALID_VERSION; CTG_API_LEAVE(TSDB_CODE_SUCCESS); } @@ -2449,6 +2321,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); + +_return: + + CTG_API_LEAVE(code); } int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { @@ -2549,11 +2425,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { } STableMeta *tblMeta = NULL; - int32_t exist = 0; + bool inCache = false; uint64_t dbId = 0; - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); - if (0 == exist) { + if (!inCache) { ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); goto _return; } @@ -2851,8 +2727,13 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); - tsem_post(&gCtgMgmt.queue.reqSem); - tsem_post(&gCtgMgmt.queue.rspSem); + if (tsem_post(&gCtgMgmt.queue.reqSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } + + if (tsem_post(&gCtgMgmt.queue.rspSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); diff --git a/source/libs/catalog/src/catalogDbg.c b/source/libs/catalog/src/catalogDbg.c new file mode 100644 index 0000000000..1d4ad0082c --- /dev/null +++ b/source/libs/catalog/src/catalogDbg.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" + +extern SCatalogMgmt gCtgMgmt; +SCtgDebug gCTGDebug = {0}; + +int32_t ctgdEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockEnable = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheEnable = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "meta")) { + gCTGDebug.metaEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; +} + +int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; +} + +int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { + int32_t num = 0; + for (uint16_t i = 0; i < rent->slotNum; ++i) { + SCtgRentSlot *slot = &rent->slots[i]; + if (NULL == slot->meta) { + continue; + } + + num += taosArrayGetSize(slot->meta); + } + + return num; +} + +int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { + return 0; + } + + switch (type) { + case CTG_DBG_DB_NUM: + return (int32_t)taosHashGetSize(pCtg->dbCache); + case CTG_DBG_DB_RENT_NUM: + return ctgdGetRentNum(&pCtg->dbRent); + case CTG_DBG_STB_RENT_NUM: + return ctgdGetRentNum(&pCtg->stbRent); + default: + break; + } + + SCtgDBCache *dbCache = NULL; + int32_t num = 0; + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + dbCache = (SCtgDBCache *)pIter; + switch (type) { + case CTG_DBG_META_NUM: + num += ctgdGetTbMetaNum(dbCache); + break; + case CTG_DBG_STB_NUM: + num += ctgdGetStbNum(dbCache); + break; + default: + ctgError("invalid type:%d", type); + break; + } + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + return num; +} + +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { + if (!gCTGDebug.metaEnable) { + return; + } + + STableComInfo *c = &p->tableInfo; + + if (TSDB_CHILD_TABLE == p->tableType) { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + return; + } else { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); + } + + int32_t colNum = c->numOfColumns + c->numOfTags; + for (int32_t i = 0; i < colNum; ++i) { + SSchema *s = &p->schema[i]; + ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + } +} + +void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { + if (NULL == dbHash || !gCTGDebug.cacheEnable) { + return; + } + + int32_t i = 0; + SCtgDBCache *dbCache = NULL; + void *pIter = taosHashIterate(dbHash, NULL); + while (pIter) { + char *dbFName = NULL; + size_t len = 0; + + dbCache = (SCtgDBCache *)pIter; + + dbFName = taosHashGetKey(pIter, &len); + + int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; + int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; + int32_t hashMethod = -1; + int32_t vgNum = 0; + + if (dbCache->vgInfo) { + vgVersion = dbCache->vgInfo->vgVersion; + hashMethod = dbCache->vgInfo->hashMethod; + if (dbCache->vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + } + } + + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + + pIter = taosHashIterate(dbHash, pIter); + } +} + + + + +void ctgdShowClusterCache(SCatalog* pCtg) { + if (!gCTGDebug.cacheEnable || NULL == pCtg) { + return; + } + + ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg); + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); + + ctgdShowDBCache(pCtg, pCtg->dbCache); + + ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg); +} + +int32_t ctgdShowCacheInfo(void) { + if (!gCTGDebug.cacheEnable) { + return TSDB_CODE_CTG_OUT_OF_SERVICE; + } + + CTG_API_ENTER(); + + SCatalog *pCtg = NULL; + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + ctgdShowClusterCache(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e62819b078..73d0dc2011 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,11 +38,11 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist, int32_t flag, uint64_t *dbId); -extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); + bool *inCache, int32_t flag, uint64_t *dbId); +extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); -extern "C" int32_t ctgDbgEnableDebug(char *option); -extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); +extern "C" int32_t ctgdEnableDebug(char *option); +extern "C" int32_t ctgdGetStatNum(char *option, void *res); void ctgTestSetRspTableMeta(); void ctgTestSetRspCTableMeta(); @@ -140,9 +140,9 @@ void ctgTestInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, "/var/log/taos"); - ctgDbgEnableDebug("api"); - ctgDbgEnableDebug("meta"); - ctgDbgEnableDebug("cache"); + ctgdEnableDebug("api"); + ctgdEnableDebug("meta"); + ctgdEnableDebug("cache"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) { int32_t code = 0; int32_t n = 0; STableMeta *tbMeta = NULL; - int32_t exist = 0; + bool inCache = false; SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL); - if (code || 0 == exist) { + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); + if (code || !inCache) { assert(0); } @@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epSet.numOfEps, 3); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { taosMsleep(50); } @@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { taosMsleep(50); } else { @@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(code, 0); while (true) { - int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); - int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { taosMsleep(50); } else { @@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) { } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) { } } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { taosMsleep(50); } diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h new file mode 100644 index 0000000000..771baca2ab --- /dev/null +++ b/source/libs/command/inc/commandInt.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "nodes.h" +#include "plannodes.h" +#include "ttime.h" + +#define EXPLAIN_MAX_GROUP_NUM 100 + +//newline area +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d" +#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" +#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" +#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" +#define EXPLAIN_ORDER_FORMAT "Order: %s" +#define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " +#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" + +//append area +#define EXPLAIN_GROUPS_FORMAT " groups=%d" +#define EXPLAIN_WIDTH_FORMAT " width=%d" +#define EXPLAIN_LOOPS_FORMAT " loops=%d" +#define EXPLAIN_REVERSE_FORMAT " reverse=%d" + +typedef struct SExplainGroup { + int32_t nodeNum; + SSubplan *plan; + void *execInfo; //TODO +} SExplainGroup; + +typedef struct SExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SExplainResNode; + +typedef struct SQueryExplainRowInfo { + int32_t level; + int32_t len; + char *buf; +} SQueryExplainRowInfo; + +typedef struct SExplainCtx { + int32_t totalSize; + bool verbose; + char *tbuf; + SArray *rows; + SHashObj *groupHash; +} SExplainCtx; + +#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") +#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") + +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) + +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + if (isVerboseLine) { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, ""); \ + } else { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> "); \ + } \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ + } while (0) + +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c new file mode 100644 index 0000000000..847a863e76 --- /dev/null +++ b/source/libs/command/src/explain.c @@ -0,0 +1,687 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "query.h" +#include "plannodes.h" +#include "commandInt.h" + +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); + + +void qFreeExplainResTree(SExplainResNode *res) { + if (NULL == res) { + return; + } + + taosMemoryFreeClear(res->pExecInfo); + + SNode* node = NULL; + FOREACH(node, res->pChildren) { + qFreeExplainResTree((SExplainResNode *)node); + } + nodesClearList(res->pChildren); + + taosMemoryFreeClear(res); +} + +void qFreeExplainCtx(void *ctx) { + if (NULL == ctx) { + return; + } + + SExplainCtx *pCtx = (SExplainCtx *)ctx; + int32_t rowSize = taosArrayGetSize(pCtx->rows); + for (int32_t i = 0; i < rowSize; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + taosMemoryFreeClear(row->buf); + } + + taosHashCleanup(pCtx->groupHash); + taosArrayDestroy(pCtx->rows); + taosMemoryFree(pCtx); +} + +int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) { + int32_t code = 0; + SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); + if (NULL == ctx) { + qError("calloc SExplainCtx failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); + if (NULL == tbuf) { + qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ctx->verbose = verbose; + ctx->tbuf = tbuf; + ctx->rows = rows; + ctx->groupHash = groupHash; + + *pCtx = ctx; + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroy(rows); + taosHashCleanup(groupHash); + taosMemoryFree(ctx); + + QRY_RET(code); +} + + +char *qFillModeString(EFillMode mode) { + switch (mode) { + case FILL_MODE_NONE: + return "none"; + case FILL_MODE_VALUE: + return "value"; + case FILL_MODE_PREV: + return "prev"; + case FILL_MODE_NULL: + return "null"; + case FILL_MODE_LINEAR: + return "linear"; + case FILL_MODE_NEXT: + return "next"; + default: + return "unknown"; + } +} + +char *qGetNameFromColumnNode(SNode *pNode) { + if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) { + return "NULL"; + } + + return ((SColumnNode *)pNode)->colName; +} + +int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { + int32_t tlen = 0; + SNodeList *pPhysiChildren = NULL; + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + pPhysiChildren = pTagScanNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + pPhysiChildren = pTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + pPhysiChildren = pSTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + pPhysiChildren = pPrjNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + pPhysiChildren = pJoinNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + pPhysiChildren = pAggNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + pPhysiChildren = pExchNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + pPhysiChildren = pSortNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + pPhysiChildren = pIntNode->window.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; + pPhysiChildren = pSessNode->window.node.pChildren; + break; + } + default: + qError("not supported physical node type %d", pNode->type); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (pPhysiChildren) { + *pChildren = nodesMakeList(); + if (NULL == *pChildren) { + qError("nodesMakeList failed"); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + SNode* node = NULL; + SExplainResNode *pResNode = NULL; + FOREACH(node, pPhysiChildren) { + QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { + if (NULL == pNode) { + *pRes = NULL; + qError("physical node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode)); + if (NULL == res) { + qError("calloc SPhysiNodeExplainRes failed"); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + int32_t code = 0; + res->pNode = pNode; + res->pExecInfo = pExecInfo; + QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren)); + + *pRes = res; + + return TSDB_CODE_SUCCESS; + +_return: + + qFreeExplainResTree(res); + + QRY_RET(code); +} + +int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { + int32_t tlen = *len; + + EXPLAIN_ROW_APPEND("(exec info here)"); + + *len = tlen; + + return TSDB_CODE_SUCCESS; +} + +int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { + SQueryExplainRowInfo row = {0}; + row.buf = taosMemoryMalloc(len); + if (NULL == row.buf) { + qError("taosMemoryMalloc %d failed", len); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(row.buf, tbuf, len); + row.level = level; + row.len = len; + ctx->totalSize += len; + + if (NULL == taosArrayPush(ctx->rows, &row)) { + qError("taosArrayPush row to explain res rows failed"); + taosMemoryFree(row.buf); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { + int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; + SPhysiNode* pNode = pResNode->pNode; + if (NULL == pNode) { + qError("pyhsical node in explain res node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); + if (pTagScanNode->reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); + if (pTblScanNode->scan.reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); + if (pSTblScanNode->scan.reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pSTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pPrjNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pJoinNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length); + if (pAggNode->pGroupKeys) { + EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pAggNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + if (NULL == group) { + qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pExchNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + + QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1)); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pSortNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + pIntNode->offset, getPrecisionUnit(pIntNode->precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, + pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pIntNode->pFill) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + } + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + default: + qError("not supported physical node type %d", pNode->type); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { + if (NULL == pResNode) { + qError("explain res node is NULL"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t code = 0; + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); + + SNode* pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) { + SExplainResNode *node = NULL; + int32_t code = 0; + SExplainCtx *ctx = (SExplainCtx *)pCtx; + + SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); + if (NULL == group) { + qError("group %d not in groupHash", groupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node)); + + QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); + +_return: + + qFreeExplainResTree(node); + + QRY_RET(code); +} + + +int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { + SExplainCtx *pCtx = (SExplainCtx *)ctx; + int32_t rowNum = taosArrayGetSize(pCtx->rows); + if (rowNum <= 0) { + qError("empty explain res rows"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t colNum = 1; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + rsp->completed = 1; + rsp->numOfRows = htonl(rowNum); + + *(int32_t *)rsp->data = htonl(pCtx->totalSize); + + int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + char *data = (char *)(offset + rowNum); + int32_t tOffset = 0; + + for (int32_t i = 0; i < rowNum; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + *offset = tOffset; + tOffset += row->len; + + memcpy(data, row->buf, row->len); + + ++offset; + data += row->len; + } + + *pRsp = rsp; + + return TSDB_CODE_SUCCESS; +} + +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { + int32_t code = 0; + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SExplainGroup *pGroup = NULL; + void *pCtx = NULL; + int32_t rootGroupId = 0; + + if (pDag->numOfSubplans <= 0) { + qError("invalid subplan num:%d", pDag->numOfSubplans); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); + if (levelNum <= 0) { + qError("invalid level num:%d", levelNum); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == groupHash) { + qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); + + for (int32_t i = 0; i < levelNum; ++i) { + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); + if (NULL == plans) { + qError("empty level plan, level:%d", i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); + if (taskNum <= 0) { + qError("invalid level plan number:%d, level:%d", taskNum, i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SSubplan *plan = NULL; + for (int32_t n = 0; n < taskNum; ++n) { + plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); + if (pGroup) { + ++pGroup->nodeNum; + continue; + } + + SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { + qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (0 == i) { + if (taskNum > 1) { + qError("invalid taskNum %d for level 0", taskNum); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + rootGroupId = plan->id.groupId; + } + + qDebug("level %d group handled, taskNum:%d", i, taskNum); + } + + QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); + + QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp)); + +_return: + + qFreeExplainCtx(pCtx); + + QRY_RET(code); +} + + + diff --git a/source/libs/nodes/inc/nodesUtil.h b/source/libs/nodes/inc/nodesUtil.h index de00b6bca4..976044c16f 100644 --- a/source/libs/nodes/inc/nodesUtil.h +++ b/source/libs/nodes/inc/nodesUtil.h @@ -20,12 +20,16 @@ extern "C" { #endif -#define nodesFatal(param, ...) qFatal("NODES: " param, __VA_ARGS__) -#define nodesError(param, ...) qError("NODES: " param, __VA_ARGS__) -#define nodesWarn(param, ...) qWarn("NODES: " param, __VA_ARGS__) -#define nodesInfo(param, ...) qInfo("NODES: " param, __VA_ARGS__) -#define nodesDebug(param, ...) qDebug("NODES: " param, __VA_ARGS__) -#define nodesTrace(param, ...) qTrace("NODES: " param, __VA_ARGS__) +#define nodesFatal(...) qFatal("NODES: " __VA_ARGS__) +#define nodesError(...) qError("NODES: " __VA_ARGS__) +#define nodesWarn(...) qWarn("NODES: " __VA_ARGS__) +#define nodesInfo(...) qInfo("NODES: " __VA_ARGS__) +#define nodesDebug(...) qDebug("NODES: " __VA_ARGS__) +#define nodesTrace(...) qTrace("NODES: " __VA_ARGS__) + +#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c new file mode 100644 index 0000000000..68f38bb6a7 --- /dev/null +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "cmdnodes.h" +#include "nodesUtil.h" +#include "plannodes.h" +#include "querynodes.h" +#include "taos.h" +#include "taoserror.h" +#include "thash.h" + +char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>", + "IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL", + "IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"}; +char *gLogicConditionStr[] = {"AND", "OR", "NOT"}; + +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { + switch (pNode->type) { + case QUERY_NODE_COLUMN: { + SColumnNode *colNode = (SColumnNode *)pNode; + if (colNode->dbName[0]) { + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName); + } + + if (colNode->tableAlias[0]) { + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias); + } else if (colNode->tableName[0]) { + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName); + } + + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_VALUE:{ + SValueNode *colNode = (SValueNode *)pNode; + char *t = nodesGetStrValueFromNode(colNode); + if (NULL == t) { + nodesError("fail to get str value from valueNode"); + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *len += snprintf(buf + *len, bufSize - *len, "%s", t); + taosMemoryFree(t); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + *len += snprintf(buf + *len, bufSize - *len, "("); + if (pOpNode->pLeft) { + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); + } + + if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { + nodesError("unknown operation type:%d", pOpNode->opType); + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + + if (pOpNode->pRight) { + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); + } + + *len += snprintf(buf + *len, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_LOGIC_CONDITION:{ + SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf + *len, bufSize - *len, "("); + + FOREACH(node, pLogicNode->pParameterList) { + if (!first) { + *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + } + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf + *len, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_FUNCTION:{ + SFunctionNode* pFuncNode = (SFunctionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName); + + FOREACH(node, pFuncNode->pParameterList) { + if (!first) { + *len += snprintf(buf + *len, bufSize - *len, ", "); + } + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf + *len, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_NODE_LIST:{ + SNodeListNode* pListNode = (SNodeListNode *)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf + *len, bufSize - *len, "("); + + FOREACH(node, pListNode->pNodeList) { + if (!first) { + *len += snprintf(buf + *len, bufSize - *len, ", "); + } + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf + *len, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); + NODES_RET(TSDB_CODE_QRY_APP_ERROR); +} diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 7eaa049946..bbd0473edd 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -17,7 +17,8 @@ typedef enum ETraversalOrder { TRAVERSAL_PREORDER = 1, - TRAVERSAL_POSTORDER + TRAVERSAL_INORDER, + TRAVERSAL_POSTORDER, } ETraversalOrder; static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 799d07e5d9..732d9acfbe 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -790,6 +790,71 @@ void* nodesGetValueFromNode(SValueNode *pNode) { return NULL; } +char* nodesGetStrValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); + return buf; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRId64, pNode->datum.i); + return buf; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRIu64, pNode->datum.u); + return buf; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%e", pNode->datum.d); + return buf; + } + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1; + void *buf = taosMemoryMalloc(bufSize); + if (NULL == buf) { + return NULL; + } + + snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p)); + return buf; + } + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a8ef762a73..e3aa0fc0c3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1952,6 +1952,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; + strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 7f6d8826e6..f57f476d51 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // set the output if (TSDB_CODE_SUCCESS == code) { pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); - if (NULL == pJoin->pOnConditions) { + if (NULL == pJoin->node.pTargets) { code = TSDB_CODE_OUT_OF_MEMORY; } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index a9bf0f5594..d50047e592 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( qcom - PRIVATE os util transport + PRIVATE os util transport nodes ) if(${BUILD_TEST}) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 75c1e134cd..f120bf26ce 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -21,7 +21,6 @@ extern "C" { #endif - #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 834a722bd8..3e04e7b30a 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -36,8 +36,6 @@ extern "C" { #define FILTER_DUMMY_EMPTY_OPTR 127 -#define MAX_NUM_STR_SIZE 40 - #define FILTER_RM_UNIT_MIN_ROWS 100 enum { diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index a4a299317c..1a62c7d89d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport + PUBLIC os util nodes planner qcom common catalog transport command ) if(${BUILD_TEST}) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 518da6e2b8..22bd039219 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -142,10 +142,10 @@ typedef struct SSchTask { } SSchTask; typedef struct SSchJobAttr { - bool needFetch; - bool syncSchedule; - bool queryJob; - bool needFlowCtrl; + EExplainMode explainMode; + bool syncSchedule; + bool queryJob; + bool needFlowCtrl; } SSchJobAttr; typedef struct SSchJob { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 61462f1dc3..9c5019aedc 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -19,6 +19,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "command.h" SSchedulerMgmt schMgmt = {0}; @@ -2103,6 +2104,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; pJob->sql = sql; @@ -2136,19 +2138,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD tsem_init(&pJob->rspSem, 0, 0); - pJob->refId = taosAddRef(schMgmt.jobRef, pJob); - if (pJob->refId < 0) { - SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno)); + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_ERR_JRET(terrno); } + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); - schAcquireJob(pJob->refId); - *job = pJob->refId; if (syncSchedule) { @@ -2168,6 +2175,54 @@ _return: SCH_RET(code); } +int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, + bool syncSchedule) { + qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); + + int32_t code = 0; + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); + if (NULL == pJob) { + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->sql = sql; + pJob->attr.queryJob = true; + pJob->attr.explainMode = pDag->explainInfo.mode; + pJob->queryId = pDag->queryId; + pJob->subPlans = pDag->pSubplans; + + SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); + + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); + } + + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + + pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + *job = pJob->refId; + SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + + schReleaseJob(pJob->refId); + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeJobImpl(pJob); + SCH_RET(code); +} + int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobRef) { qError("scheduler already initialized"); @@ -2216,7 +2271,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { + SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); + } else { + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + } SSchJob *job = schAcquireJob(*pJob); @@ -2399,11 +2458,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(pJob)); + if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); + tsem_wait(&pJob->rspSem); + } + } else { + SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - tsem_wait(&pJob->rspSem); - status = SCH_GET_JOB_STATUS(pJob); if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {