xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #11178 from taosdata/feature/qnode 

Feature/qnode
This commit is contained in:
dapan1121 2022-04-01 11:43:12 +08:00 committed by GitHub
parent 3f6e96c66d a60ce69b02
commit 6b09fef912
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1561 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/ttime.h
View File

@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);

5
include/libs/command/command.h
View File

@ -15,5 +15,10 @@
#include "cmdnodes.h"
#include "tmsg.h"
#include "plannodes.h"
int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);

2
include/libs/nodes/nodes.h
View File

@ -215,6 +215,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode);
int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
int32_t nodesStringToList(const char* pStr, SNodeList** pList);
int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
#ifdef __cplusplus
}
#endif

2
include/libs/nodes/plannodes.h
View File

@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode {
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
uint8_t precision;
uint8_t precision;
SFillNode* pFill;
} SIntervalPhysiNode;

1
include/libs/nodes/querynodes.h
View File

@ -314,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery);
bool nodesIsTimelineQuery(const SNode* pQuery);
void* nodesGetValueFromNode(SValueNode *pNode);
char* nodesGetStrValueFromNode(SValueNode *pNode);
#ifdef __cplusplus
}

6
include/libs/qcom/query.h
View File

@ -53,6 +53,7 @@ typedef struct SIndexMeta {
} SIndexMeta;
/*
* ASSERT(sizeof(SCTableMeta) == 24)
* ASSERT(tableType == TSDB_CHILD_TABLE)
@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
} \
} while (0)
#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#ifdef __cplusplus
}
#endif

8
include/os/osTime.h
View File

@ -46,6 +46,14 @@ extern "C" {
#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24)
#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7)
#define NANOSECOND_PER_USEC (1000L)
#define NANOSECOND_PER_MSEC (1000000L)
#define NANOSECOND_PER_SEC (1000000000L)
#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60)
#define NANOSECOND_PER_HOUR (NANOSECOND_PER_MINUTE * 60)
#define NANOSECOND_PER_DAY (NANOSECOND_PER_HOUR * 24)
#define NANOSECOND_PER_WEEK (NANOSECOND_PER_DAY * 7)
int32_t taosGetTimeOfDay(struct timeval *tv);
//@return timestamp in second

5
include/util/tdef.h
View File

@ -224,6 +224,7 @@ typedef enum ELogicConditionType {
#define TSDB_APP_NAME_LEN TSDB_UNI_LEN
#define TSDB_STB_COMMENT_LEN 1024
/**
* In some scenarios uint16_t (0~65535) is used to store the row len.
* - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
@ -388,6 +389,7 @@ typedef enum ELogicConditionType {
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
#define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5
@ -479,6 +481,9 @@ enum {
#define QND_VGID 1
#define VND_VGID 0
#define MAX_NUM_STR_SIZE 40
#ifdef __cplusplus
}
#endif

41
source/common/src/ttime.c
View File

@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
return 0;
}
char getPrecisionUnit(int32_t precision) {
static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
switch (precision) {
case TSDB_TIME_PRECISION_MILLI:
case TSDB_TIME_PRECISION_MICRO:
case TSDB_TIME_PRECISION_NANO:
return units[precision];
default:
return 0;
}
}
int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
fromPrecision == TSDB_TIME_PRECISION_NANO);
@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec
return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
}
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
fromPrecision == TSDB_TIME_PRECISION_NANO);
static double factors[3] = {1000000., 1000., 1.};
switch (toUnit) {
case 's':
return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
case 'm':
return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
case 'h':
return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
case 'd':
return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
case 'w':
return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
case 'a':
return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
case 'u':
return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
case 'b':
return time * factors[fromPrecision];
default: {
return -1;
}
}
}
static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
switch (unit) {
case 's':
@ -688,4 +727,4 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
tstrncpy(buf, ts, bufLen);
}
}

23
source/libs/catalog/inc/catalogInt.h
View File

@ -112,7 +112,14 @@ typedef struct SCtgRuntimeStat {
} SCtgRuntimeStat;
typedef struct SCtgCacheStat {
uint64_t clusterNum;
uint64_t dbNum;
uint64_t tblNum;
uint64_t stblNum;
uint64_t vgHitNum;
uint64_t vgMissNum;
uint64_t tblHitNum;
uint64_t tblMissNum;
} SCtgCacheStat;
typedef struct SCatalogStat {
@ -186,7 +193,7 @@ typedef struct SCatalogMgmt {
bool exit;
SRWLatch lock;
SCtgQueue queue;
TdThread updateThread;
TdThread updateThread;
SHashObj *pCluster; //key: clusterId, value: SCatalog*
SCatalogStat stat;
SCatalogCfg cfg;
@ -204,8 +211,13 @@ typedef struct SCtgAction {
#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1)
#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1)
#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n)
#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n)
#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n))
#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n))
#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n))
#define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
#define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
@ -291,6 +303,9 @@ typedef struct SCtgAction {
#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0)
extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
extern void ctgdShowClusterCache(SCatalog* pCtg);
extern int32_t ctgdShowCacheInfo(void);
#ifdef __cplusplus
}

353
source/libs/catalog/src/catalog.c
View File

@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action);
int32_t ctgActRemoveStb(SCtgMetaAction *action);
int32_t ctgActRemoveTbl(SCtgMetaAction *action);
extern SCtgDebug gCTGDebug;
SCatalogMgmt gCtgMgmt = {0};
SCtgDebug gCTGDebug = {0};
SCtgAction gCtgAction[CTG_ACT_MAX] = {{
CTG_ACT_UPDATE_VG,
"update vgInfo",
@ -53,182 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{
}
};
int32_t ctgDbgEnableDebug(char *option) {
if (0 == strcasecmp(option, "lock")) {
gCTGDebug.lockEnable = true;
qDebug("lock debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "cache")) {
gCTGDebug.cacheEnable = true;
qDebug("cache debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "api")) {
gCTGDebug.apiEnable = true;
qDebug("api debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "meta")) {
gCTGDebug.metaEnable = true;
qDebug("api debug enabled");
return TSDB_CODE_SUCCESS;
}
qError("invalid debug option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t ctgDbgGetStatNum(char *option, void *res) {
if (0 == strcasecmp(option, "runtime.qDoneNum")) {
*(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
return TSDB_CODE_SUCCESS;
}
qError("invalid stat option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) {
return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
}
int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) {
return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
}
int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) {
int32_t num = 0;
for (uint16_t i = 0; i < rent->slotNum; ++i) {
SCtgRentSlot *slot = &rent->slots[i];
if (NULL == slot->meta) {
continue;
}
num += taosArrayGetSize(slot->meta);
}
return num;
}
int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
if (NULL == pCtg || NULL == pCtg->dbCache) {
return 0;
}
switch (type) {
case CTG_DBG_DB_NUM:
return (int32_t)taosHashGetSize(pCtg->dbCache);
case CTG_DBG_DB_RENT_NUM:
return ctgDbgGetRentNum(&pCtg->dbRent);
case CTG_DBG_STB_RENT_NUM:
return ctgDbgGetRentNum(&pCtg->stbRent);
default:
break;
}
SCtgDBCache *dbCache = NULL;
int32_t num = 0;
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
dbCache = (SCtgDBCache *)pIter;
switch (type) {
case CTG_DBG_META_NUM:
num += ctgDbgGetTbMetaNum(dbCache);
break;
case CTG_DBG_STB_NUM:
num += ctgDbgGetStbNum(dbCache);
break;
default:
ctgError("invalid type:%d", type);
break;
}
pIter = taosHashIterate(pCtg->dbCache, pIter);
}
return num;
}
void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
if (!gCTGDebug.metaEnable) {
return;
}
STableComInfo *c = &p->tableInfo;
if (TSDB_CHILD_TABLE == p->tableType) {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
return;
} else {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
for (int32_t i = 0; i < colNum; ++i) {
SSchema *s = &p->schema[i];
ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes);
}
}
void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
if (NULL == dbHash || !gCTGDebug.cacheEnable) {
return;
}
int32_t i = 0;
SCtgDBCache *dbCache = NULL;
void *pIter = taosHashIterate(dbHash, NULL);
while (pIter) {
char *dbFName = NULL;
size_t len = 0;
dbCache = (SCtgDBCache *)pIter;
dbFName = taosHashGetKey(pIter, &len);
int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
int32_t hashMethod = -1;
int32_t vgNum = 0;
if (dbCache->vgInfo) {
vgVersion = dbCache->vgInfo->vgVersion;
hashMethod = dbCache->vgInfo->hashMethod;
if (dbCache->vgInfo->vgHash) {
vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
}
}
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
pIter = taosHashIterate(dbHash, pIter);
}
}
void ctgDbgShowClusterCache(SCatalog* pCtg) {
if (!gCTGDebug.cacheEnable || NULL == pCtg) {
return;
}
ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgDbgShowDBCache(pCtg, pCtg->dbCache);
}
void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
if (NULL == mgmt->slots) {
return;
@ -249,15 +73,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) {
CTG_LOCK(CTG_WRITE, &cache->stbLock);
if (cache->stbCache) {
int32_t stblNum = taosHashGetSize(cache->stbCache);
taosHashCleanup(cache->stbCache);
cache->stbCache = NULL;
CTG_CACHE_STAT_SUB(stblNum, stblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
CTG_LOCK(CTG_WRITE, &cache->metaLock);
if (cache->metaCache) {
int32_t tblNum = taosHashGetSize(cache->metaCache);
taosHashCleanup(cache->metaCache);
cache->metaCache = NULL;
CTG_CACHE_STAT_SUB(tblNum, tblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->metaLock);
}
@ -293,6 +121,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->stbRent);
if (pCtg->dbCache) {
int32_t dbNum = taosHashGetSize(pCtg->dbCache);
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
@ -305,6 +135,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
}
taosHashCleanup(pCtg->dbCache);
CTG_CACHE_STAT_SUB(dbNum, dbNum);
}
taosMemoryFree(pCtg);
@ -361,7 +193,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) {
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_QUEUE_ADD();
CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum);
CTG_RUNTIME_STAT_ADD(qNum, 1);
tsem_post(&gCtgMgmt.queue.reqSem);
@ -620,34 +452,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache)
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) {
SCtgDBCache *dbCache = NULL;
if (NULL == pCtg->dbCache) {
*pCache = NULL;
*inCache = false;
ctgWarn("empty db cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
ctgDebug("empty db cache, dbFName:%s", dbFName);
goto _return;
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
*pCache = NULL;
*inCache = false;
return TSDB_CODE_SUCCESS;
ctgDebug("db %s not in cache", dbFName);
goto _return;
}
ctgAcquireVgInfo(pCtg, dbCache, inCache);
if (!(*inCache)) {
ctgReleaseDBCache(pCtg, dbCache);
*pCache = NULL;
return TSDB_CODE_SUCCESS;
ctgDebug("vgInfo of db %s not in cache", dbFName);
goto _return;
}
*pCache = dbCache;
*inCache = true;
CTG_CACHE_STAT_ADD(vgHitNum, 1);
ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
_return:
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
*pCache = NULL;
*inCache = false;
CTG_CACHE_STAT_ADD(vgMissNum, 1);
return TSDB_CODE_SUCCESS;
}
@ -763,11 +606,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName,
}
int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) {
int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) {
if (NULL == pCtg->dbCache) {
*exist = 0;
ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname);
return TSDB_CODE_SUCCESS;
ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
goto _return;
}
char dbFName[TSDB_DB_FNAME_LEN] = {0};
@ -782,8 +624,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
ctgDebug("db %s not in cache", pTableName->tname);
goto _return;
}
int32_t sz = 0;
@ -792,13 +634,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == *pTableMeta) {
*exist = 0;
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
goto _return;
}
*exist = 1;
if (dbId) {
*dbId = dbCache->dbId;
}
@ -808,6 +648,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
*inCache = true;
CTG_CACHE_STAT_ADD(tblHitNum, 1);
return TSDB_CODE_SUCCESS;
}
@ -819,8 +663,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
ctgReleaseDBCache(pCtg, dbCache);
ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid);
taosMemoryFreeClear(*pTableMeta);
*exist = 0;
return TSDB_CODE_SUCCESS;
goto _return;
}
if ((*stbMeta)->suid != tbMeta->suid) {
@ -846,8 +689,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
ctgReleaseDBCache(pCtg, dbCache);
*inCache = true;
CTG_CACHE_STAT_ADD(tblHitNum, 1);
ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
_return:
*inCache = false;
CTG_CACHE_STAT_ADD(tblMissNum, 1);
return TSDB_CODE_SUCCESS;
}
@ -1377,6 +1230,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(dbNum, 1);
SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
@ -1436,6 +1291,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) {
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
CTG_CACHE_STAT_SUB(dbNum, 1);
ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
return TSDB_CODE_SUCCESS;
@ -1568,6 +1425,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
}
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
@ -1594,8 +1453,12 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (NULL == orig) {
CTG_CACHE_STAT_ADD(tblNum, 1);
}
ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
ctgDbgShowTableMeta(pCtg, tbName, meta);
ctgdShowTableMeta(pCtg, tbName, meta);
if (!isStb) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
@ -1615,6 +1478,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(stblNum, 1);
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
@ -1874,7 +1739,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t exist = 0;
bool inCache = false;
int32_t code = 0;
uint64_t dbId = 0;
uint64_t suid = 0;
@ -1884,11 +1749,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
CTG_FLAG_SET_INF_DB(flag);
}
CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId));
CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId));
int32_t tbType = 0;
if (exist) {
if (inCache) {
if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) {
goto _return;
}
@ -1930,8 +1795,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
SName stbName = *pTableName;
strcpy(stbName.tname, output->tbName);
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL));
if (0 == exist) {
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL));
if (!inCache) {
ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
continue;
}
@ -1943,7 +1808,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
_return:
if (CTG_TABLE_NOT_EXIST(code) && exist) {
if (CTG_TABLE_NOT_EXIST(code) && inCache) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_INF_DB(flag)) {
strcpy(dbFName, pTableName->dbname);
@ -1962,7 +1827,7 @@ _return:
if (*pTableMeta) {
ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType);
ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
}
CTG_RET(code);
@ -2075,12 +1940,16 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) {
CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) {
ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) {
ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
}
} else {
CTG_CACHE_STAT_SUB(tblNum, 1);
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
@ -2119,6 +1988,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
} else {
CTG_CACHE_STAT_SUB(tblNum, 1);
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
@ -2140,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) {
CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
while (true) {
tsem_wait(&gCtgMgmt.queue.reqSem);
if (tsem_wait(&gCtgMgmt.queue.reqSem)) {
qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
tsem_post(&gCtgMgmt.queue.rspSem);
@ -2161,9 +2034,9 @@ void* ctgUpdateThreadFunc(void* param) {
tsem_post(&gCtgMgmt.queue.rspSem);
}
CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum);
CTG_RUNTIME_STAT_ADD(qDoneNum, 1);
ctgDbgShowClusterCache(pCtg);
ctgdShowClusterCache(pCtg);
}
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
@ -2304,8 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) {
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
tsem_init(&gCtgMgmt.queue.reqSem, 0, 0);
tsem_init(&gCtgMgmt.queue.rspSem, 0, 0);
if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) {
qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
}
if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) {
qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
}
gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode));
if (NULL == gCtgMgmt.queue.head) {
@ -2383,6 +2263,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
}
*catalogHandle = clusterCtg;
CTG_CACHE_STAT_ADD(clusterNum, 1);
return TSDB_CODE_SUCCESS;
@ -2403,10 +2285,12 @@ void catalogFreeHandle(SCatalog* pCtg) {
return;
}
CTG_CACHE_STAT_SUB(clusterNum, 1);
uint64_t clusterId = pCtg->clusterId;
ctgFreeHandle(pCtg);
ctgInfo("handle freed, culsterId:%"PRIx64, clusterId);
}
@ -2417,24 +2301,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
if (NULL == pCtg->dbCache) {
*version = CTG_DEFAULT_INVALID_VERSION;
ctgInfo("empty db cache, dbFName:%s", dbFName);
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
*version = CTG_DEFAULT_INVALID_VERSION;
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
bool inCache = false;
ctgAcquireVgInfo(pCtg, dbCache, &inCache);
if (!inCache) {
ctgReleaseDBCache(pCtg, dbCache);
int32_t code = 0;
CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache));
if (!inCache) {
*version = CTG_DEFAULT_INVALID_VERSION;
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
@ -2449,6 +2321,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version);
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
_return:
CTG_API_LEAVE(code);
}
int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) {
@ -2549,11 +2425,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) {
}
STableMeta *tblMeta = NULL;
int32_t exist = 0;
bool inCache = false;
uint64_t dbId = 0;
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId));
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId));
if (0 == exist) {
if (!inCache) {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
goto _return;
}
@ -2851,8 +2727,13 @@ void catalogDestroy(void) {
atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
tsem_post(&gCtgMgmt.queue.reqSem);
tsem_post(&gCtgMgmt.queue.rspSem);
if (tsem_post(&gCtgMgmt.queue.reqSem)) {
qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
if (tsem_post(&gCtgMgmt.queue.rspSem)) {
qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
while (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
taosUsleep(1);

222
source/libs/catalog/src/catalogDbg.c Normal file
View File

@ -0,0 +1,222 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {0};
int32_t ctgdEnableDebug(char *option) {
if (0 == strcasecmp(option, "lock")) {
gCTGDebug.lockEnable = true;
qDebug("lock debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "cache")) {
gCTGDebug.cacheEnable = true;
qDebug("cache debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "api")) {
gCTGDebug.apiEnable = true;
qDebug("api debug enabled");
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "meta")) {
gCTGDebug.metaEnable = true;
qDebug("api debug enabled");
return TSDB_CODE_SUCCESS;
}
qError("invalid debug option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t ctgdGetStatNum(char *option, void *res) {
if (0 == strcasecmp(option, "runtime.qDoneNum")) {
*(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
return TSDB_CODE_SUCCESS;
}
qError("invalid stat option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
}
int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
}
int32_t ctgdGetRentNum(SCtgRentMgmt *rent) {
int32_t num = 0;
for (uint16_t i = 0; i < rent->slotNum; ++i) {
SCtgRentSlot *slot = &rent->slots[i];
if (NULL == slot->meta) {
continue;
}
num += taosArrayGetSize(slot->meta);
}
return num;
}
int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
if (NULL == pCtg || NULL == pCtg->dbCache) {
return 0;
}
switch (type) {
case CTG_DBG_DB_NUM:
return (int32_t)taosHashGetSize(pCtg->dbCache);
case CTG_DBG_DB_RENT_NUM:
return ctgdGetRentNum(&pCtg->dbRent);
case CTG_DBG_STB_RENT_NUM:
return ctgdGetRentNum(&pCtg->stbRent);
default:
break;
}
SCtgDBCache *dbCache = NULL;
int32_t num = 0;
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
dbCache = (SCtgDBCache *)pIter;
switch (type) {
case CTG_DBG_META_NUM:
num += ctgdGetTbMetaNum(dbCache);
break;
case CTG_DBG_STB_NUM:
num += ctgdGetStbNum(dbCache);
break;
default:
ctgError("invalid type:%d", type);
break;
}
pIter = taosHashIterate(pCtg->dbCache, pIter);
}
return num;
}
void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
if (!gCTGDebug.metaEnable) {
return;
}
STableComInfo *c = &p->tableInfo;
if (TSDB_CHILD_TABLE == p->tableType) {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
return;
} else {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
for (int32_t i = 0; i < colNum; ++i) {
SSchema *s = &p->schema[i];
ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes);
}
}
void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
if (NULL == dbHash || !gCTGDebug.cacheEnable) {
return;
}
int32_t i = 0;
SCtgDBCache *dbCache = NULL;
void *pIter = taosHashIterate(dbHash, NULL);
while (pIter) {
char *dbFName = NULL;
size_t len = 0;
dbCache = (SCtgDBCache *)pIter;
dbFName = taosHashGetKey(pIter, &len);
int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
int32_t hashMethod = -1;
int32_t vgNum = 0;
if (dbCache->vgInfo) {
vgVersion = dbCache->vgInfo->vgVersion;
hashMethod = dbCache->vgInfo->hashMethod;
if (dbCache->vgInfo->vgHash) {
vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
}
}
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
pIter = taosHashIterate(dbHash, pIter);
}
}
void ctgdShowClusterCache(SCatalog* pCtg) {
if (!gCTGDebug.cacheEnable || NULL == pCtg) {
return;
}
ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgdShowDBCache(pCtg, pCtg->dbCache);
ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg);
}
int32_t ctgdShowCacheInfo(void) {
if (!gCTGDebug.cacheEnable) {
return TSDB_CODE_CTG_OUT_OF_SERVICE;
}
CTG_API_ENTER();
SCatalog *pCtg = NULL;
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog **)pIter;
if (pCtg) {
ctgdShowClusterCache(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
}
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}

90
source/libs/catalog/test/catalogTests.cpp
View File

@ -38,11 +38,11 @@
namespace {
extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
int32_t *exist, int32_t flag, uint64_t *dbId);
extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
bool *inCache, int32_t flag, uint64_t *dbId);
extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action);
extern "C" int32_t ctgDbgEnableDebug(char *option);
extern "C" int32_t ctgDbgGetStatNum(char *option, void *res);
extern "C" int32_t ctgdEnableDebug(char *option);
extern "C" int32_t ctgdGetStatNum(char *option, void *res);
void ctgTestSetRspTableMeta();
void ctgTestSetRspCTableMeta();
@ -140,9 +140,9 @@ void ctgTestInitLogFile() {
qDebugFlag = 159;
strcpy(tsLogDir, "/var/log/taos");
ctgDbgEnableDebug("api");
ctgDbgEnableDebug("meta");
ctgDbgEnableDebug("cache");
ctgdEnableDebug("api");
ctgdEnableDebug("meta");
ctgdEnableDebug("cache");
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) {
int32_t code = 0;
int32_t n = 0;
STableMeta *tbMeta = NULL;
int32_t exist = 0;
bool inCache = false;
SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename);
while (!ctgTestStop) {
code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL);
if (code || 0 == exist) {
code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL);
if (code || !inCache) {
assert(0);
}
@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
taosMsleep(50);
}
@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
taosMsleep(50);
} else {
@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
taosMsleep(50);
} else {
@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
taosMsleep(50);
} else {
@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (2 != n) {
taosMsleep(50);
} else {
@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
taosMsleep(50);
} else {
@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(code, 0);
while (true) {
int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
if (n || m) {
taosMsleep(50);
} else {
@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) {
}
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
taosMsleep(50);
} else {
@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
taosMsleep(50);
} else {
@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) {
}
}
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFreeClear(tableMeta);
while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
taosMsleep(50);
}
@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n > 0) {
break;
}
@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
taosMsleep(50);
} else {
@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
taosMsleep(50);
}

99
source/libs/command/inc/commandInt.h Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QUERY_INT_H_
#define _TD_QUERY_INT_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "nodes.h"
#include "plannodes.h"
#include "ttime.h"
#define EXPLAIN_MAX_GROUP_NUM 100
//newline area
#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d"
#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d"
#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d"
#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d"
#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d"
#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d"
#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d"
#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d"
#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d"
#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d"
#define EXPLAIN_ORDER_FORMAT "Order: %s"
#define EXPLAIN_FILTER_FORMAT "Filter: "
#define EXPLAIN_FILL_FORMAT "Fill: %s"
#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
//append area
#define EXPLAIN_GROUPS_FORMAT " groups=%d"
#define EXPLAIN_WIDTH_FORMAT " width=%d"
#define EXPLAIN_LOOPS_FORMAT " loops=%d"
#define EXPLAIN_REVERSE_FORMAT " reverse=%d"
typedef struct SExplainGroup {
int32_t nodeNum;
SSubplan *plan;
void *execInfo; //TODO
} SExplainGroup;
typedef struct SExplainResNode {
SNodeList* pChildren;
SPhysiNode* pNode;
void* pExecInfo;
} SExplainResNode;
typedef struct SQueryExplainRowInfo {
int32_t level;
int32_t len;
char *buf;
} SQueryExplainRowInfo;
typedef struct SExplainCtx {
int32_t totalSize;
bool verbose;
char *tbuf;
SArray *rows;
SHashObj *groupHash;
} SExplainCtx;
#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending")
#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u)))
#define EXPLAIN_ROW_NEW(level, ...) \
do { \
if (isVerboseLine) { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, ""); \
} else { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> "); \
} \
tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \
} while (0)
#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)
#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
#ifdef __cplusplus
}
#endif
#endif /*_TD_QUERY_INT_H_*/

687
source/libs/command/src/explain.c Normal file
View File

@ -0,0 +1,687 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "query.h"
#include "plannodes.h"
#include "commandInt.h"
int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes);
int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level);
void qFreeExplainResTree(SExplainResNode *res) {
if (NULL == res) {
return;
}
taosMemoryFreeClear(res->pExecInfo);
SNode* node = NULL;
FOREACH(node, res->pChildren) {
qFreeExplainResTree((SExplainResNode *)node);
}
nodesClearList(res->pChildren);
taosMemoryFreeClear(res);
}
void qFreeExplainCtx(void *ctx) {
if (NULL == ctx) {
return;
}
SExplainCtx *pCtx = (SExplainCtx *)ctx;
int32_t rowSize = taosArrayGetSize(pCtx->rows);
for (int32_t i = 0; i < rowSize; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
taosMemoryFreeClear(row->buf);
}
taosHashCleanup(pCtx->groupHash);
taosArrayDestroy(pCtx->rows);
taosMemoryFree(pCtx);
}
int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) {
int32_t code = 0;
SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx));
if (NULL == ctx) {
qError("calloc SExplainCtx failed");
QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo));
if (NULL == rows) {
qError("taosArrayInit SQueryExplainRowInfo failed");
QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE);
if (NULL == tbuf) {
qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE);
QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
ctx->verbose = verbose;
ctx->tbuf = tbuf;
ctx->rows = rows;
ctx->groupHash = groupHash;
*pCtx = ctx;
return TSDB_CODE_SUCCESS;
_return:
taosArrayDestroy(rows);
taosHashCleanup(groupHash);
taosMemoryFree(ctx);
QRY_RET(code);
}
char *qFillModeString(EFillMode mode) {
switch (mode) {
case FILL_MODE_NONE:
return "none";
case FILL_MODE_VALUE:
return "value";
case FILL_MODE_PREV:
return "prev";
case FILL_MODE_NULL:
return "null";
case FILL_MODE_LINEAR:
return "linear";
case FILL_MODE_NEXT:
return "next";
default:
return "unknown";
}
}
char *qGetNameFromColumnNode(SNode *pNode) {
if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) {
return "NULL";
}
return ((SColumnNode *)pNode)->colName;
}
int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) {
int32_t tlen = 0;
SNodeList *pPhysiChildren = NULL;
switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
pPhysiChildren = pTagScanNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
pPhysiChildren = pTblScanNode->scan.node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
pPhysiChildren = pSTblScanNode->scan.node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
pPhysiChildren = pPrjNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
pPhysiChildren = pJoinNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_AGG:{
SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
pPhysiChildren = pAggNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
pPhysiChildren = pExchNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT:{
SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
pPhysiChildren = pSortNode->node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
pPhysiChildren = pIntNode->window.node.pChildren;
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode;
pPhysiChildren = pSessNode->window.node.pChildren;
break;
}
default:
qError("not supported physical node type %d", pNode->type);
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (pPhysiChildren) {
*pChildren = nodesMakeList();
if (NULL == *pChildren) {
qError("nodesMakeList failed");
QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
SNode* node = NULL;
SExplainResNode *pResNode = NULL;
FOREACH(node, pPhysiChildren) {
QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode));
QRY_ERR_RET(nodesListAppend(*pChildren, pResNode));
}
return TSDB_CODE_SUCCESS;
}
int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) {
if (NULL == pNode) {
*pRes = NULL;
qError("physical node is NULL");
return TSDB_CODE_QRY_APP_ERROR;
}
SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode));
if (NULL == res) {
qError("calloc SPhysiNodeExplainRes failed");
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
int32_t code = 0;
res->pNode = pNode;
res->pExecInfo = pExecInfo;
QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren));
*pRes = res;
return TSDB_CODE_SUCCESS;
_return:
qFreeExplainResTree(res);
QRY_RET(code);
}
int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) {
int32_t tlen = *len;
EXPLAIN_ROW_APPEND("(exec info here)");
*len = tlen;
return TSDB_CODE_SUCCESS;
}
int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) {
SQueryExplainRowInfo row = {0};
row.buf = taosMemoryMalloc(len);
if (NULL == row.buf) {
qError("taosMemoryMalloc %d failed", len);
QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
memcpy(row.buf, tbuf, len);
row.level = level;
row.len = len;
ctx->totalSize += len;
if (NULL == taosArrayPush(ctx->rows, &row)) {
qError("taosArrayPush row to explain res rows failed");
taosMemoryFree(row.buf);
QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS;
}
int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
int32_t tlen = 0;
bool isVerboseLine = false;
char *tbuf = ctx->tbuf;
bool verbose = ctx->verbose;
SPhysiNode* pNode = pResNode->pNode;
if (NULL == pNode) {
qError("pyhsical node in explain res node is NULL");
return TSDB_CODE_QRY_APP_ERROR;
}
switch (pNode->type) {
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count);
if (pTagScanNode->reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count);
if (pTblScanNode->scan.reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pTblScanNode->scan.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count);
if (pSTblScanNode->scan.reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pSTblScanNode->scan.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pPrjNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_AGG:{
SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length);
if (pAggNode->pGroupKeys) {
EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length);
}
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pAggNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId));
if (NULL == group) {
qError("exchange src group %d not in groupHash", pExchNode->srcGroupId);
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pExchNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1));
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT:{
SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pSortNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit,
pIntNode->offset, getPrecisionUnit(pIntNode->precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit,
pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pIntNode->pFill) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (pIntNode->window.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
if (pIntNode->window.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
default:
qError("not supported physical node type %d", pNode->type);
return TSDB_CODE_QRY_APP_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
if (NULL == pResNode) {
qError("explain res node is NULL");
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t code = 0;
QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level));
SNode* pNode = NULL;
FOREACH(pNode, pResNode->pChildren) {
QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1));
}
return TSDB_CODE_SUCCESS;
}
int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) {
SExplainResNode *node = NULL;
int32_t code = 0;
SExplainCtx *ctx = (SExplainCtx *)pCtx;
SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId));
if (NULL == group) {
qError("group %d not in groupHash", groupId);
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node));
QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level));
_return:
qFreeExplainResTree(node);
QRY_RET(code);
}
int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SExplainCtx *pCtx = (SExplainCtx *)ctx;
int32_t rowNum = taosArrayGetSize(pCtx->rows);
if (rowNum <= 0) {
qError("empty explain res rows");
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t colNum = 1;
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
rsp->completed = 1;
rsp->numOfRows = htonl(rowNum);
*(int32_t *)rsp->data = htonl(pCtx->totalSize);
int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t));
char *data = (char *)(offset + rowNum);
int32_t tOffset = 0;
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
*offset = tOffset;
tOffset += row->len;
memcpy(data, row->buf, row->len);
++offset;
data += row->len;
}
*pRsp = rsp;
return TSDB_CODE_SUCCESS;
}
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
int32_t code = 0;
SNodeListNode *plans = NULL;
int32_t taskNum = 0;
SExplainGroup *pGroup = NULL;
void *pCtx = NULL;
int32_t rootGroupId = 0;
if (pDag->numOfSubplans <= 0) {
qError("invalid subplan num:%d", pDag->numOfSubplans);
QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans);
if (levelNum <= 0) {
qError("invalid level num:%d", levelNum);
QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (NULL == groupHash) {
qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM);
QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose));
for (int32_t i = 0; i < levelNum; ++i) {
plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
if (NULL == plans) {
qError("empty level plan, level:%d", i);
QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
taskNum = (int32_t)LIST_LENGTH(plans->pNodeList);
if (taskNum <= 0) {
qError("invalid level plan number:%d, level:%d", taskNum, i);
QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
SSubplan *plan = NULL;
for (int32_t n = 0; n < taskNum; ++n) {
plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId));
if (pGroup) {
++pGroup->nodeNum;
continue;
}
SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL};
if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) {
qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n);
QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
if (0 == i) {
if (taskNum > 1) {
qError("invalid taskNum %d for level 0", taskNum);
QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
rootGroupId = plan->id.groupId;
}
qDebug("level %d group handled, taskNum:%d", i, taskNum);
}
QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0));
QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp));
_return:
qFreeExplainCtx(pCtx);
QRY_RET(code);
}

16
source/libs/nodes/inc/nodesUtil.h
View File

@ -20,12 +20,16 @@
extern "C" {
#endif
#define nodesFatal(param, ...) qFatal("NODES: " param, __VA_ARGS__)
#define nodesError(param, ...) qError("NODES: " param, __VA_ARGS__)
#define nodesWarn(param, ...) qWarn("NODES: " param, __VA_ARGS__)
#define nodesInfo(param, ...) qInfo("NODES: " param, __VA_ARGS__)
#define nodesDebug(param, ...) qDebug("NODES: " param, __VA_ARGS__)
#define nodesTrace(param, ...) qTrace("NODES: " param, __VA_ARGS__)
#define nodesFatal(...) qFatal("NODES: " __VA_ARGS__)
#define nodesError(...) qError("NODES: " __VA_ARGS__)
#define nodesWarn(...) qWarn("NODES: " __VA_ARGS__)
#define nodesInfo(...) qInfo("NODES: " __VA_ARGS__)
#define nodesDebug(...) qDebug("NODES: " __VA_ARGS__)
#define nodesTrace(...) qTrace("NODES: " __VA_ARGS__)
#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#ifdef __cplusplus
}

145
source/libs/nodes/src/nodesToSQLFuncs.c Normal file
View File

@ -0,0 +1,145 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cmdnodes.h"
#include "nodesUtil.h"
#include "plannodes.h"
#include "querynodes.h"
#include "taos.h"
#include "taoserror.h"
#include "thash.h"
char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>",
"IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL",
"IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"};
char *gLogicConditionStr[] = {"AND", "OR", "NOT"};
int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
switch (pNode->type) {
case QUERY_NODE_COLUMN: {
SColumnNode *colNode = (SColumnNode *)pNode;
if (colNode->dbName[0]) {
*len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName);
}
if (colNode->tableAlias[0]) {
*len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias);
} else if (colNode->tableName[0]) {
*len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName);
}
*len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName);
return TSDB_CODE_SUCCESS;
}
case QUERY_NODE_VALUE:{
SValueNode *colNode = (SValueNode *)pNode;
char *t = nodesGetStrValueFromNode(colNode);
if (NULL == t) {
nodesError("fail to get str value from valueNode");
NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
*len += snprintf(buf + *len, bufSize - *len, "%s", t);
taosMemoryFree(t);
return TSDB_CODE_SUCCESS;
}
case QUERY_NODE_OPERATOR: {
SOperatorNode* pOpNode = (SOperatorNode*)pNode;
*len += snprintf(buf + *len, bufSize - *len, "(");
if (pOpNode->pLeft) {
NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len));
}
if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) {
nodesError("unknown operation type:%d", pOpNode->opType);
NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
*len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]);
if (pOpNode->pRight) {
NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len));
}
*len += snprintf(buf + *len, bufSize - *len, ")");
return TSDB_CODE_SUCCESS;
}
case QUERY_NODE_LOGIC_CONDITION:{
SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode;
SNode* node = NULL;
bool first = true;
*len += snprintf(buf + *len, bufSize - *len, "(");
FOREACH(node, pLogicNode->pParameterList) {
if (!first) {
*len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]);
}
NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
first = false;
}
*len += snprintf(buf + *len, bufSize - *len, ")");
return TSDB_CODE_SUCCESS;
}
case QUERY_NODE_FUNCTION:{
SFunctionNode* pFuncNode = (SFunctionNode*)pNode;
SNode* node = NULL;
bool first = true;
*len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName);
FOREACH(node, pFuncNode->pParameterList) {
if (!first) {
*len += snprintf(buf + *len, bufSize - *len, ", ");
}
NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
first = false;
}
*len += snprintf(buf + *len, bufSize - *len, ")");
return TSDB_CODE_SUCCESS;
}
case QUERY_NODE_NODE_LIST:{
SNodeListNode* pListNode = (SNodeListNode *)pNode;
SNode* node = NULL;
bool first = true;
*len += snprintf(buf + *len, bufSize - *len, "(");
FOREACH(node, pListNode->pNodeList) {
if (!first) {
*len += snprintf(buf + *len, bufSize - *len, ", ");
}
NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
first = false;
}
*len += snprintf(buf + *len, bufSize - *len, ")");
return TSDB_CODE_SUCCESS;
}
default:
break;
}
nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type));
NODES_RET(TSDB_CODE_QRY_APP_ERROR);
}

3
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -17,7 +17,8 @@
typedef enum ETraversalOrder {
TRAVERSAL_PREORDER = 1,
TRAVERSAL_POSTORDER
TRAVERSAL_INORDER,
TRAVERSAL_POSTORDER,
} ETraversalOrder;
static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext);

65
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -790,6 +790,71 @@ void* nodesGetValueFromNode(SValueNode *pNode) {
return NULL;
}
char* nodesGetStrValueFromNode(SValueNode *pNode) {
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_BOOL: {
void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
if (NULL == buf) {
return NULL;
}
sprintf(buf, "%s", pNode->datum.b ? "true" : "false");
return buf;
}
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
if (NULL == buf) {
return NULL;
}
sprintf(buf, "%" PRId64, pNode->datum.i);
return buf;
}
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT: {
void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
if (NULL == buf) {
return NULL;
}
sprintf(buf, "%" PRIu64, pNode->datum.u);
return buf;
}
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE: {
void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
if (NULL == buf) {
return NULL;
}
sprintf(buf, "%e", pNode->datum.d);
return buf;
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY: {
int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1;
void *buf = taosMemoryMalloc(bufSize);
if (NULL == buf) {
return NULL;
}
snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p));
return buf;
}
default:
break;
}
return NULL;
}
bool nodesIsExprNode(const SNode* pNode) {
ENodeType type = nodeType(pNode);
return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);

1
source/libs/parser/src/parTranslater.c
View File

@ -1952,6 +1952,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema)
}
(*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME);
return TSDB_CODE_SUCCESS;
}

2
source/libs/planner/src/planLogicCreater.c
View File

@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
// set the output
if (TSDB_CODE_SUCCESS == code) {
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
if (NULL == pJoin->pOnConditions) {
if (NULL == pJoin->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (TSDB_CODE_SUCCESS == code) {

2
source/libs/qcom/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
qcom
PRIVATE os util transport
PRIVATE os util transport nodes
)
if(${BUILD_TEST})

1
source/libs/qcom/inc/queryInt.h
View File

@ -21,7 +21,6 @@ extern "C" {
#endif
#ifdef __cplusplus
}
#endif

2
source/libs/scalar/inc/filterInt.h
View File

@ -36,8 +36,6 @@ extern "C" {
#define FILTER_DUMMY_EMPTY_OPTR 127
#define MAX_NUM_STR_SIZE 40
#define FILTER_RM_UNIT_MIN_ROWS 100
enum {

2
source/libs/scheduler/CMakeLists.txt
View File

@ -9,7 +9,7 @@ target_include_directories(
target_link_libraries(
scheduler
PUBLIC os util nodes planner qcom common catalog transport
PUBLIC os util nodes planner qcom common catalog transport command
)
if(${BUILD_TEST})

8
source/libs/scheduler/inc/schedulerInt.h
View File

@ -142,10 +142,10 @@ typedef struct SSchTask {
} SSchTask;
typedef struct SSchJobAttr {
bool needFetch;
bool syncSchedule;
bool queryJob;
bool needFlowCtrl;
EExplainMode explainMode;
bool syncSchedule;
bool queryJob;
bool needFlowCtrl;
} SSchJobAttr;
typedef struct SSchJob {

81
source/libs/scheduler/src/scheduler.c
View File

@ -19,6 +19,7 @@
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "command.h"
SSchedulerMgmt schMgmt = {0};
@ -2103,6 +2104,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
pJob->attr.explainMode = pDag->explainInfo.mode;
pJob->attr.syncSchedule = syncSchedule;
pJob->transport = transport;
pJob->sql = sql;
@ -2136,19 +2138,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
tsem_init(&pJob->rspSem, 0, 0);
pJob->refId = taosAddRef(schMgmt.jobRef, pJob);
if (pJob->refId < 0) {
SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno));
int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
if (refId < 0) {
SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
SCH_ERR_JRET(terrno);
}
if (NULL == schAcquireJob(refId)) {
SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
pJob->refId = refId;
SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
pJob->status = JOB_TASK_STATUS_NOT_START;
SCH_ERR_JRET(schLaunchJob(pJob));
schAcquireJob(pJob->refId);
*job = pJob->refId;
if (syncSchedule) {
@ -2168,6 +2175,54 @@ _return:
SCH_RET(code);
}
int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql,
bool syncSchedule) {
qDebug("QID:0x%" PRIx64 " job started", pDag->queryId);
int32_t code = 0;
SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
if (NULL == pJob) {
qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
pJob->sql = sql;
pJob->attr.queryJob = true;
pJob->attr.explainMode = pDag->explainInfo.mode;
pJob->queryId = pDag->queryId;
pJob->subPlans = pDag->pSubplans;
SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData));
int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
if (refId < 0) {
SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
SCH_ERR_JRET(terrno);
}
if (NULL == schAcquireJob(refId)) {
SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
pJob->refId = refId;
SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
*job = pJob->refId;
SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
schReleaseJob(pJob->refId);
return TSDB_CODE_SUCCESS;
_return:
schFreeJobImpl(pJob);
SCH_RET(code);
}
int32_t schedulerInit(SSchedulerCfg *cfg) {
if (schMgmt.jobRef) {
qError("scheduler already initialized");
@ -2216,7 +2271,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) {
SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true));
} else {
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
}
SSchJob *job = schAcquireJob(*pJob);
@ -2399,11 +2458,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) {
SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status));
goto _return;
} else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_ERR_JRET(schFetchFromRemote(pJob));
if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) {
SCH_ERR_JRET(schFetchFromRemote(pJob));
tsem_wait(&pJob->rspSem);
}
} else {
SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status));
SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
}
tsem_wait(&pJob->rspSem);
status = SCH_GET_JOB_STATUS(pJob);
if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {