xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28771 from taosdata/enh/addDiskUsage 

Enh/addDiskUsage
This commit is contained in:
Shengliang Guan 2024-12-03 13:42:02 +08:00 committed by GitHub
parent 68dca47759 8fafd0e0a5
commit ccf28466e0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
46 changed files with 2421 additions and 1143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/systable.h
View File

@ -61,6 +61,7 @@ extern "C" {
#define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_INS_DISK_USAGE "ins_disk_usage"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"

9
include/common/tcommon.h
View File

@ -302,6 +302,15 @@ typedef struct STableBlockDistInfo {
int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDistInfo* pInfo);
int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo* pInfo);
typedef struct SDBBlockUsageInfo {
uint64_t dataInDiskSize;
uint64_t walInDiskSize;
uint64_t rawDataSize;
} SDBBlockUsageInfo;
int32_t tSerializeBlockDbUsage(void* buf, int32_t bufLen, const SDBBlockUsageInfo* pInfo);
int32_t tDeserializeBlockDbUsage(void* buf, int32_t bufLen, SDBBlockUsageInfo* pInfo);
enum {
FUNC_PARAM_TYPE_VALUE = 0x1,
FUNC_PARAM_TYPE_COLUMN = 0x2,

17
include/common/tmsg.h
View File

@ -161,6 +161,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_USER_FULL,
TSDB_MGMT_TABLE_ANODE,
TSDB_MGMT_TABLE_ANODE_FULL,
TSDB_MGMT_TABLE_USAGE,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -397,6 +398,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_SHOW_ANODES_STMT,
QUERY_NODE_SHOW_ANODES_FULL_STMT,
QUERY_NODE_SHOW_USAGE_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
@ -1754,6 +1756,21 @@ typedef struct {
int32_t learnerProgress; // use one reservered
} SVnodeLoad;
typedef struct {
int32_t vgId;
int64_t numOfTables;
int64_t memSize;
int64_t l1Size;
int64_t l2Size;
int64_t l3Size;
int64_t cacheSize;
int64_t walSize;
int64_t metaSize;
int64_t rawDataSize;
int64_t s3Size;
const char* dbname;
} SDbSizeStatisInfo;
typedef struct {
int32_t vgId;
int64_t nTimeSeries;

4
include/libs/executor/storageapi.h
View File

@ -281,6 +281,7 @@ typedef struct SStoreMeta {
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int32_t (*getDBSize)(void* pVnode, SDbSizeStatisInfo* pInfo);
SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
@ -400,7 +401,8 @@ typedef struct SStateStore {
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType,
int32_t pkLen, SUpdateInfo** ppInfo);

7
include/libs/function/functionMgt.h
View File

@ -219,6 +219,9 @@ typedef enum EFunctionType {
FUNCTION_TYPE_CONTAINS,
FUNCTION_TYPE_CONTAINS_PROPERLY,
FUNCTION_TYPE_DB_USAGE = 4300,
FUNCTION_TYPE_DB_USAGE_INFO,
// user defined funcion
FUNCTION_TYPE_UDF = 10000
} EFunctionType;
@ -290,13 +293,15 @@ bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
bool fmIsElapsedFunc(int32_t funcId);
bool fmIsDBUsageFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);
int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFunc, SNodeList* pParameterList, SFunctionNode** pFunc);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc,
SFunctionNode** pMergeFunc);
typedef enum EFuncDataRequired {
FUNC_DATA_REQUIRED_DATA_LOAD = 1,

5
include/libs/nodes/cmdnodes.h
View File

@ -387,6 +387,11 @@ typedef struct SShowTableDistributedStmt {
char tableName[TSDB_TABLE_NAME_LEN];
} SShowTableDistributedStmt;
typedef struct SShowDBUsageStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SShowDBUsageStmt;
typedef struct SShowDnodeVariablesStmt {
ENodeType type;
SNode* pDnodeId;

2
include/libs/nodes/plannodes.h
View File

@ -77,7 +77,7 @@ typedef enum EScanType {
SCAN_TYPE_TABLE_MERGE,
SCAN_TYPE_BLOCK_INFO,
SCAN_TYPE_LAST_ROW,
SCAN_TYPE_TABLE_COUNT
SCAN_TYPE_TABLE_COUNT,
} EScanType;
typedef struct SScanLogicNode {

14
include/libs/nodes/querynodes.h
View File

@ -22,8 +22,8 @@ extern "C" {
#include "nodes.h"
#include "tmsg.h"
#include "tvariant.h"
#include "tsimplehash.h"
#include "tvariant.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \
@ -469,6 +469,7 @@ typedef struct SSelectStmt {
bool groupSort;
bool tagScan;
bool joinContains;
bool mixSysTableAndActualTable;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
@ -638,13 +639,15 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias,
ECollectColType type, SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier, SNodeList** pFuncs);
int32_t nodesCollectSelectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier, SNodeList* pFuncs);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier,
SNodeList** pFuncs);
int32_t nodesCollectSelectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAlias, FFuncClassifier classifier,
SNodeList* pFuncs);
int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes);
@ -683,7 +686,6 @@ char* getJoinSTypeString(EJoinSubType type);
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype);
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc);
#ifdef __cplusplus
}
#endif

1
include/os/osDir.h
View File

@ -100,6 +100,7 @@ bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
char *taosGetDirEntryName(TdDirEntryPtr pDirEntry);
int32_t taosCloseDir(TdDirPtr *ppDir);
int taosGetDirSize(const char *path, int64_t *size);
#ifdef __cplusplus
}
#endif

14
source/common/src/systable.c
View File

@ -453,6 +453,19 @@ static const SSysDbTableSchema encryptionsSchema[] = {
{.name = "key_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema diskUsageSchema[] = {
{.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "wal", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "data1", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "data2", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "data3", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "cache_rdb", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "table_meta", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "s3",.bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
{.name = "raw_data", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
};
static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -491,6 +504,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
{TSDB_INS_TABLE_ANODES, anodesSchema, tListLen(anodesSchema), true},
{TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true},
{TSDB_INS_DISK_USAGE, diskUsageSchema, tListLen(diskUsageSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

7
source/dnode/mnode/impl/src/mndShow.c
View File

@ -16,8 +16,8 @@
#define _DEFAULT_SOURCE
#include "mndShow.h"
#include "mndPrivilege.h"
#include "systable.h"
#include "mndUser.h"
#include "systable.h"
#define SHOW_STEP_SIZE 100
#define SHOW_COLS_STEP_SIZE 4096
@ -142,6 +142,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_ENCRYPTIONS;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS;
} else if (strncasecmp(name, TSDB_INS_DISK_USAGE, len) == 0) {
type = TSDB_MGMT_TABLE_USAGE;
} else {
mError("invalid show name:%s len:%d", name, len);
}
@ -334,7 +336,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
}
size_t dataEncodeBufSize = blockGetEncodeSize(pBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + dataEncodeBufSize;
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
dataEncodeBufSize;
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {

4
source/dnode/vnode/inc/vnode.h
View File

@ -80,6 +80,7 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeIsCatchUp(SVnode *pVnode);
ESyncRole vnodeGetRole(SVnode *pVnode);
int32_t vnodeGetArbToken(SVnode *pVnode, char *outToken);
int32_t vnodeGetDBSize(void *pVnode, SDbSizeStatisInfo *pInfo);
int32_t vnodeUpdateArbTerm(SVnode *pVnode, int64_t arbTerm);
@ -243,7 +244,8 @@ int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, con
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, int64_t *createTime);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet,
int64_t *createTime);
int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma

2
source/dnode/vnode/src/inc/tsdb.h
View File

@ -959,6 +959,8 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbGetS3Size(STsdb *tsdb, int64_t *size);
// ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
TSDBKEY *pKey1 = (TSDBKEY *)p1;

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -94,6 +94,8 @@ typedef struct SQueryNode SQueryNode;
#define VNODE_RSMA1_DIR "rsma1"
#define VNODE_RSMA2_DIR "rsma2"
#define VNODE_TQ_STREAM "stream"
#define VNODE_CACHE_DIR "cache.rdb"
#define VNODE_TSDB_CACHE_DIR VNODE_TSDB_DIR TD_DIRSEP VNODE_CACHE_DIR
#if SUSPEND_RESUME_TEST // only for test purpose
#define VNODE_BUFPOOL_SEGMENTS 1

27
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -733,3 +733,30 @@ int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
}
return code;
}
static int32_t tsdbGetS3SizeImpl(STsdb *tsdb, int64_t *size) {
int32_t code = 0;
SVnodeCfg *pCfg = &tsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
STFileSet *fset;
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (fobj) {
int32_t lcn = fobj->f->lcn;
if (lcn > 1) {
*size += ((lcn - 1) * chunksize);
}
}
}
return code;
}
int32_t tsdbGetS3Size(STsdb *tsdb, int64_t *size) {
int32_t code = 0;
(void)taosThreadMutexLock(&tsdb->mutex);
code = tsdbGetS3SizeImpl(tsdb, size);
(void)taosThreadMutexUnlock(&tsdb->mutex);
return code;
}

1
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -105,6 +105,7 @@ void initMetadataAPI(SStoreMeta* pMeta) {
pMeta->pauseCtbCursor = metaPauseCtbCursor;
pMeta->closeCtbCursor = metaCloseCtbCursor;
pMeta->ctbCursorNext = metaCtbCursorNext;
pMeta->getDBSize = vnodeGetDBSize;
}
void initTqAPI(SStoreTqReader* pTq) {

35
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -870,6 +870,41 @@ int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
int32_t vnodeGetDBSize(void *pVnode, SDbSizeStatisInfo *pInfo) {
SVnode *pVnodeObj = pVnode;
if (pVnodeObj == NULL) {
return TSDB_CODE_VND_NOT_EXIST;
}
int32_t code = 0;
char path[TSDB_FILENAME_LEN] = {0};
char *dirName[] = {VNODE_TSDB_DIR, VNODE_WAL_DIR, VNODE_META_DIR, VNODE_TSDB_CACHE_DIR};
int64_t dirSize[4];
vnodeGetPrimaryDir(pVnodeObj->path, pVnodeObj->diskPrimary, pVnodeObj->pTfs, path, TSDB_FILENAME_LEN);
int32_t offset = strlen(path);
for (int i = 0; i < sizeof(dirName) / sizeof(dirName[0]); i++) {
int64_t size = {0};
(void)snprintf(path + offset, TSDB_FILENAME_LEN, "%s%s", TD_DIRSEP, dirName[i]);
code = taosGetDirSize(path, &size);
if (code != 0) {
return code;
}
path[offset] = 0;
dirSize[i] = size;
}
pInfo->l1Size = dirSize[0] - dirSize[3];
pInfo->walSize = dirSize[1];
pInfo->metaSize = dirSize[2];
pInfo->cacheSize = dirSize[3];
code = tsdbGetS3Size(pVnodeObj->pTsdb, &pInfo->s3Size);
return code;
}
int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
int32_t code = 0;
SStreamProgressReq req;

31
source/libs/executor/src/operator.c
View File

@ -29,10 +29,9 @@
#include "storageapi.h"
#include "tdatablock.h"
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
__optr_explain_fn_t explain, __optr_get_ext_fn_t nextExtFn, __optr_notify_fn_t notifyFn) {
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain,
__optr_get_ext_fn_t nextExtFn, __optr_notify_fn_t notifyFn) {
SOperatorFpSet fpSet = {
._openFn = openFn,
.getNextFn = nextFn,
@ -288,7 +287,6 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
int32_t type = nodeType(pPhyNode);
const char* idstr = GET_TASKID(pTaskInfo);
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
SOperatorInfo* pOperator = NULL;
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
@ -318,8 +316,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
pTableListInfo->idInfo.suid = pTableScanNode->scan.suid;
pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType;
} else {
code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort,
pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
tableListDestroy(pTableListInfo);
@ -346,8 +344,8 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
return terrno;
}
code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo,
pTagCond, pTagIndexCond, pTaskInfo);
if (code) {
pTaskInfo->code = code;
tableListDestroy(pTableListInfo);
@ -655,7 +653,6 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
return code;
}
void destroyOperator(SOperatorInfo* pOperator) {
if (pOperator == NULL) {
return;
@ -758,12 +755,14 @@ int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc) {
tSimpleHashSetFreeFp(pBatch->pBatchs, freeExchangeGetBasicOperatorParam);
int32_t code = tSimpleHashPut(pBatch->pBatchs, &pDExc->basic.vgId, sizeof(pDExc->basic.vgId), &pDExc->basic, sizeof(pDExc->basic));
int32_t code = tSimpleHashPut(pBatch->pBatchs, &pDExc->basic.vgId, sizeof(pDExc->basic.vgId), &pDExc->basic,
sizeof(pDExc->basic));
if (code) {
return code;
}
code = tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic, sizeof(pSExc->basic));
code = tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic,
sizeof(pSExc->basic));
if (code) {
return code;
}
@ -778,14 +777,16 @@ int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc) {
}
} else {
SExchangeOperatorBatchParam* pBatch = pDst->value;
SExchangeOperatorBasicParam* pBasic = tSimpleHashGet(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId));
SExchangeOperatorBasicParam* pBasic =
tSimpleHashGet(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId));
if (pBasic) {
void* p = taosArrayAddAll(pBasic->uidList, pSExc->basic.uidList);
if (p == NULL) {
return terrno;
}
} else {
int32_t code = tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic, sizeof(pSExc->basic));
int32_t code = tSimpleHashPut(pBatch->pBatchs, &pSExc->basic.vgId, sizeof(pSExc->basic.vgId), &pSExc->basic,
sizeof(pSExc->basic));
if (code) {
return code;
}
@ -801,7 +802,6 @@ int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc) {
return TSDB_CODE_SUCCESS;
}
int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInput, SOperatorParamType type) {
SOperatorParam** ppParam = NULL;
SOperatorParam*** pppDownstramParam = NULL;
@ -869,7 +869,6 @@ int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInpu
return TSDB_CODE_SUCCESS;
}
SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx) {
SSDataBlock* p = NULL;
int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, true, &p);
@ -943,5 +942,3 @@ int16_t getOperatorResultBlockId(struct SOperatorInfo* pOperator, int32_t idx) {
}
return pOperator->resultDataBlockId;
}

440
source/libs/executor/src/sysscanoperator.c
View File

@ -61,6 +61,7 @@ typedef struct SSysTableScanInfo {
bool sysInfo;
bool showRewrite;
bool restore;
bool skipFilterTable;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
SMTbCursor* pCur; // cursor for iterate the local table meta store.
SSysTableIndex* pIdx; // idx for local table meta
@ -72,6 +73,8 @@ typedef struct SSysTableScanInfo {
SLoadRemoteDataInfo loadInfo;
SLimitInfo limitInfo;
int32_t tbnameSlotId;
STableListInfo* pTableListInfo;
SReadHandle* pHandle;
SStorageAPI* pAPI;
} SSysTableScanInfo;
@ -94,6 +97,11 @@ typedef struct SBlockDistInfo {
uint64_t uid; // table uid
} SBlockDistInfo;
typedef struct {
int8_t type;
tb_uid_t uid;
} STableId;
static int32_t sysChkFilter__Comm(SNode* pNode);
static int32_t sysChkFilter__DBName(SNode* pNode);
static int32_t sysChkFilter__VgroupId(SNode* pNode);
@ -150,6 +158,8 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo,
static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock,
SFilterInfo* pFilterInfo, SExecTaskInfo* pTaskInfo);
static int32_t vnodeEstimateRawDataSize(SOperatorInfo* pOperator, SDbSizeStatisInfo* pStatisInfo);
int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
SSTabFltArg* pArg = arg;
void* pVnode = pArg->pVnode;
@ -848,9 +858,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
break;
}
}
// if pInfo->pRes->info.rows == 0, also need to add this meta into datablock.
// if pInfo->pRes->info.rows == 0, also need to add the meta to pDataBlock
code = sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
dataBlock);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
pAPI->metaReaderFn.clearReader(&smrSuperTable);
@ -1912,6 +1924,194 @@ _end:
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
static int32_t buildVgDiskUsage(SOperatorInfo* pOperator, SDbSizeStatisInfo* pStaticsInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
int32_t vgId = 0;
const char* db = NULL;
pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &pStaticsInfo->dbname, &vgId, NULL, NULL);
pStaticsInfo->vgId = vgId;
code = pAPI->metaFn.getDBSize(pInfo->readHandle.vnode, pStaticsInfo);
QUERY_CHECK_CODE(code, lino, _end);
code = vnodeEstimateRawDataSize(pOperator, pStaticsInfo);
QUERY_CHECK_CODE(code, lino, _end);
pStaticsInfo->memSize = pStaticsInfo->memSize >> 10;
pStaticsInfo->l1Size = pStaticsInfo->l1Size >> 10;
pStaticsInfo->l2Size = pStaticsInfo->l2Size >> 10;
pStaticsInfo->l3Size = pStaticsInfo->l3Size >> 10;
pStaticsInfo->cacheSize = pStaticsInfo->cacheSize >> 10;
pStaticsInfo->walSize = pStaticsInfo->walSize >> 10;
pStaticsInfo->metaSize = pStaticsInfo->metaSize >> 10;
pStaticsInfo->rawDataSize = pStaticsInfo->rawDataSize >> 10;
pStaticsInfo->s3Size = pStaticsInfo->s3Size >> 10;
_end:
return code;
}
static SSDataBlock* sysTableBuildVgUsage(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
SDbSizeStatisInfo staticsInfo = {0};
char* buf = NULL;
SSDataBlock* p = NULL;
const char* db = NULL;
int32_t numOfCols = 0;
int32_t numOfRows = 0;
// the retrieve is executed on the mnode, so return tables that belongs to the information schema database.
if (pInfo->readHandle.mnd != NULL) {
setOperatorCompleted(pOperator);
return NULL;
}
if (pInfo->pCur == NULL) {
pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
if (pInfo->pCur == NULL) {
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
}
}
SSDataBlock* pBlock = pInfo->pRes;
code = buildVgDiskUsage(pOperator, &staticsInfo);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->showRewrite) {
SSDataBlock* pBlock = pInfo->pRes;
SDBBlockUsageInfo usageInfo = {0};
int32_t len = tSerializeBlockDbUsage(NULL, 0, &usageInfo);
usageInfo.dataInDiskSize = staticsInfo.l1Size + staticsInfo.l2Size + staticsInfo.l3Size;
usageInfo.walInDiskSize = staticsInfo.walSize;
usageInfo.rawDataSize = staticsInfo.rawDataSize;
buf = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE);
QUERY_CHECK_NULL(buf, code, lino, _end, terrno);
int32_t tempRes = tSerializeBlockDbUsage(varDataVal(buf), len, &usageInfo);
if (tempRes != len) {
QUERY_CHECK_CODE(TSDB_CODE_INVALID_MSG, lino, _end);
}
varDataSetLen(buf, len);
int32_t slotId = 1;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 1);
QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno);
code = colDataSetVal(pColInfo, 0, buf, false);
QUERY_CHECK_CODE(code, lino, _end);
taosMemoryFreeClear(buf);
if (slotId != 0) {
SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0);
QUERY_CHECK_NULL(p1, code, lino, _end, terrno);
}
pBlock->info.rows = 1;
pOperator->status = OP_EXEC_DONE;
pInfo->pRes->info.rows = pBlock->info.rows;
QUERY_CHECK_CODE(code, lino, _end);
} else {
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
code = tNameFromString(&sn, staticsInfo.dbname, T_NAME_ACCT | T_NAME_DB);
QUERY_CHECK_CODE(code, lino, _end);
code = tNameGetDbName(&sn, varDataVal(dbname));
QUERY_CHECK_CODE(code, lino, _end);
varDataSetLen(dbname, strlen(varDataVal(dbname)));
p = buildInfoSchemaTableMetaBlock(TSDB_INS_DISK_USAGE);
QUERY_CHECK_NULL(p, code, lino, _end, terrno);
code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, dbname, false);
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.vgId, false);
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.walSize, false); // wal
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.l1Size, false); // l1_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.l2Size, false); // l2_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.l3Size, false); // l3_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.cacheSize, false); // cache_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.metaSize, false); // meta_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.s3Size, false); // s3_size
QUERY_CHECK_CODE(code, lino, _end);
pColInfoData = taosArrayGet(p->pDataBlock, numOfCols++);
code = colDataSetVal(pColInfoData, numOfRows, (char*)&staticsInfo.rawDataSize, false); // estimate_size
QUERY_CHECK_CODE(code, lino, _end);
numOfRows += 1;
if (numOfRows > 0) {
p->info.rows = numOfRows;
pInfo->pRes->info.rows = numOfRows;
code = relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
QUERY_CHECK_CODE(code, lino, _end);
code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
blockDataDestroy(p);
p = NULL;
pInfo->loadInfo.totalRows += pInfo->pRes->info.rows;
setOperatorCompleted(pOperator);
}
_end:
taosMemoryFree(buf);
if (pInfo->pCur) {
pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
}
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
blockDataDestroy(p);
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -1966,6 +2166,26 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
return sysTableBuildUserTables(pOperator);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
return NULL;
}
static SSDataBlock* sysTableScanUsage(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSysTableScanInfo* pInfo = pOperator->info;
SNode* pCondition = pInfo->pCondition;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
return sysTableBuildVgUsage(pOperator);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -2075,7 +2295,7 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes)
if (dbName[0]) TAOS_UNUSED(tsnprintf(pInfo->req.db, sizeof(pInfo->req.db), "%d.%s", pInfo->accountId, dbName));
(void)sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
}
bool filter = true;
SSDataBlock* pBlock = NULL;
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUserTables(pOperator);
@ -2086,11 +2306,13 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes)
} else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
IS_SYS_DBNAME(dbName)) {
pBlock = sysTableScanUserSTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_DISK_USAGE, TSDB_TABLE_FNAME_LEN) == 0) {
pBlock = sysTableScanUsage(pOperator);
} else { // load the meta from mnode of the given epset
pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
}
sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
if (!pInfo->skipFilterTable) sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
if (pBlock != NULL) {
bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
if (limitReached) {
@ -2266,6 +2488,7 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo
SScanPhysiNode* pScanNode = &pScanPhyNode->scan;
SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
QUERY_CHECK_CODE(code, lino, _error);
int32_t num = 0;
code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
@ -2282,8 +2505,21 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo
pInfo->showRewrite = pScanPhyNode->showRewrite;
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
pInfo->pCondition = pScanNode->node.pConditions;
tNameAssign(&pInfo->name, &pScanNode->tableName);
const char* name = tNameGetTableName(&pInfo->name);
if (pInfo->showRewrite == false) {
code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
} else {
if (strncasecmp(name, TSDB_INS_DISK_USAGE, TSDB_TABLE_FNAME_LEN) == 0) {
pInfo->skipFilterTable = true;
code = filterInitFromNode(NULL, &pOperator->exprSupp.pFilterInfo, 0);
} else {
code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
}
}
QUERY_CHECK_CODE(code, lino, _error);
initLimitInfo(pScanPhyNode->scan.node.pLimit, pScanPhyNode->scan.node.pSlimit, &pInfo->limitInfo);
@ -2291,9 +2527,6 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);
tNameAssign(&pInfo->name, &pScanNode->tableName);
const char* name = tNameGetTableName(&pInfo->name);
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 ||
strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
pInfo->readHandle = *(SReadHandle*)readHandle;
@ -2792,8 +3025,7 @@ static int32_t doBlockInfoScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes
code = pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
QUERY_CHECK_CODE(code, lino, _end);
blockDistInfo.numOfInmemRows = 0;
code = pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle, &blockDistInfo.numOfInmemRows);
code = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle, &blockDistInfo.numOfInmemRows);
QUERY_CHECK_CODE(code, lino, _end);
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
@ -2944,3 +3176,195 @@ _error:
}
return code;
}
static int32_t buildTableListInfo(SOperatorInfo* pOperator, STableId* id, STableListInfo** ppTableListInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t line = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
SReadHandle* pReadHandle = &pInfo->readHandle;
SArray* pList = NULL;
STableListInfo* pTableListInfo = tableListCreate();
QUERY_CHECK_NULL(ppTableListInfo, code, line, _end, terrno);
if (id->type == TSDB_SUPER_TABLE) {
pList = taosArrayInit(4, sizeof(uint64_t));
QUERY_CHECK_NULL(pList, code, line, _end, terrno);
code = pReadHandle->api.metaFn.getChildTableList(pReadHandle->vnode, id->uid, pList);
QUERY_CHECK_CODE(code, line, _end);
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
uint64_t* id = taosArrayGet(pList, i);
if (id == NULL) {
continue;
}
code = tableListAddTableInfo(pTableListInfo, *id, 0);
QUERY_CHECK_CODE(code, line, _end);
}
taosArrayDestroy(pList);
pList = NULL;
} else if (id->type == TSDB_NORMAL_TABLE) {
code = tableListAddTableInfo(pTableListInfo, id->uid, 0);
QUERY_CHECK_CODE(code, line, _end);
}
*ppTableListInfo = pTableListInfo;
return code;
_end:
taosArrayDestroy(pList);
tableListDestroy(pTableListInfo);
return code;
}
static int32_t vnodeEstimateDataSizeByUid(SOperatorInfo* pOperator, STableId* id, SDbSizeStatisInfo* pStaticInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t line = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
SQueryTableDataCond cond = {0};
SReadHandle* pReadHandle = &pInfo->readHandle;
STableListInfo* pTableListInfo = NULL;
code = buildTableListInfo(pOperator, id, &pTableListInfo);
QUERY_CHECK_CODE(code, line, _end);
tb_uid_t tbId = id->type == TSDB_SUPER_TABLE ? id->uid : 0;
code = initTableblockDistQueryCond(tbId, &cond);
QUERY_CHECK_CODE(code, line, _end);
pInfo->pTableListInfo = pTableListInfo;
int32_t num = 0;
code = tableListGetSize(pTableListInfo, &num);
QUERY_CHECK_CODE(code, line, _end);
void* pList = tableListGetInfo(pTableListInfo, 0);
code = pReadHandle->api.tsdReader.tsdReaderOpen(pReadHandle->vnode, &cond, pList, num, NULL, (void**)&pInfo->pHandle,
pTaskInfo->id.str, NULL);
cleanupQueryTableDataCond(&cond);
QUERY_CHECK_CODE(code, line, _end);
STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
code = doGetTableRowSize(pReadHandle, id->uid, (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
QUERY_CHECK_CODE(code, line, _end);
code = pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pInfo->pHandle, &blockDistInfo);
QUERY_CHECK_CODE(code, line, _end);
code = pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pInfo->pHandle, &blockDistInfo.numOfInmemRows);
QUERY_CHECK_CODE(code, line, _end);
int64_t rawDiskSize = 0, rawCacheSize = 0;
rawDiskSize = (blockDistInfo.totalRows + blockDistInfo.numOfSttRows) * blockDistInfo.rowSize;
rawCacheSize = blockDistInfo.numOfInmemRows * blockDistInfo.rowSize;
pStaticInfo->rawDataSize += rawDiskSize;
pStaticInfo->cacheSize += rawCacheSize;
if (pInfo->pHandle != NULL) {
pReadHandle->api.tsdReader.tsdReaderClose(pInfo->pHandle);
pInfo->pHandle = NULL;
}
tableListDestroy(pInfo->pTableListInfo);
pInfo->pTableListInfo = NULL;
return code;
_end:
if (pInfo->pHandle != NULL) {
pReadHandle->api.tsdReader.tsdReaderClose(pInfo->pHandle);
pInfo->pHandle = NULL;
}
tableListDestroy(pInfo->pTableListInfo);
pInfo->pTableListInfo = NULL;
if (code) {
pTaskInfo->code = code;
return code;
}
cleanupQueryTableDataCond(&cond);
return code;
}
static int32_t vnodeEstimateRawDataSizeImpl(SOperatorInfo* pOperator, SArray* pTableList,
SDbSizeStatisInfo* pStaticInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
int32_t rowLen = 0;
int32_t code = TSDB_CODE_SUCCESS;
for (int i = 0; i < taosArrayGetSize(pTableList); i++) {
STableId* id = (STableId*)taosArrayGet(pTableList, i);
code = vnodeEstimateDataSizeByUid(pOperator, id, pStaticInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
return code;
}
static int32_t vnodeEstimateRawDataSize(SOperatorInfo* pOperator, SDbSizeStatisInfo* pStaticInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t line = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
int32_t numOfRows = 0;
int32_t ret = 0;
if (pInfo->pCur == NULL) {
pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
if (pInfo->pCur == NULL) {
TAOS_CHECK_GOTO(terrno, &line, _exit);
}
}
SArray* pIdList = taosArrayInit(16, sizeof(STableId));
if (pIdList == NULL) {
TAOS_CHECK_GOTO(terrno, &line, _exit);
}
while (((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_CHILD_TABLE)) == 0)) {
if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) {
STableId id = {.type = TSDB_SUPER_TABLE, .uid = pInfo->pCur->mr.me.uid};
if (taosArrayPush(pIdList, &id) == NULL) {
TAOS_CHECK_GOTO(terrno, &line, _exit);
}
} else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) {
continue;
} else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) {
STableId id = {.type = TSDB_NORMAL_TABLE, .uid = pInfo->pCur->mr.me.uid};
if (taosArrayPush(pIdList, &id) == NULL) {
TAOS_CHECK_GOTO(terrno, &line, _exit);
}
}
}
if (pInfo->pCur) {
pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
}
code = vnodeEstimateRawDataSizeImpl(pOperator, pIdList, pStaticInfo);
_exit:
if (pInfo->pCur) {
pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
}
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, line, tstrerror(code));
pTaskInfo->code = code;
}
taosArrayDestroy(pIdList);
return code;
}

4
source/libs/function/inc/builtinsimpl.h
View File

@ -247,6 +247,10 @@ int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t groupConstValueFunction(SqlFunctionCtx* pCtx);
int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t blockDBUsageSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t blockDBUsageFunction(SqlFunctionCtx* pCtx);
int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
#ifdef __cplusplus
}
#endif

83
source/libs/function/src/builtins.c
View File

@ -698,14 +698,12 @@ static int32_t checkRangeValue(SNode *pNode, SParamRange range, bool *isMatch) {
if (pNode->type == QUERY_NODE_VALUE) {
SValueNode* pVal = (SValueNode*)pNode;
if (IS_INTEGER_TYPE(getSDataTypeFromNode(pNode)->type)) {
if (pVal->datum.i < range.iMinVal ||
pVal->datum.i > range.iMaxVal) {
if (pVal->datum.i < range.iMinVal || pVal->datum.i > range.iMaxVal) {
code = TSDB_CODE_FUNC_FUNTION_PARA_RANGE;
*isMatch = false;
}
} else {
if ((int64_t)pVal->datum.d < range.iMinVal ||
(int64_t)pVal->datum.d > range.iMaxVal) {
if ((int64_t)pVal->datum.d < range.iMinVal || (int64_t)pVal->datum.d > range.iMaxVal) {
code = TSDB_CODE_FUNC_FUNTION_PARA_RANGE;
*isMatch = false;
}
@ -719,8 +717,7 @@ static int32_t checkRangeValue(SNode *pNode, SParamRange range, bool *isMatch) {
static int32_t checkFixedValue(SNode* pNode, const SParamInfo* paramPattern, int32_t paramIdx, bool* isMatch) {
int32_t code = TSDB_CODE_SUCCESS;
bool checkStr = paramSupportVarBinary(paramPattern->validDataType) ||
paramSupportVarchar(paramPattern->validDataType) ||
paramSupportNchar(paramPattern->validDataType);
paramSupportVarchar(paramPattern->validDataType) || paramSupportNchar(paramPattern->validDataType);
if (pNode->type == QUERY_NODE_VALUE) {
SValueNode* pVal = (SValueNode*)pNode;
if (!checkStr) {
@ -831,7 +828,8 @@ static int32_t validateParam(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
const SParamInfo* paramPattern = funcMgtBuiltins[pFunc->funcId].parameters.inputParaInfo[i];
while (1) {
for (int8_t j = paramPattern[paramIdx].startParam; j <= (paramPattern[paramIdx].endParam == -1 ? INT8_MAX : paramPattern[paramIdx].endParam); j++) {
for (int8_t j = paramPattern[paramIdx].startParam;
j <= (paramPattern[paramIdx].endParam == -1 ? INT8_MAX : paramPattern[paramIdx].endParam); j++) {
if (j > LIST_LENGTH(paramList)) {
code = TSDB_CODE_SUCCESS;
isMatch = true;
@ -915,8 +913,8 @@ static int32_t validateParam(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_RANGE, "Invalid parameter range : %s",
pFunc->functionName);
case TSDB_CODE_FUNC_FUNTION_PARA_PRIMTS:
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_PRIMTS, "Parameter should be primary timestamp : %s",
pFunc->functionName);
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_PRIMTS,
"Parameter should be primary timestamp : %s", pFunc->functionName);
case TSDB_CODE_FUNC_FUNTION_PARA_PK:
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_PK, "Parameter should be primary key : %s",
pFunc->functionName);
@ -969,7 +967,6 @@ static int32_t translateOutDouble(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return TSDB_CODE_SUCCESS;
}
static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isLtrim) {
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
@ -1024,7 +1021,8 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT,
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes,
.type = TSDB_DATA_TYPE_BIGINT,
.precision = pFunc->node.resType.precision};
return TSDB_CODE_SUCCESS;
}
@ -1054,8 +1052,7 @@ static int32_t translateRand(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
}
}
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
return TSDB_CODE_SUCCESS;
}
@ -1069,8 +1066,8 @@ static int32_t translateOutFirstIn(SFunctionNode* pFunc, char* pErrBuf, int32_t
static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP,
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
.type = TSDB_DATA_TYPE_TIMESTAMP,
.precision = pFunc->node.resType.precision};
return TSDB_CODE_SUCCESS;
}
@ -1101,7 +1098,6 @@ static int32_t translateVgIdColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS;
}
static int32_t reserveFirstMergeParam(SNodeList* pRawParameters, SNode* pPartialRes, SNodeList** pParameters) {
int32_t code = nodesListMakeAppend(pParameters, pPartialRes);
if (TSDB_CODE_SUCCESS == code) {
@ -1130,13 +1126,9 @@ int32_t apercentileCreateMergeParam(SNodeList* pRawParameters, SNode* pPartialRe
return code;
}
static int32_t translateElapsedPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return 0;
}
static int32_t translateElapsedPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return 0; }
static int32_t translateElapsedMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return 0;
}
static int32_t translateElapsedMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return 0; }
static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
@ -1164,8 +1156,8 @@ static int32_t translateSampleTail(SFunctionNode* pFunc, char* pErrBuf, int32_t
uint8_t colType = pSDataType->type;
// set result type
pFunc->node.resType = (SDataType){.bytes = IS_STR_DATA_TYPE(colType) ? pSDataType->bytes : tDataTypes[colType].bytes,
.type = colType};
pFunc->node.resType =
(SDataType){.bytes = IS_STR_DATA_TYPE(colType) ? pSDataType->bytes : tDataTypes[colType].bytes, .type = colType};
return TSDB_CODE_SUCCESS;
}
@ -1331,14 +1323,14 @@ static int32_t translateChar(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
static int32_t translateAscii(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UTINYINT].bytes, .type = TSDB_DATA_TYPE_UTINYINT};
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UTINYINT].bytes, .type = TSDB_DATA_TYPE_UTINYINT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateTrim(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
uint8_t para0Type = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
int32_t resLen = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->bytes;
uint8_t type = para0Type;
@ -1529,8 +1521,7 @@ static int32_t translateAddPrecOutBigint(SFunctionNode* pFunc, char* pErrBuf, in
return code;
}
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
@ -1582,6 +1573,10 @@ static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv
pEnv->calcMemSize = sizeof(STableBlockDistInfo);
return true;
}
static bool getBlockDBUsageFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SDBBlockUsageInfo);
return true;
}
static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
@ -1591,7 +1586,6 @@ static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return TSDB_CODE_SUCCESS;
}
static int32_t translateServerStatusFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
return TSDB_CODE_SUCCESS;
@ -1674,12 +1668,17 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t
case FUNCTION_TYPE_FIRST_STATE:
case FUNCTION_TYPE_LAST_STATE:
bytes = getFirstLastInfoSize(getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->bytes,
(pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE;
(pFunc->hasPk) ? pFunc->pkBytes : 0) +
VARSTR_HEADER_SIZE;
break;
case FUNCTION_TYPE_FIRST_STATE_MERGE:
case FUNCTION_TYPE_LAST_STATE_MERGE:
bytes = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->bytes;
break;
case FUNCTION_TYPE_DB_USAGE:
case FUNCTION_TYPE_DB_USAGE_INFO:
bytes = 128;
break;
default:
bytes = 0;
break;
@ -5610,6 +5609,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_db_usage",
.type = FUNCTION_TYPE_DB_USAGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
.translateFunc = translateOutVarchar,
.getEnvFunc = getBlockDBUsageFuncEnv,
.initFunc = blockDBUsageSetup,
.processFunc = blockDBUsageFunction,
.finalizeFunc = blockDBUsageFinalize
},
{
.name = "_db_usage_info",
.type = FUNCTION_TYPE_DB_USAGE_INFO,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
.translateFunc = translateOutVarchar,
}
};
// clang-format on

190
source/libs/function/src/builtinsimpl.c
View File

@ -28,12 +28,8 @@
#include "thistogram.h"
#include "tpercentile.h"
bool ignoreNegative(int8_t ignoreOption){
return (ignoreOption & 0x1) == 0x1;
}
bool ignoreNull(int8_t ignoreOption){
return (ignoreOption & 0x2) == 0x2;
}
bool ignoreNegative(int8_t ignoreOption) { return (ignoreOption & 0x1) == 0x1; }
bool ignoreNull(int8_t ignoreOption) { return (ignoreOption & 0x2) == 0x2; }
typedef enum {
APERCT_ALGO_UNKNOWN = 0,
@ -312,7 +308,8 @@ int32_t funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow,
qError("out of memory when function get input row.");
return terrno;
}
(void)memcpy(pIter->pPrevData, colDataGetData(pIter->pDataCol, pIter->inputEndIndex), pIter->pDataCol->info.bytes);
(void)memcpy(pIter->pPrevData, colDataGetData(pIter->pDataCol, pIter->inputEndIndex),
pIter->pDataCol->info.bytes);
pIter->pPrevPk = taosMemoryMalloc(pIter->pPkCol->info.bytes);
if (NULL == pIter->pPrevPk) {
qError("out of memory when function get input row.");
@ -448,7 +445,8 @@ int32_t appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int3
return TSDB_CODE_SUCCESS;
}
bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool firstOccur, int32_t* pRowIndex, int32_t* nextFrom);
bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool firstOccur, int32_t* pRowIndex,
int32_t* nextFrom);
static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst);
@ -1805,7 +1803,8 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
pResInfo->complete = true;
return TSDB_CODE_SUCCESS;
} else {
code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, &pInfo->pMemBucket, pInfo->numOfElems);
code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup,
&pInfo->pMemBucket, pInfo->numOfElems);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
@ -1894,7 +1893,8 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if ((*pMemBucket) != NULL && (*pMemBucket)->total > 0) { // check for null
if (pCtx->numOfParams > 2) {
char buf[3200] = {0};
// max length of double num is 317, e.g. use %.6lf to print -1.0e+308, consider the comma and bracket, 3200 is enough.
// max length of double num is 317, e.g. use %.6lf to print -1.0e+308, consider the comma and bracket, 3200 is
// enough.
size_t len = 1;
varDataVal(buf)[0] = '[';
@ -2444,7 +2444,8 @@ static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowI
return code;
}
static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, char* pkData, int32_t type, char* pData) {
static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, char* pkData, int32_t type,
char* pData) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
@ -2978,7 +2979,6 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex
SColumnInfoData* pInputCol = pInput->pData[0];
SColumnInfoData* pkCol = pInput->pPrimaryKey;
if (colDataIsNull_s(pInputCol, rowIndex)) {
pInfo->isNull = true;
} else {
@ -3089,7 +3089,6 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
pResInfo->numOfRes = 1;
}
}
}
SET_VAL(pResInfo, numOfElems, 1);
@ -3305,7 +3304,8 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv,
// TODO: the primary key compare can be skipped for ordered pk if knonwn before
// TODO: for desc ordered, pk shall select the smallest one for one ts. if across block boundaries.
bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool firstOccur, int32_t* pRowIndex, int32_t* nextFrom) {
bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool firstOccur, int32_t* pRowIndex,
int32_t* nextFrom) {
if (pInput->pPrimaryKey == NULL) {
if (from == -1) {
from = pInput->startRowIndex;
@ -3424,9 +3424,7 @@ int32_t setDoDiffResult(SqlFunctionCtx* pCtx, SFuncInputRow* pRow, int32_t pos)
return TSDB_CODE_SUCCESS;
}
int32_t diffFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t diffFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; }
int32_t diffFunctionByRow(SArray* pCtxArray) {
int32_t code = TSDB_CODE_SUCCESS;
@ -3865,7 +3863,8 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock*
return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore);
}
static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) {
static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos,
SFunctionStateStore* pStore) {
if (pHandle->pBuf != NULL) {
SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
if (pPage == NULL) {
@ -3898,7 +3897,8 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc
return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore);
}
static int32_t doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore, char** value) {
static int32_t doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore,
char** value) {
if (pHandle->pBuf != NULL) {
SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
if (pPage == NULL) {
@ -4743,11 +4743,13 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t len;
char buf[512] = {0};
if (!pInfo->normalized) {
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}",
pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count);
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE,
"{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", pInfo->bins[i].lower,
pInfo->bins[i].upper, pInfo->bins[i].count);
} else {
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower,
pInfo->bins[i].upper, pInfo->bins[i].percentage);
len = tsnprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE,
"{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower, pInfo->bins[i].upper,
pInfo->bins[i].percentage);
}
varDataSetLen(buf, len);
code = colDataSetVal(pCol, currentRow, buf, false);
@ -6389,23 +6391,26 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
avgRows = pData->totalRows / pData->numOfBlocks;
}
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]",
pData->totalRows, pData->minRows, pData->maxRows, avgRows);
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]", pData->totalRows,
pData->minRows, pData->maxRows, avgRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows);
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ",
pData->numOfInmemRows, pData->numOfSttRows);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables,
pData->numOfFiles, pData->numOfVgroups);
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE,
"Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables, pData->numOfFiles,
pData->numOfVgroups);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
@ -6440,7 +6445,8 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t bucketRange = ceil(((double)(pData->defMaxRows - pData->defMinRows)) / numOfBuckets);
for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) {
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
len =
tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1));
int32_t num = 0;
if (pData->blockRowsHisto[i] > 0) {
@ -6454,7 +6460,8 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if (pData->blockRowsHisto[i] > 0) {
double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks;
len += tsnprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%');
len += tsnprintf(varDataVal(st) + len, sizeof(st) - VARSTR_HEADER_SIZE - len, " %d (%.2f%c)",
pData->blockRowsHisto[i], v, '%');
}
varDataSetLen(st, len);
@ -6466,6 +6473,114 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return TSDB_CODE_SUCCESS;
}
int32_t blockDBUsageSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (pResultInfo->initialized) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) {
return TSDB_CODE_FUNC_SETUP_ERROR;
}
SDBBlockUsageInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
return TSDB_CODE_SUCCESS;
}
int32_t blockDBUsageFunction(SqlFunctionCtx* pCtx) {
const int32_t BLOCK_DISK_USAGE_RESULT_ROWS = 2;
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDBBlockUsageInfo* pDistInfo = GET_ROWCELL_INTERBUF(pResInfo);
SDBBlockUsageInfo p1 = {0};
if (tDeserializeBlockDbUsage(varDataVal(pInputCol->pData), varDataLen(pInputCol->pData), &p1) < 0) {
qError("failed to deserialize block dist info");
return TSDB_CODE_FAILED;
}
pDistInfo->dataInDiskSize += p1.dataInDiskSize;
pDistInfo->walInDiskSize += p1.walInDiskSize;
pDistInfo->rawDataSize += p1.rawDataSize;
pResInfo->numOfRes = BLOCK_DISK_USAGE_RESULT_ROWS; // default output rows
return TSDB_CODE_SUCCESS;
}
int32_t tSerializeBlockDbUsage(void* buf, int32_t bufLen, const SDBBlockUsageInfo* pInfo) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pInfo->dataInDiskSize));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pInfo->walInDiskSize));
TAOS_CHECK_EXIT(tEncodeU64(&encoder, pInfo->rawDataSize));
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeBlockDbUsage(void* buf, int32_t bufLen, SDBBlockUsageInfo* pInfo) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pInfo->dataInDiskSize));
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pInfo->walInDiskSize));
TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pInfo->rawDataSize));
_exit:
tDecoderClear(&decoder);
return code;
}
int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SDBBlockUsageInfo* pData = GET_ROWCELL_INTERBUF(pResInfo);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
if (NULL == pColInfo) {
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t len = 0;
int32_t row = 0;
char st[256] = {0};
uint64_t totalDiskSize = pData->dataInDiskSize;
uint64_t rawDataSize = pData->rawDataSize;
double compressRadio = 0;
if (rawDataSize != 0) {
compressRadio = totalDiskSize * 100 / (double)rawDataSize;
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f]", compressRadio);
} else {
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]");
}
varDataSetLen(st, len);
int32_t code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
len =
tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Disk_occupied=[%" PRId64 "k]", pData->dataInDiskSize);
varDataSetLen(st, len);
code = colDataSetVal(pColInfo, row++, st, false);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
return code;
}
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SDerivInfo);
@ -6734,7 +6849,6 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
}
if ((INT64_MIN == pRateInfo->firstKey) || row.ts > pRateInfo->firstKey) {
doSaveRateInfo(pRateInfo, true, row.ts, row.pPk, v);
} else if (row.ts == pRateInfo->firstKey) {
@ -6795,7 +6909,8 @@ static void irateCopyInfo(SRateInfo* pInput, SRateInfo* pOutput) {
}
static int32_t irateTransferInfo(SRateInfo* pInput, SRateInfo* pOutput) {
if ((pInput->firstKey != INT64_MIN && (pInput->firstKey == pOutput->firstKey || pInput->firstKey == pOutput->lastKey)) ||
if ((pInput->firstKey != INT64_MIN &&
(pInput->firstKey == pOutput->firstKey || pInput->firstKey == pOutput->lastKey)) ||
(pInput->lastKey != INT64_MIN && (pInput->lastKey == pOutput->firstKey || pInput->lastKey == pOutput->lastKey))) {
return TSDB_CODE_FUNC_DUP_TIMESTAMP;
}
@ -6925,9 +7040,7 @@ _group_value_over:
return TSDB_CODE_SUCCESS;
}
int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
return groupConstValueFunction(pCtx);
}
int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { return groupConstValueFunction(pCtx); }
int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
@ -6956,9 +7069,7 @@ int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return code;
}
int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock){
return groupConstValueFinalize(pCtx, pBlock);
}
int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return groupConstValueFinalize(pCtx, pBlock); }
int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
@ -6980,7 +7091,8 @@ int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
if (IS_VAR_DATA_TYPE(pSourceCtx->resDataInfo.type)) {
(void)memcpy(pDBuf->data, pSBuf->data,
(pSourceCtx->resDataInfo.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(pSBuf->data) : varDataTLen(pSBuf->data));
(pSourceCtx->resDataInfo.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(pSBuf->data)
: varDataTLen(pSBuf->data));
} else {
(void)memcpy(pDBuf->data, pSBuf->data, pSourceCtx->resDataInfo.bytes);
}

22
source/libs/function/src/functionMgt.c
View File

@ -122,7 +122,8 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockIn
const char* name = funcMgtBuiltins[funcId].name;
if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) {
return FUNC_DATA_REQUIRED_NOT_LOAD;;
return FUNC_DATA_REQUIRED_NOT_LOAD;
;
}
if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) {
@ -188,7 +189,6 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWi
bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); }
bool fmIsSpecialDataRequiredFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED);
}
@ -299,6 +299,13 @@ bool fmIsBlockDistFunc(int32_t funcId) {
return FUNCTION_TYPE_BLOCK_DIST == funcMgtBuiltins[funcId].type;
}
bool fmIsDBUsageFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_DB_USAGE == funcMgtBuiltins[funcId].type;
}
bool fmIsProcessByRowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PROCESS_BY_ROW); }
bool fmIsIgnoreNullFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IGNORE_NULL_FUNC); }
@ -377,13 +384,9 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc) {
return true;
}
bool fmIsSkipScanCheckFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_SKIP_SCAN_CHECK_FUNC);
}
bool fmIsSkipScanCheckFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SKIP_SCAN_CHECK_FUNC); }
bool fmIsPrimaryKeyFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_PRIMARY_KEY_FUNC);
}
bool fmIsPrimaryKeyFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PRIMARY_KEY_FUNC); }
void getLastCacheDataType(SDataType* pType, int32_t pkBytes) {
// TODO: do it later.
pType->bytes = getFirstLastInfoSize(pType->bytes, pkBytes) + VARSTR_HEADER_SIZE;
@ -551,7 +554,8 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
return code;
}
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc) {
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc,
SFunctionNode** pMergeFunc) {
if (!fmIsDistExecFunc(pFunc->funcId)) {
return TSDB_CODE_FAILED;
}

10
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -298,6 +298,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowClusterMachinesStmt";
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return "ShowEncryptionsStmt";
case QUERY_NODE_SHOW_USAGE_STMT:
return "ShowUsageStmt";
case QUERY_NODE_DELETE_STMT:
return "DeleteStmt";
case QUERY_NODE_INSERT_STMT:
@ -5361,7 +5363,6 @@ static int32_t jsonToWindowOffsetNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkDatabaseOptionsBuffer = "Buffer";
static const char* jkDatabaseOptionsCacheModel = "CacheModel";
static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel";
@ -7446,6 +7447,9 @@ static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { r
static int32_t jsonToShowEncryptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showUsageStmtStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowUsageStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId";
static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern";
@ -8087,6 +8091,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showLocalVariablesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return showTableTagsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_USAGE_STMT:
return showUsageStmtStmtToJson(pObj, pJson);
case QUERY_NODE_DELETE_STMT:
return deleteStmtToJson(pObj, pJson);
case QUERY_NODE_INSERT_STMT:
@ -8450,6 +8456,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowLocalVariablesStmt(pJson, pObj);
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return jsonToShowTableTagsStmt(pJson, pObj);
case QUERY_NODE_SHOW_USAGE_STMT:
return jsonToShowUsageStmt(pJson, pObj);
case QUERY_NODE_DELETE_STMT:
return jsonToDeleteStmt(pJson, pObj);
case QUERY_NODE_INSERT_STMT:

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -679,6 +679,7 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT:
case QUERY_NODE_SHOW_USAGE_STMT:
code = makeNode(type, sizeof(SShowStmt), &pNode);
break;
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
@ -1515,7 +1516,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: {
case QUERY_NODE_SHOW_TSMAS_STMT:
case QUERY_NODE_SHOW_USAGE_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName);

1
source/libs/parser/inc/parAst.h
View File

@ -323,6 +323,7 @@ SNode* createDefaultTSMAOptions(SAstCreateContext* pCxt);
SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createShowCreateTSMAStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName);
SNode* createShowDiskUsageStmt(SAstCreateContext* pCxt, SNode* dbName, ENodeType type);
#ifdef __cplusplus
}

6
source/libs/parser/inc/parTranslater.h
View File

@ -20,10 +20,10 @@
extern "C" {
#endif
#include "cmdnodes.h"
#include "parToken.h"
#include "parUtil.h"
#include "parser.h"
#include "cmdnodes.h"
typedef struct STranslateContext {
SParseContext* pParseCxt;
@ -47,13 +47,15 @@ typedef struct STranslateContext {
SNode* pPrevRoot;
SNode* pPostRoot;
bool dual; // whether select stmt without from stmt, true for without.
bool skipCheck;
} STranslateContext;
int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRet);
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt);
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView);
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta,
bool couldBeView);
#ifdef __cplusplus
}

1
source/libs/parser/inc/sql.y
View File

@ -587,6 +587,7 @@ cmd ::= SHOW db_name_cond_opt(A) VIEWS like_pattern_opt(B).
cmd ::= SHOW CREATE VIEW full_table_name(A). { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, A); }
cmd ::= SHOW COMPACTS. { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); }
cmd ::= SHOW COMPACT NK_INTEGER(A). { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); }
cmd ::= SHOW db_name_cond_opt(A) DISK_INFO. { pCxt->pRootNode = createShowDiskUsageStmt(pCxt, A, QUERY_NODE_SHOW_USAGE_STMT); }
%type table_kind_db_name_cond_opt { SShowTablesOption }
%destructor table_kind_db_name_cond_opt { }

20
source/libs/parser/src/parAstCreater.c
View File

@ -2630,7 +2630,7 @@ static bool needDbShowStmt(ENodeType type) {
return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type ||
QUERY_NODE_SHOW_VGROUPS_STMT == type || QUERY_NODE_SHOW_INDEXES_STMT == type ||
QUERY_NODE_SHOW_TAGS_STMT == type || QUERY_NODE_SHOW_TABLE_TAGS_STMT == type ||
QUERY_NODE_SHOW_VIEWS_STMT == type || QUERY_NODE_SHOW_TSMAS_STMT == type;
QUERY_NODE_SHOW_VIEWS_STMT == type || QUERY_NODE_SHOW_TSMAS_STMT == type || QUERY_NODE_SHOW_USAGE_STMT == type;
}
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
@ -3939,3 +3939,21 @@ _err:
nodesDestroyNode(dbName);
return NULL;
}
SNode* createShowDiskUsageStmt(SAstCreateContext* pCxt, SNode* dbName, ENodeType type) {
CHECK_PARSER_STATUS(pCxt);
if (NULL == dbName) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
CHECK_PARSER_STATUS(pCxt);
}
SShowStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
pStmt->pDbName = dbName;
return (SNode*)pStmt;
_err:
nodesDestroyNode(dbName);
return NULL;
}

18
source/libs/parser/src/parAstParser.c
View File

@ -184,7 +184,7 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
}
if (TSDB_CODE_SUCCESS == code &&
(0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) ||
0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) &&
0 == strcmp(pTable, TSDB_INS_TABLE_COLS) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE)) &&
QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) {
code = collectMetaKeyFromInsTags(pCxt);
}
@ -933,6 +933,20 @@ static int32_t collectMetaKeyFromDropTSMAStmt(SCollectMetaKeyCxt* pCxt, SDropTSM
return code;
}
static int32_t collectMetaKeyFromShowUsage(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_DISK_USAGE,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code =
reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal,
NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromShowTSMASStmt(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TSMAS,
pCxt->pMetaCache);
@ -1076,6 +1090,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowApps(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_USAGE_STMT:
return collectMetaKeyFromShowUsage(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_DELETE_STMT:
return collectMetaKeyFromDelete(pCxt, (SDeleteStmt*)pStmt);
case QUERY_NODE_INSERT_STMT:

18
source/libs/parser/src/parAuthenticator.c
View File

@ -53,7 +53,8 @@ static int32_t setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const c
return TSDB_CODE_SUCCESS;
}
static int32_t checkAuthImpl(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond, bool isView, bool effective) {
static int32_t checkAuthImpl(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond,
bool isView, bool effective) {
SParseContext* pParseCxt = pCxt->pParseCxt;
if (pParseCxt->isSuperUser) {
return TSDB_CODE_SUCCESS;
@ -82,15 +83,16 @@ static int32_t checkAuthImpl(SAuthCxt* pCxt, const char* pDbName, const char* pT
if (TSDB_CODE_SUCCESS == code && NULL != pCond) {
*pCond = authRes.pCond[auth_res_type];
}
return TSDB_CODE_SUCCESS == code ? (authRes.pass[auth_res_type] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
return TSDB_CODE_SUCCESS == code ? (authRes.pass[auth_res_type] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED)
: code;
}
static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, pCond, false, false);
}
static int32_t checkEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
static int32_t checkEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type,
SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, false, true);
}
@ -98,7 +100,8 @@ static int32_t checkViewAuth(SAuthCxt* pCxt, const char* pDbName, const char* pT
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, true, false);
}
static int32_t checkViewEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
static int32_t checkViewEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type,
SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, true, true);
}
@ -247,6 +250,10 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) {
return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, NULL, AUTH_TYPE_READ_OR_WRITE, NULL);
}
static int32_t authShowUsage(SAuthCxt* pCxt, SShowStmt* pStmt) {
return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, NULL, AUTH_TYPE_READ_OR_WRITE, NULL);
}
static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) {
SNode* pTagCond = NULL;
// todo check tag condition for subtable
@ -380,6 +387,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_USAGE_STMT:
return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
case QUERY_NODE_SHOW_TABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT:

4
source/libs/parser/src/parCalcConst.c
View File

@ -363,6 +363,9 @@ static int32_t calcConstSelectFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect
pSelect->isEmptyResult = true;
return code;
}
if (pSelect->mixSysTableAndActualTable) {
return code;
}
if (TSDB_CODE_SUCCESS == code) {
code = calcConstProjections(pCxt, pSelect, subquery);
}
@ -621,6 +624,7 @@ int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) {
.msgBuf.buf = pParseCxt->pMsg,
.msgBuf.len = pParseCxt->msgLen,
.code = TSDB_CODE_SUCCESS};
int32_t code = calcConstQuery(&cxt, pQuery->pRoot, false);
if (TSDB_CODE_SUCCESS == code) {
resetProjectNullType(pQuery->pRoot);

1
source/libs/parser/src/parTokenizer.c
View File

@ -350,6 +350,7 @@ static SKeyword keywordTable[] = {
{"ARBGROUPS", TK_ARBGROUPS},
{"IS_IMPORT", TK_IS_IMPORT},
{"FORCE_WINDOW_CLOSE", TK_FORCE_WINDOW_CLOSE},
{"DISK_INFO", TK_DISK_INFO},
};
// clang-format on

120
source/libs/parser/src/parTranslater.c
View File

@ -350,8 +350,7 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_SHOW_ANODES_STMT,
{ .showType = QUERY_NODE_SHOW_ANODES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_ANODES,
.numOfShowCols = 1,
@ -364,6 +363,13 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_SHOW_USAGE_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_DISK_USAGE,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
};
// clang-format on
@ -1501,6 +1507,17 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef,
break;
}
}
if (pCxt->showRewrite && pMeta->tableType == TSDB_SYSTEM_TABLE) {
if (strncmp(pCol->dbName, TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)) == 0 &&
strncmp(pCol->tableName, TSDB_INS_DISK_USAGE, strlen(TSDB_INS_DISK_USAGE)) == 0 &&
strncmp(pCol->colName, "db_name", strlen("db_name")) == 0) {
pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
pCol->node.resType.bytes = 8;
pCxt->skipCheck = true;
((SSelectStmt*)pCxt->pCurrStmt)->mixSysTableAndActualTable = true;
}
}
} else {
STempTableNode* pTempTable = (STempTableNode*)pTable;
SNodeList* pProjectList = getProjectList(pTempTable->pSubquery);
@ -2781,6 +2798,26 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateDBUsageFunc(STranslateContext* pCtx, SFunctionNode* pFunc) {
if (!fmIsDBUsageFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (!isSelectStmt(pCtx->pCurrStmt)) {
return generateSyntaxErrMsgExt(&pCtx->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE,
"%s is only supported in single table query", pFunc->functionName);
}
SSelectStmt* pSelect = (SSelectStmt*)pCtx->pCurrStmt;
SNode* pTable = pSelect->pFromTable;
// if (NULL != pTable && (QUERY_NODE_REAL_TABLE != nodeType(pTable) ||
// (TSDB_SUPER_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType &&
// TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType &&
// TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType))) {
// return generateSyntaxErrMsgExt(&pCtx->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
// "%s is only supported on super table, child table or normal table",
// pFunc->functionName);
// }
return TSDB_CODE_SUCCESS;
}
static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); }
@ -3138,6 +3175,10 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode)
if (TSDB_CODE_SUCCESS == code) {
code = translateBlockDistFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateDBUsageFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
setFuncClassification(pCxt, pFunc);
}
@ -4077,7 +4118,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) {
static bool sysTableFromVnode(const char* pTable) {
return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) ||
(0 == strcmp(pTable, TSDB_INS_TABLE_COLS)));
(0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE));
}
static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
@ -4148,8 +4189,9 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName,
}
if (TSDB_CODE_SUCCESS == code &&
(0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) ||
0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS)) {
((0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) ||
0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) ||
(0 == strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && !hasUserDbCond))) {
code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs);
}
@ -4248,7 +4290,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) {
if (TSDB_SYSTEM_TABLE == tableType) {
return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS);
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE);
}
return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType);
}
@ -13407,6 +13450,12 @@ static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt)
static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) {
return createSimpleSelectStmtFromCols(pStmt->dbName, pStmt->tableName, 0, NULL, pOutput);
}
static int32_t createSelectStmtForShowDBUsage(SShowStmt* pStmt, SSelectStmt** pOutput) {
int32_t type = nodeType(pStmt);
const SSysTableShowAdapter* pShow = &sysTableShowAdapter[type - SYSTABLE_SHOW_TYPE_OFFSET];
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols,
pOutput);
}
static int32_t createOperatorNode(EOperatorType opType, const char* pColName, const SNode* pRight, SNode** pOp) {
if (NULL == pRight) {
@ -13938,6 +13987,62 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) {
return code;
}
static int32_t createBlockDBUsageInfoFunc(SFunctionNode** ppNode) {
SFunctionNode* pFunc = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc);
if (NULL == pFunc) {
return code;
}
strcpy(pFunc->functionName, "_db_usage_info");
strcpy(pFunc->node.aliasName, "_db_usage_info");
*ppNode = pFunc;
return code;
}
static int32_t createDBUsageFunc(SFunctionNode** ppNode) {
SFunctionNode* pFunc = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc);
if (NULL == pFunc) {
return code;
}
strcpy(pFunc->functionName, "_db_usage");
strcpy(pFunc->node.aliasName, "_db_usage");
SFunctionNode* pFuncNew = NULL;
code = createBlockDBUsageInfoFunc(&pFuncNew);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pFunc->pParameterList, (SNode*)pFuncNew);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc);
return code;
}
*ppNode = pFunc;
return code;
}
static int32_t rewriteShowDBUsage(STranslateContext* pCtx, SQuery* pQuery) {
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShowDBUsage((SShowStmt*)pQuery->pRoot, &pStmt);
if (TSDB_CODE_SUCCESS == code) {
NODES_DESTORY_LIST(pStmt->pProjectionList);
SFunctionNode* pFuncNew = NULL;
code = createDBUsageFunc(&pFuncNew);
if (TSDB_CODE_SUCCESS == code) code = nodesListMakeStrictAppend(&pStmt->pProjectionList, (SNode*)pFuncNew);
}
if (TSDB_CODE_SUCCESS == code) {
code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
pCtx->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pStmt;
}
return code;
}
typedef struct SVgroupCreateTableBatch {
SVCreateTbBatchReq req;
SVgroupInfo info;
@ -16460,6 +16565,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_TSMAS_STMT:
code = rewriteShow(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_USAGE_STMT:
code = rewriteShowDBUsage(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_TAGS_STMT:
code = rewriteShowTags(pCxt, pQuery);
break;

37
source/libs/planner/src/planLogicCreater.c
View File

@ -13,11 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "planInt.h"
#include "filter.h"
#include "functionMgt.h"
#include "tglobal.h"
#include "parser.h"
#include "planInt.h"
#include "tglobal.h"
// primary key column always the second column if exists
#define PRIMARY_COLUMN_SLOT 1
@ -164,7 +164,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) {
nodesWalkExpr(pExpr, doNameExpr, NULL);
bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false;
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
SRewriteExprCxt cxt = {
.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr);
if (TSDB_CODE_SUCCESS == cxt.errCode) {
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
@ -198,7 +199,8 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES
SNodeList** pRewriteExprs) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false;
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
SRewriteExprCxt cxt = {
.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
if (NULL != pRewriteExprs) {
cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool));
if (NULL == cxt.pOutputs) {
@ -271,6 +273,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
}
if (TSDB_SYSTEM_TABLE == tableType) {
// if (NULL != pScanPseudoCols &&
// FUNCTION_TYPE_DB_USAGE_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType) {
// return SCAN_TYPE_BLOCK_INFO;
// }
return SCAN_TYPE_SYSTEM_TABLE;
}
@ -427,7 +433,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols);
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc,
&pScan->pScanPseudoCols);
}
pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan);
@ -555,7 +562,8 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = nodesMakeList(&pJoin->node.pChildren);
}
pJoin->seqWinGroup = (JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc);
pJoin->seqWinGroup =
(JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc);
SLogicNode* pLeft = NULL;
if (TSDB_CODE_SUCCESS == code) {
@ -649,7 +657,6 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
#endif
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pJoin;
} else {
@ -732,11 +739,13 @@ static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt*
}
static bool isWindowJoinStmt(SSelectStmt* pSelect) {
return (QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) && IS_WINDOW_JOIN(((SJoinTableNode*)pSelect->pFromTable)->subType));
return (QUERY_NODE_JOIN_TABLE == nodeType(pSelect->pFromTable) &&
IS_WINDOW_JOIN(((SJoinTableNode*)pSelect->pFromTable)->subType));
}
static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP
return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct)
? GROUP_ACTION_KEEP
: GROUP_ACTION_NONE;
}
@ -923,7 +932,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
}
static bool isInterpFunc(int32_t funcId) {
return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmisSelectGroupConstValueFunc(funcId);
return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) ||
fmisSelectGroupConstValueFunc(funcId);
}
static void initStreamOption(SLogicPlanContext* pCxt, SStreamNodeOption* pOption) {
@ -1002,7 +1012,8 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
}
static bool isForecastFunc(int32_t funcId) {
return fmIsForecastFunc(funcId) || fmIsForecastPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmisSelectGroupConstValueFunc(funcId);
return fmIsForecastFunc(funcId) || fmIsForecastPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) ||
fmisSelectGroupConstValueFunc(funcId);
}
static int32_t createForecastFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
@ -1365,9 +1376,7 @@ static EDealRes needFillValueImpl(SNode* pNode, void* pContext) {
return res;
}
static void needFillValue(SNode* pNode, SWalkFillSubExprCtx* pCtx) {
nodesWalkExpr(pNode, needFillValueImpl, pCtx);
}
static void needFillValue(SNode* pNode, SWalkFillSubExprCtx* pCtx) { nodesWalkExpr(pNode, needFillValueImpl, pCtx); }
static int32_t collectFillExpr(SNode* pNode, SCollectFillExprsCtx* pCollectFillCtx) {
SNode* pNew = NULL;

270
source/libs/planner/src/planOptimizer.c
View File

@ -15,11 +15,11 @@
#include "filter.h"
#include "functionMgt.h"
#include "parser.h"
#include "planInt.h"
#include "systable.h"
#include "tglobal.h"
#include "ttime.h"
#include "parser.h"
#define OPTIMIZE_FLAG_MASK(n) (1 << n)
@ -86,7 +86,6 @@ typedef struct SCpdCollRewriteTableColsCxt {
SSHashObj* pRightCols;
} SCpdCollRewriteTableColsCxt;
typedef struct SCpdCollectTableColCxt {
SSHashObj* pTables;
SNodeList* pResCols;
@ -94,7 +93,6 @@ typedef struct SCpdCollectTableColCxt {
int32_t errCode;
} SCpdCollectTableColCxt;
typedef enum ECondAction {
COND_ACTION_STAY = 1,
COND_ACTION_PUSH_JOIN,
@ -128,8 +126,15 @@ static SJoinOptimizeOpt gJoinOpt[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = {
static SJoinOptimizeOpt gJoinWhereOpt[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = {
/* NONE OUTER SEMI ANTI ASOF WINDOW */
/*INNER*/ {{PUSH_DOWN_ALL_COND}, {0}, {0}, {0}, {0}, {0}},
/*LEFT*/ {{0}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}},
/*RIGHT*/ {{0}, {PUSH_DOWN_RIGHT_FLT}, {PUSH_DOWN_RIGHT_FLT},{PUSH_DOWN_RIGHT_FLT}, {PUSH_DOWN_RIGHT_FLT}, {PUSH_DOWN_RIGHT_FLT}},
/*LEFT*/
{{0}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}, {PUSH_DOWN_LEFT_FLT}},
/*RIGHT*/
{{0},
{PUSH_DOWN_RIGHT_FLT},
{PUSH_DOWN_RIGHT_FLT},
{PUSH_DOWN_RIGHT_FLT},
{PUSH_DOWN_RIGHT_FLT},
{PUSH_DOWN_RIGHT_FLT}},
/*FULL*/ {{0}, {0}, {0}, {0}, {0}, {0}},
};
@ -141,7 +146,6 @@ static SJoinOptimizeOpt gJoinOnOpt[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] =
/*FULL*/ {{0}, {0}, {0}, {0}, {0}, {0}},
};
#endif
static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func, void* pCtx) {
@ -590,7 +594,8 @@ static bool pdcColBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
if (QUERY_NODE_COLUMN == nodeType(pCondCol) && QUERY_NODE_COLUMN == nodeType(pTableCol)) {
SColumnNode* pCondColNode = (SColumnNode*)pCondCol;
SColumnNode* pTblColNode = (SColumnNode*)pTableCol;
if (0 == strcmp(pCondColNode->tableAlias, pTblColNode->tableAlias) && 0 == strcmp(pCondColNode->colName, pTblColNode->colName)) {
if (0 == strcmp(pCondColNode->tableAlias, pTblColNode->tableAlias) &&
0 == strcmp(pCondColNode->colName, pTblColNode->colName)) {
return true;
}
}
@ -646,14 +651,16 @@ static ECondAction pdcJoinGetCondAction(SJoinLogicNode* pJoin, SSHashObj* pLeftT
}
return COND_ACTION_STAY;
}
if ((whereCond && gJoinWhereOpt[t][s].pushDownFlag & PUSH_DOWN_LEFT_FLT) || (!whereCond && gJoinOnOpt[t][s].pushDownFlag & PUSH_DOWN_LEFT_FLT)) {
if ((whereCond && gJoinWhereOpt[t][s].pushDownFlag & PUSH_DOWN_LEFT_FLT) ||
(!whereCond && gJoinOnOpt[t][s].pushDownFlag & PUSH_DOWN_LEFT_FLT)) {
return COND_ACTION_PUSH_LEFT_CHILD;
}
return COND_ACTION_STAY;
}
if (cxt.haveRightCol) {
if ((whereCond && gJoinWhereOpt[t][s].pushDownFlag & PUSH_DOWN_RIGHT_FLT) || (!whereCond && gJoinOnOpt[t][s].pushDownFlag & PUSH_DOWN_RIGHT_FLT)) {
if ((whereCond && gJoinWhereOpt[t][s].pushDownFlag & PUSH_DOWN_RIGHT_FLT) ||
(!whereCond && gJoinOnOpt[t][s].pushDownFlag & PUSH_DOWN_RIGHT_FLT)) {
return COND_ACTION_PUSH_RIGHT_CHILD;
}
return COND_ACTION_STAY;
@ -694,7 +701,9 @@ static int32_t pdcJoinSplitLogicCond(SJoinLogicNode* pJoin, SNode** pSrcCond, SN
ECondAction condAction = pdcJoinGetCondAction(pJoin, pLeftTables, pRightTables, pCond, whereCond);
SNode* pNew = NULL;
code = nodesCloneNode(pCond, &pNew);
if (TSDB_CODE_SUCCESS != code) { break; }
if (TSDB_CODE_SUCCESS != code) {
break;
}
if (COND_ACTION_PUSH_JOIN == condAction && NULL != pOnCond) {
code = nodesListMakeAppend(&pOnConds, pNew);
} else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) {
@ -963,9 +972,13 @@ static int32_t pdcJoinIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond, bool* a
return code;
}
SOperatorNode* pOper = (SOperatorNode*)pCond;
if ((QUERY_NODE_COLUMN != nodeType(pOper->pLeft) && !(QUERY_NODE_OPERATOR == nodeType(pOper->pLeft) && OP_TYPE_JSON_GET_VALUE ==((SOperatorNode*)pOper->pLeft)->opType))
|| NULL == pOper->pRight ||
(QUERY_NODE_COLUMN != nodeType(pOper->pRight) && !(QUERY_NODE_OPERATOR == nodeType(pOper->pRight) && OP_TYPE_JSON_GET_VALUE ==((SOperatorNode*)pOper->pRight)->opType))) {
if ((QUERY_NODE_COLUMN != nodeType(pOper->pLeft) &&
!(QUERY_NODE_OPERATOR == nodeType(pOper->pLeft) &&
OP_TYPE_JSON_GET_VALUE == ((SOperatorNode*)pOper->pLeft)->opType)) ||
NULL == pOper->pRight ||
(QUERY_NODE_COLUMN != nodeType(pOper->pRight) &&
!(QUERY_NODE_OPERATOR == nodeType(pOper->pRight) &&
OP_TYPE_JSON_GET_VALUE == ((SOperatorNode*)pOper->pRight)->opType))) {
return code;
}
@ -1192,8 +1205,7 @@ static int32_t pdcJoinCollectColsFromParent(SJoinLogicNode* pJoin, SSHashObj* pT
.errCode = TSDB_CODE_SUCCESS,
.pTables = pTables,
.pResCols = pCondCols,
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)
};
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
if (NULL == cxt.pColHash) {
return terrno;
}
@ -1281,7 +1293,6 @@ static int32_t pdcJoinAddParentOnColsToTarget(SOptimizeContext* pCxt, SJoinLogic
return code;
}
static int32_t pdcJoinAddPreFilterColsToTarget(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pFullOnCond) {
return TSDB_CODE_SUCCESS;
@ -1388,7 +1399,6 @@ static int32_t pdcJoinAddFilterColsToTarget(SOptimizeContext* pCxt, SJoinLogicNo
return code;
}
static int32_t pdcJoinCheckAllCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pFullOnCond) {
if (IS_WINDOW_JOIN(pJoin->subType) || IS_ASOF_JOIN(pJoin->subType)) {
@ -1407,11 +1417,13 @@ static int32_t pdcJoinCheckAllCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin
SNode* pCond = pJoin->pFullOnCond ? pJoin->pFullOnCond : pJoin->node.pConditions;
bool errCond = false;
if (!pdcJoinHasPrimEqualCond(pJoin, pCond, &errCond)) {
if (errCond && !(IS_INNER_NONE_JOIN(pJoin->joinType, pJoin->subType) && NULL != pJoin->pFullOnCond && NULL != pJoin->node.pConditions)) {
if (errCond && !(IS_INNER_NONE_JOIN(pJoin->joinType, pJoin->subType) && NULL != pJoin->pFullOnCond &&
NULL != pJoin->node.pConditions)) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
}
if (IS_INNER_NONE_JOIN(pJoin->joinType, pJoin->subType) && NULL != pJoin->pFullOnCond && NULL != pJoin->node.pConditions) {
if (IS_INNER_NONE_JOIN(pJoin->joinType, pJoin->subType) && NULL != pJoin->pFullOnCond &&
NULL != pJoin->node.pConditions) {
if (pdcJoinHasPrimEqualCond(pJoin, pJoin->node.pConditions, &errCond)) {
return TSDB_CODE_SUCCESS;
}
@ -1457,13 +1469,15 @@ static int32_t pdcJoinHandleGrpJoinCond(SOptimizeContext* pCxt, SJoinLogicNode*
FOREACH(pNode, pJoin->pLeftEqNodes) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (COLUMN_TYPE_TAG != pCol->colType && PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen,
TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
}
}
FOREACH(pNode, pJoin->pRightEqNodes) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (COLUMN_TYPE_TAG != pCol->colType && PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen,
TSDB_CODE_PLAN_NOT_SUPPORT_JOIN_COND);
}
}
}
@ -1501,9 +1515,13 @@ static EDealRes pdcCheckTableCondType(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t pdcJoinGetOpTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables, bool* tableCondTypes) {
SCpdIsMultiTableCondCxt cxt = {
.pLeftTbls = pLeftTables, .pRightTbls = pRightTables, .havaLeftCol = false, .haveRightCol = false, .condIsNull = false};
static int32_t pdcJoinGetOpTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables,
bool* tableCondTypes) {
SCpdIsMultiTableCondCxt cxt = {.pLeftTbls = pLeftTables,
.pRightTbls = pRightTables,
.havaLeftCol = false,
.haveRightCol = false,
.condIsNull = false};
nodesWalkExpr(pCond, pdcCheckTableCondType, &cxt);
if (cxt.havaLeftCol) {
@ -1538,9 +1556,8 @@ static int32_t pdcJoinGetOpTableCondTypes(SNode* pCond, SSHashObj* pLeftTables,
return TSDB_CODE_SUCCESS;
}
static int32_t pdcJoinGetLogicTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables, bool* tableCondTypes) {
static int32_t pdcJoinGetLogicTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables,
bool* tableCondTypes) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond;
int32_t code = 0;
SNode* pSCond = NULL;
@ -1559,7 +1576,8 @@ static int32_t pdcJoinGetLogicTableCondTypes(SNode* pCond, SSHashObj* pLeftTable
return code;
}
static int32_t pdcGetTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables, bool* tableCondTypes) {
static int32_t pdcGetTableCondTypes(SNode* pCond, SSHashObj* pLeftTables, SSHashObj* pRightTables,
bool* tableCondTypes) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) {
return pdcJoinGetLogicTableCondTypes(pCond, pLeftTables, pRightTables, tableCondTypes);
} else {
@ -1640,8 +1658,12 @@ static EDealRes pdcCheckTableResType(SNode* pNode, void* pContext) {
break;
case QUERY_NODE_FUNCTION: {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
SCpdIsMultiTableResCxt cxt = {.pLeftTbls = pCxt->pLeftTbls, .pRightTbls = pCxt->pRightTbls,
.haveLeftCol = false, .haveRightCol = false, .leftColNonNull = true, .rightColNonNull = true};
SCpdIsMultiTableResCxt cxt = {.pLeftTbls = pCxt->pLeftTbls,
.pRightTbls = pCxt->pRightTbls,
.haveLeftCol = false,
.haveRightCol = false,
.leftColNonNull = true,
.rightColNonNull = true};
nodesWalkExprs(pFunc->pParameterList, pdcCheckTableResType, &cxt);
if (!cxt.leftColNonNull) {
@ -1715,8 +1737,14 @@ static int32_t pdcRewriteTypeBasedOnJoinRes(SOptimizeContext* pCxt, SJoinLogicNo
tableResNonNull[0] = false;
tableResNonNull[1] = false;
} else {
SCpdIsMultiTableResCxt cxt = {.pLeftTbls = pLeftTables, .pRightTbls = pRightTables,
.haveLeftCol = false, .haveRightCol = false, .leftColNonNull = true, .rightColNonNull = true, .leftColOp = false, .rightColOp = false};
SCpdIsMultiTableResCxt cxt = {.pLeftTbls = pLeftTables,
.pRightTbls = pRightTables,
.haveLeftCol = false,
.haveRightCol = false,
.leftColNonNull = true,
.rightColNonNull = true,
.leftColOp = false,
.rightColOp = false};
nodesWalkExprs(pAgg->pAggFuncs, pdcCheckTableResType, &cxt);
if (!cxt.leftColNonNull) {
@ -1797,7 +1825,8 @@ static int32_t pdcDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
code = pdcPushDownCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) {
code = pdcPushDownCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
code =
pdcPushDownCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoin->node.pConditions) {
@ -1827,7 +1856,8 @@ static int32_t pdcDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoin->pFullOnCond && !IS_WINDOW_JOIN(pJoin->subType) && NULL == pJoin->addPrimEqCond) {
if (TSDB_CODE_SUCCESS == code && NULL != pJoin->pFullOnCond && !IS_WINDOW_JOIN(pJoin->subType) &&
NULL == pJoin->addPrimEqCond) {
code = pdcJoinSplitPrimEqCond(pCxt, pJoin);
}
@ -2145,7 +2175,6 @@ static int32_t pdcOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan)
return pdcOptimizeImpl(pCxt, pLogicSubplan->pNode);
}
static bool eliminateNotNullCondMayBeOptimized(SLogicNode* pNode, void* pCtx) {
if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode)) {
return false;
@ -2220,7 +2249,6 @@ static int32_t eliminateNotNullCondOptimize(SOptimizeContext* pCxt, SLogicSubpla
return TSDB_CODE_SUCCESS;
}
static bool sortPriKeyOptIsPriKeyOrderBy(SNodeList* pSortKeys) {
if (1 != LIST_LENGTH(pSortKeys)) {
return false;
@ -2255,7 +2283,8 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode, void* pCtx) {
return true;
}
static int32_t sortPriKeyOptHandleLeftRightJoinSort(SJoinLogicNode* pJoin, SSortLogicNode* pSort, bool* pNotOptimize, bool* keepSort) {
static int32_t sortPriKeyOptHandleLeftRightJoinSort(SJoinLogicNode* pJoin, SSortLogicNode* pSort, bool* pNotOptimize,
bool* keepSort) {
if (JOIN_STYPE_SEMI == pJoin->subType || JOIN_STYPE_NONE == pJoin->subType) {
return TSDB_CODE_SUCCESS;
}
@ -2321,9 +2350,8 @@ static int32_t sortPriKeyOptHandleLeftRightJoinSort(SJoinLogicNode* pJoin, SSort
return TSDB_CODE_SUCCESS;
}
static int32_t sortPriKeyOptHandleJoinSort(SLogicNode* pNode, bool groupSort, SSortLogicNode* pSort,
bool* pNotOptimize, SNodeList** pSequencingNodes, bool* keepSort) {
static int32_t sortPriKeyOptHandleJoinSort(SLogicNode* pNode, bool groupSort, SSortLogicNode* pSort, bool* pNotOptimize,
SNodeList** pSequencingNodes, bool* keepSort) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode;
int32_t code = TSDB_CODE_SUCCESS;
@ -2343,22 +2371,20 @@ static int32_t sortPriKeyOptHandleJoinSort(SLogicNode* pNode, bool groupSort, SS
break;
}
code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort,
pSort, pNotOptimize, pSequencingNodes, keepSort);
code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, pSort,
pNotOptimize, pSequencingNodes, keepSort);
if (TSDB_CODE_SUCCESS == code) {
code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), groupSort,
pSort, pNotOptimize, pSequencingNodes, keepSort);
code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), groupSort, pSort,
pNotOptimize, pSequencingNodes, keepSort);
}
return code;
}
static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) {
return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order;
}
static bool sortPriKeyOptHasUnsupportedPkFunc(SLogicNode* pLogicNode, EOrder sortOrder) {
if (sortOrder == ORDER_ASC) {
return false;
@ -2394,10 +2420,8 @@ static bool sortPriKeyOptHasUnsupportedPkFunc(SLogicNode* pLogicNode, EOrder sor
}
SFunctionNode* pFuncNode = (SFunctionNode*)pLogicNode;
if (pFuncNode->hasPk &&
(pFuncNode->funcType == FUNCTION_TYPE_DIFF ||
pFuncNode->funcType == FUNCTION_TYPE_DERIVATIVE ||
pFuncNode->funcType == FUNCTION_TYPE_IRATE ||
pFuncNode->funcType == FUNCTION_TYPE_TWA)) {
(pFuncNode->funcType == FUNCTION_TYPE_DIFF || pFuncNode->funcType == FUNCTION_TYPE_DERIVATIVE ||
pFuncNode->funcType == FUNCTION_TYPE_IRATE || pFuncNode->funcType == FUNCTION_TYPE_TWA)) {
return true;
}
}
@ -2460,11 +2484,10 @@ int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, S
pNotOptimize, pSequencingNodes, keepSort);
}
static int32_t sortPriKeyOptGetSequencingNodes(SSortLogicNode* pSort, bool groupSort, SNodeList** pSequencingNodes, bool* keepSort) {
static int32_t sortPriKeyOptGetSequencingNodes(SSortLogicNode* pSort, bool groupSort, SNodeList** pSequencingNodes,
bool* keepSort) {
bool notOptimize = false;
int32_t code =
sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), groupSort,
int32_t code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), groupSort,
pSort, &notOptimize, pSequencingNodes, keepSort);
if (TSDB_CODE_SUCCESS != code || notOptimize) {
NODES_CLEAR_LIST(*pSequencingNodes);
@ -2556,12 +2579,14 @@ static int32_t sortForJoinOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pL
EOrder targetOrder = 0;
SSHashObj* pTables = NULL;
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pLeft) && ((SScanLogicNode*)pLeft)->node.outputTsOrder != SCAN_ORDER_BOTH) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pLeft) &&
((SScanLogicNode*)pLeft)->node.outputTsOrder != SCAN_ORDER_BOTH) {
pScan = (SScanLogicNode*)pLeft;
pChild = pRight;
pChildPos = &pJoin->node.pChildren->pTail->pNode;
targetOrder = pScan->node.outputTsOrder;
} else if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pRight) && ((SScanLogicNode*)pRight)->node.outputTsOrder != SCAN_ORDER_BOTH) {
} else if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pRight) &&
((SScanLogicNode*)pRight)->node.outputTsOrder != SCAN_ORDER_BOTH) {
pScan = (SScanLogicNode*)pRight;
pChild = pLeft;
pChildPos = &pJoin->node.pChildren->pHead->pNode;
@ -2588,9 +2613,11 @@ static int32_t sortForJoinOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pL
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL != tSimpleHashGet(pTables, ((SColumnNode*)pOp->pLeft)->tableAlias, strlen(((SColumnNode*)pOp->pLeft)->tableAlias))) {
if (NULL !=
tSimpleHashGet(pTables, ((SColumnNode*)pOp->pLeft)->tableAlias, strlen(((SColumnNode*)pOp->pLeft)->tableAlias))) {
pOrderByNode = pOp->pLeft;
} else if (NULL != tSimpleHashGet(pTables, ((SColumnNode*)pOp->pRight)->tableAlias, strlen(((SColumnNode*)pOp->pRight)->tableAlias))) {
} else if (NULL != tSimpleHashGet(pTables, ((SColumnNode*)pOp->pRight)->tableAlias,
strlen(((SColumnNode*)pOp->pRight)->tableAlias))) {
pOrderByNode = pOp->pRight;
}
@ -2642,7 +2669,8 @@ static int32_t sortForJoinOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pL
return code;
}
*pChildPos = (SNode*)pSort;
pSort->node.pParent = (SLogicNode*)pJoin;;
pSort->node.pParent = (SLogicNode*)pJoin;
;
_return:
@ -2651,7 +2679,6 @@ _return:
return TSDB_CODE_SUCCESS;
}
static bool sortForJoinOptMayBeOptimized(SLogicNode* pNode, void* pCtx) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode)) {
return false;
@ -2679,9 +2706,9 @@ static bool sortForJoinOptMayBeOptimized(SLogicNode* pNode, void* pCtx) {
return true;
}
static int32_t sortForJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortForJoinOptMayBeOptimized, NULL);
SJoinLogicNode* pJoin =
(SJoinLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortForJoinOptMayBeOptimized, NULL);
if (NULL == pJoin) {
return TSDB_CODE_SUCCESS;
}
@ -2724,14 +2751,15 @@ static int32_t joinCondGetAllScanNodes(SLogicNode* pNode, SNodeList** pList) {
}
}
static bool joinCondMayBeOptimized(SLogicNode* pNode, void* pCtx) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode) || OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode) ||
OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
return false;
}
SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode;
if (pNode->pChildren->length != 2 || JOIN_STYPE_ASOF == pJoin->subType || JOIN_STYPE_WIN == pJoin->subType || JOIN_TYPE_FULL == pJoin->joinType) {
if (pNode->pChildren->length != 2 || JOIN_STYPE_ASOF == pJoin->subType || JOIN_STYPE_WIN == pJoin->subType ||
JOIN_TYPE_FULL == pJoin->joinType) {
OPTIMIZE_FLAG_SET_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND);
return false;
}
@ -2739,21 +2767,26 @@ static bool joinCondMayBeOptimized(SLogicNode* pNode, void* pCtx) {
SLogicNode* pLeft = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0);
SLogicNode* pRight = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1);
if ((JOIN_TYPE_LEFT == pJoin->joinType || JOIN_TYPE_RIGHT == pJoin->joinType) && (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pLeft) || QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pRight))) {
if ((JOIN_TYPE_LEFT == pJoin->joinType || JOIN_TYPE_RIGHT == pJoin->joinType) &&
(QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pLeft) || QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pRight))) {
OPTIMIZE_FLAG_SET_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND);
return false;
}
if (JOIN_TYPE_INNER == pJoin->joinType && ((QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pLeft) && QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pLeft)) || (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pRight) && QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pRight)))) {
if (JOIN_TYPE_INNER == pJoin->joinType &&
((QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pLeft) && QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pLeft)) ||
(QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pRight) && QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pRight)))) {
OPTIMIZE_FLAG_SET_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND);
return false;
}
if (JOIN_TYPE_INNER == pJoin->joinType) {
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pLeft) && !OPTIMIZE_FLAG_TEST_MASK(pLeft->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pLeft) &&
!OPTIMIZE_FLAG_TEST_MASK(pLeft->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
return false;
}
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pRight) && !OPTIMIZE_FLAG_TEST_MASK(pRight->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pRight) &&
!OPTIMIZE_FLAG_TEST_MASK(pRight->optimizedFlag, OPTIMIZE_FLAG_JOIN_COND)) {
return false;
}
}
@ -2802,9 +2835,7 @@ static int32_t joinCondOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
}
SNode* pNode = NULL;
STimeWindow scanRange = TSWINDOW_INITIALIZER;
FOREACH(pNode, pScanList) {
joinCondMergeScanRand(&scanRange, &((SScanLogicNode*)pNode)->scanRange);
}
FOREACH(pNode, pScanList) { joinCondMergeScanRand(&scanRange, &((SScanLogicNode*)pNode)->scanRange); }
FOREACH(pNode, pScanList) {
((SScanLogicNode*)pNode)->scanRange.skey = scanRange.skey;
((SScanLogicNode*)pNode)->scanRange.ekey = scanRange.ekey;
@ -3349,7 +3380,8 @@ static int32_t eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectN
static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode, void* pCtx) {
// Super table scan requires project operator to merge packets to improve performance.
if (NULL == pNode->pParent && (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
if (NULL == pNode->pParent &&
(QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
(QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0)) &&
TSDB_SUPER_TABLE == ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->tableType))) {
return false;
@ -3359,8 +3391,10 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode, void* pCtx) {
return false;
}
if (NULL != pNode->pParent && (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) || QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode->pParent))) {
if (NULL != pNode->pParent &&
(QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) ||
QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode->pParent))) {
return false;
}
@ -3477,7 +3511,6 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte
return DEAL_RES_CONTINUE;
}
static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* pChildTargets) {
SNode *pChildTarget = NULL, *pParentProject = NULL;
FOREACH(pChildTarget, pChildTargets) {
@ -4139,9 +4172,13 @@ static bool lastRowScanOptCheckColNum(int32_t lastColNum, col_id_t lastColId, in
static bool isNeedSplitCacheLastFunc(SFunctionNode* pFunc, SScanLogicNode* pScan) {
int32_t funcType = pFunc->funcType;
if ((FUNCTION_TYPE_LAST_ROW != funcType || (FUNCTION_TYPE_LAST_ROW == funcType && TSDB_CACHE_MODEL_LAST_VALUE == pScan->cacheLastMode)) &&
(FUNCTION_TYPE_LAST != funcType || (FUNCTION_TYPE_LAST == funcType && (TSDB_CACHE_MODEL_LAST_ROW == pScan->cacheLastMode ||
QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) ||
if ((FUNCTION_TYPE_LAST_ROW != funcType ||
(FUNCTION_TYPE_LAST_ROW == funcType && TSDB_CACHE_MODEL_LAST_VALUE == pScan->cacheLastMode)) &&
(FUNCTION_TYPE_LAST != funcType ||
(FUNCTION_TYPE_LAST == funcType &&
(TSDB_CACHE_MODEL_LAST_ROW == pScan->cacheLastMode ||
QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) ||
QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) ||
COLUMN_TYPE_COLUMN != ((SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0))->colType))) &&
FUNCTION_TYPE_SELECT_VALUE != funcType && FUNCTION_TYPE_GROUP_KEY != funcType) {
return true;
@ -4170,8 +4207,7 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, int8_t cacheLastModel
lastColId = pCol->colId;
lastColNum++;
}
}
else if (QUERY_NODE_VALUE == nodeType(pParam) || QUERY_NODE_OPERATOR == nodeType(pParam)) {
} else if (QUERY_NODE_VALUE == nodeType(pParam) || QUERY_NODE_OPERATOR == nodeType(pParam)) {
needSplitFuncCount++;
*hasOtherFunc = true;
}
@ -4217,7 +4253,8 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, int8_t cacheLastModel
}
static bool lastRowScanOptCheckLastCache(SAggLogicNode* pAgg, SScanLogicNode* pScan) {
if ((pAgg->hasLastRow == pAgg->hasLast && !pAgg->hasLastRow) || (!pAgg->hasLast && !pAgg->hasLastRow) || NULL != pAgg->pGroupKeys || NULL != pScan->node.pConditions ||
if ((pAgg->hasLastRow == pAgg->hasLast && !pAgg->hasLastRow) || (!pAgg->hasLast && !pAgg->hasLastRow) ||
NULL != pAgg->pGroupKeys || NULL != pScan->node.pConditions ||
!hasSuitableCache(pScan->cacheLastMode, pAgg->hasLastRow, pAgg->hasLast) ||
IS_TSWINDOW_SPECIFIED(pScan->scanRange)) {
return false;
@ -4314,7 +4351,8 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo
}
}
static void lastRowScanOptRemoveUslessTargets(SNodeList* pTargets, SNodeList* pList1, SNodeList* pList2, SNodeList* pList3) {
static void lastRowScanOptRemoveUslessTargets(SNodeList* pTargets, SNodeList* pList1, SNodeList* pList2,
SNodeList* pList3) {
SNode* pTarget = NULL;
WHERE_EACH(pTarget, pTargets) {
bool found = false;
@ -4509,7 +4547,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
}
}
}
if (TSDB_CODE_SUCCESS != code) break;;
if (TSDB_CODE_SUCCESS != code) break;
;
FOREACH(pColNode, pScan->pScanPseudoCols) {
if (nodesEqualNode(pParamNode, pColNode)) {
if (funcType != FUNCTION_TYPE_LAST) {
@ -4526,7 +4565,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
if (TSDB_CODE_SUCCESS != code) break;
if (pFunc->hasPk) {
code = nodesListMakeAppend(&cxt.pOtherCols, nodesListGetNode(pFunc->pParameterList, LIST_LENGTH(pFunc->pParameterList) - 1));
code = nodesListMakeAppend(&cxt.pOtherCols,
nodesListGetNode(pFunc->pParameterList, LIST_LENGTH(pFunc->pParameterList) - 1));
}
if (TSDB_CODE_SUCCESS != code) break;
} else {
@ -4823,7 +4863,8 @@ static int32_t splitCacheLastFuncOptCreateMergeLogicNode(SMergeLogicNode** pNew,
}
static int32_t splitCacheLastFuncOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SAggLogicNode* pAgg = (SAggLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, splitCacheLastFuncOptMayBeOptimized, NULL);
SAggLogicNode* pAgg =
(SAggLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, splitCacheLastFuncOptMayBeOptimized, NULL);
if (NULL == pAgg) {
return TSDB_CODE_SUCCESS;
@ -5077,7 +5118,8 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode, void* pCtx) {
}
static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SScanLogicNode* pScanNode = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanOptShouldBeOptimized, NULL);
SScanLogicNode* pScanNode =
(SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanOptShouldBeOptimized, NULL);
if (NULL == pScanNode) {
return TSDB_CODE_SUCCESS;
}
@ -5243,7 +5285,8 @@ static int32_t pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLim
return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo, pPushed);
case QUERY_NODE_LOGIC_PLAN_SORT: {
SSortLogicNode* pSort = (SSortLogicNode*)pNodeWithLimit;
if (sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys)) return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo, pPushed);
if (sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys))
return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo, pPushed);
}
default:
break;
@ -5399,7 +5442,8 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit
static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) {
if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) ||
0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags) {
0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags ||
0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_DISK_USAGE)) {
return false;
}
if (1 == pInfo->pScan->pVgroupList->numOfVgroups && MNODE_HANDLE == pInfo->pScan->pVgroupList->vgroups[0].vgId) {
@ -5607,7 +5651,8 @@ static bool hashJoinOptShouldBeOptimized(SLogicNode* pNode, void* pCtx) {
goto _return;
}
if ((JOIN_STYPE_NONE != pJoin->subType && JOIN_STYPE_OUTER != pJoin->subType) || JOIN_TYPE_FULL == pJoin->joinType || pNode->pChildren->length != 2 ) {
if ((JOIN_STYPE_NONE != pJoin->subType && JOIN_STYPE_OUTER != pJoin->subType) || JOIN_TYPE_FULL == pJoin->joinType ||
pNode->pChildren->length != 2) {
goto _return;
}
@ -5676,7 +5721,6 @@ static int32_t hashJoinOptSplitPrimFromLogicCond(SNode **pCondition, SNode **pPr
return code;
}
int32_t hashJoinOptSplitPrimCond(SNode** pCondition, SNode** pPrimaryKeyCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCondition)) {
if (LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCondition)->condType) {
@ -5705,7 +5749,6 @@ int32_t hashJoinOptSplitPrimCond(SNode **pCondition, SNode **pPrimaryKeyCond) {
return TSDB_CODE_SUCCESS;
}
static int32_t hashJoinOptRewriteJoin(SOptimizeContext* pCxt, SLogicNode* pNode, SLogicSubplan* pLogicSubplan) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode;
int32_t code = TSDB_CODE_SUCCESS;
@ -5896,7 +5939,6 @@ static int32_t hashJoinOptRewriteJoin(SOptimizeContext* pCxt, SLogicNode* pNode,
return TSDB_CODE_SUCCESS;
}
static int32_t hashJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
int32_t code = 0;
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, hashJoinOptShouldBeOptimized, &code);
@ -5911,7 +5953,8 @@ static int32_t hashJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
}
static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode, void* pCtx) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode) || OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_STB_JOIN)) {
if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode) ||
OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_STB_JOIN)) {
return false;
}
@ -5920,8 +5963,8 @@ static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode, void* pCtx) {
pJoin->joinAlgo = JOIN_ALGO_MERGE;
}
if (JOIN_STYPE_NONE != pJoin->subType || pJoin->isSingleTableJoin || NULL == pJoin->pTagEqCond || pNode->pChildren->length != 2
|| pJoin->isLowLevelJoin) {
if (JOIN_STYPE_NONE != pJoin->subType || pJoin->isSingleTableJoin || NULL == pJoin->pTagEqCond ||
pNode->pChildren->length != 2 || pJoin->isLowLevelJoin) {
return false;
}
@ -5975,7 +6018,8 @@ int32_t stbJoinOptRewriteToTagScan(SLogicNode* pJoin, SNode* pNode) {
SColumnNode* pTargetCol = (SColumnNode*)pTarget;
FOREACH(pTag, pTags) {
SColumnNode* pTagCol = (SColumnNode*)pTag;
if (0 == strcasecmp(pTargetCol->node.aliasName, pTagCol->colName) && 0 == strcasecmp(pTargetCol->tableAlias, pTagCol->tableAlias)) {
if (0 == strcasecmp(pTargetCol->node.aliasName, pTagCol->colName) &&
0 == strcasecmp(pTargetCol->tableAlias, pTagCol->tableAlias)) {
found = true;
break;
}
@ -6388,7 +6432,8 @@ static bool grpJoinOptShouldBeOptimized(SLogicNode* pNode, void* pCtx) {
return true;
}
static int32_t grpJoinOptCreatePartitionNode(SLogicNode* pParent, SLogicNode* pChild, bool leftChild, SLogicNode** pNew) {
static int32_t grpJoinOptCreatePartitionNode(SLogicNode* pParent, SLogicNode* pChild, bool leftChild,
SLogicNode** pNew) {
SPartitionLogicNode* pPartition = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PARTITION, (SNode**)&pPartition);
if (NULL == pPartition) {
@ -6476,7 +6521,9 @@ static int32_t grpJoinOptPartByTags(SLogicNode* pNode) {
static int32_t grpJoinOptRewriteGroupJoin(SOptimizeContext* pCxt, SLogicNode* pNode, SLogicSubplan* pLogicSubplan) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode;
int32_t code = (pJoin->allEqTags && !pJoin->hasSubQuery && !pJoin->batchScanHint) ? grpJoinOptPartByTags(pNode) : grpJoinOptInsertPartitionNode(pNode);
int32_t code = (pJoin->allEqTags && !pJoin->hasSubQuery && !pJoin->batchScanHint)
? grpJoinOptPartByTags(pNode)
: grpJoinOptInsertPartitionNode(pNode);
if (TSDB_CODE_SUCCESS == code) {
pJoin->grpJoin = true;
pCxt->optimized = true;
@ -6484,7 +6531,6 @@ static int32_t grpJoinOptRewriteGroupJoin(SOptimizeContext* pCxt, SLogicNode* pN
return code;
}
static int32_t groupJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, grpJoinOptShouldBeOptimized, NULL);
if (NULL == pNode) {
@ -6869,11 +6915,13 @@ static int32_t tsmaOptFilterTsmas(STSMAOptCtx* pTsmaOptCtx) {
}
if (pTsmaOptCtx->pScan->tableType == TSDB_CHILD_TABLE || pTsmaOptCtx->pScan->tableType == TSDB_NORMAL_TABLE) {
const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, i);
if (ptbInfo->uid == 0) continue; // tsma res table meta not found, skip this tsma, this is possible when there is no data in this ctb
if (ptbInfo->uid == 0)
continue; // tsma res table meta not found, skip this tsma, this is possible when there is no data in this ctb
}
STableTSMAInfo* pTsma = taosArrayGetP(pTsmaOptCtx->pTsmas, i);
if (!pTsma->fillHistoryFinished || tsMaxTsmaCalcDelay * 1000 < (pTsma->rspTs - pTsma->reqTs) + pTsma->delayDuration) {
if (!pTsma->fillHistoryFinished ||
tsMaxTsmaCalcDelay * 1000 < (pTsma->rspTs - pTsma->reqTs) + pTsma->delayDuration) {
continue;
}
// filter with interval
@ -7000,21 +7048,19 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow*
STSMAOptUsefulTsma usefulTsma = {.pTsma = pTsmaFound ? pTsmaFound->pTsma : NULL,
.scanRange = scanRange,
.pTsmaScanCols = pTsmaFound ? pTsmaFound->pTsmaScanCols : NULL};
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma))
return terrno;
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma)) return terrno;
}
// the main tsma
if (endOfSkeyFirstWin < startOfEkeyFirstWin || (endOfSkeyFirstWin == startOfEkeyFirstWin && (isSkeyAlignedWithTsma || isEkeyAlignedWithTsma))) {
scanRange.ekey =
TMIN(pScanRange->ekey, isEkeyAlignedWithTsma ? pScanRange->ekey : startOfEkeyFirstWin - 1);
if (endOfSkeyFirstWin < startOfEkeyFirstWin ||
(endOfSkeyFirstWin == startOfEkeyFirstWin && (isSkeyAlignedWithTsma || isEkeyAlignedWithTsma))) {
scanRange.ekey = TMIN(pScanRange->ekey, isEkeyAlignedWithTsma ? pScanRange->ekey : startOfEkeyFirstWin - 1);
if (!isSkeyAlignedWithTsma) {
scanRange.skey = endOfSkeyFirstWin;
}
STSMAOptUsefulTsma usefulTsma = {
.pTsma = pTsma, .scanRange = scanRange, .pTsmaScanCols = pUsefulTsma->pTsmaScanCols};
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma))
return terrno;
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma)) return terrno;
}
// add tail tsma if possible
@ -7027,8 +7073,7 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow*
STSMAOptUsefulTsma usefulTsma = {.pTsma = pTsmaFound ? pTsmaFound->pTsma : NULL,
.scanRange = scanRange,
.pTsmaScanCols = pTsmaFound ? pTsmaFound->pTsmaScanCols : NULL};
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma))
return terrno;
if (NULL == taosArrayPush(pTsmaOptCtx->pUsedTsmas, &usefulTsma)) return terrno;
}
return code;
}
@ -7177,7 +7222,8 @@ EDealRes tsmaOptNodeRewriter(SNode** ppNode, void* ctx) {
return DEAL_RES_CONTINUE;
}
static int32_t tsmaOptRewriteNode(SNode** pNode, STSMAOptCtx* pCtx, const STSMAOptUsefulTsma* pTsma, bool rewriteTbName, bool rewriteTag) {
static int32_t tsmaOptRewriteNode(SNode** pNode, STSMAOptCtx* pCtx, const STSMAOptUsefulTsma* pTsma, bool rewriteTbName,
bool rewriteTag) {
struct TsmaOptRewriteCtx ctx = {
.pTsmaOptCtx = pCtx, .pTsma = pTsma, .rewriteTag = rewriteTag, .rewriteTbname = rewriteTbName, .code = 0};
SNode* pOut = *pNode;

102
source/libs/planner/src/planPhysiCreater.c
View File

@ -30,7 +30,6 @@ typedef struct SSlotIndex {
SArray* pSlotIdsInfo; // duplicate name slot
} SSlotIndex;
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t* pLen, uint16_t extraBufLen) {
int32_t code = 0;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
@ -125,7 +124,6 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
return code;
}
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId,
bool output, bool reserve) {
SSlotDescNode* pSlot = NULL;
@ -178,7 +176,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char
return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex));
}
static int32_t putSlotToHash(const char* pName, int32_t len, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
static int32_t putSlotToHash(const char* pName, int32_t len, int16_t dataBlockId, int16_t slotId, SNode* pNode,
SHashObj* pHash) {
return putSlotToHashImpl(dataBlockId, slotId, pName, len, pHash);
}
@ -299,8 +298,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
if (TSDB_CODE_SUCCESS == code) {
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
if (NULL == pIndex) {
code =
nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve));
code = nodesListStrictAppend(pDataBlockDesc->pSlots,
createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve));
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
}
@ -819,7 +818,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
pScan->sysInfo = pCxt->pPlanCxt->sysInfo;
if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS)) {
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_DISK_USAGE)) {
if (pScanLogicNode->pVgroupList) {
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
}
@ -877,18 +877,21 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
static int32_t getJoinDataBlockDescNode(SNodeList* pChildren, int32_t idx, SDataBlockDescNode** ppDesc) {
if (2 == pChildren->length) {
*ppDesc = ((SPhysiNode*)nodesListGetNode(pChildren, idx))->pOutputDataBlockDesc;
} else if (1 == pChildren->length && nodeType(nodesListGetNode(pChildren, 0)) == QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE) {
} else if (1 == pChildren->length &&
nodeType(nodesListGetNode(pChildren, 0)) == QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE) {
SGroupCachePhysiNode* pGrpCache = (SGroupCachePhysiNode*)nodesListGetNode(pChildren, 0);
*ppDesc = ((SPhysiNode*)nodesListGetNode(pGrpCache->node.pChildren, idx))->pOutputDataBlockDesc;
} else {
planError("Invalid join children num:%d or child type:%d", pChildren->length, nodeType(nodesListGetNode(pChildren, 0)));
planError("Invalid join children num:%d or child type:%d", pChildren->length,
nodeType(nodesListGetNode(pChildren, 0)));
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
static int32_t setColEqList(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkId, SNodeList** ppLeft, SNodeList** ppRight) {
static int32_t setColEqList(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkId, SNodeList** ppLeft,
SNodeList** ppRight) {
int32_t code = 0;
if (QUERY_NODE_OPERATOR == nodeType(pEqCond) && ((SOperatorNode*)pEqCond)->opType == OP_TYPE_EQUAL) {
SOperatorNode* pOp = (SOperatorNode*)pEqCond;
@ -924,7 +927,8 @@ static int32_t setColEqList(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkI
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
}
} else if (QUERY_NODE_LOGIC_CONDITION == nodeType(pEqCond) && ((SLogicConditionNode*)pEqCond)->condType == LOGIC_COND_TYPE_AND) {
} else if (QUERY_NODE_LOGIC_CONDITION == nodeType(pEqCond) &&
((SLogicConditionNode*)pEqCond)->condType == LOGIC_COND_TYPE_AND) {
SLogicConditionNode* pLogic = (SLogicConditionNode*)pEqCond;
SNode* pNode = NULL;
FOREACH(pNode, pLogic->pParameterList) {
@ -941,7 +945,8 @@ static int32_t setColEqList(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkI
return code;
}
static int32_t setMergeJoinPrimColEqCond(SNode* pEqCond, int32_t subType, int16_t leftBlkId, int16_t rightBlkId, SSortMergeJoinPhysiNode* pJoin) {
static int32_t setMergeJoinPrimColEqCond(SNode* pEqCond, int32_t subType, int16_t leftBlkId, int16_t rightBlkId,
SSortMergeJoinPhysiNode* pJoin) {
int32_t code = 0;
if (QUERY_NODE_OPERATOR == nodeType(pEqCond)) {
SOperatorNode* pOp = (SOperatorNode*)pEqCond;
@ -1084,7 +1089,8 @@ static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pPrimKeyEqCond,
&pJoin->pPrimKeyCond);
if (TSDB_CODE_SUCCESS == code) {
code = setMergeJoinPrimColEqCond(pJoin->pPrimKeyCond, pJoin->subType, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin);
code = setMergeJoinPrimColEqCond(pJoin->pPrimKeyCond, pJoin->subType, pLeftDesc->dataBlockId,
pRightDesc->dataBlockId, pJoin);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoin->leftPrimExpr) {
code = addDataBlockSlot(pCxt, &pJoin->leftPrimExpr, pLeftDesc);
@ -1099,7 +1105,8 @@ static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->addPrimEqCond,
&pPrimKeyCond);
if (TSDB_CODE_SUCCESS == code) {
code = setMergeJoinPrimColEqCond(pPrimKeyCond, pJoin->subType, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin);
code = setMergeJoinPrimColEqCond(pPrimKeyCond, pJoin->subType, pLeftDesc->dataBlockId, pRightDesc->dataBlockId,
pJoin);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoin->leftPrimExpr) {
code = addDataBlockSlot(pCxt, &pJoin->leftPrimExpr, pLeftDesc);
@ -1116,8 +1123,8 @@ static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pFullOnCond) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1,
pJoinLogicNode->pFullOnCond, &pJoin->pFullOnCond);
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pJoinLogicNode->pFullOnCond,
&pJoin->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code && ((NULL != pJoinLogicNode->pColEqCond) || (NULL != pJoinLogicNode->pTagEqCond))) {
@ -1126,14 +1133,16 @@ static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
// TODO set from input blocks for group algo
/*
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqCond) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond, &pJoin->pColEqCond);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond,
&pJoin->pColEqCond);
}
*/
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqCond) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId,
pJoinLogicNode->pColEqCond, &pJoin->pColEqCond);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond,
&pJoin->pColEqCond);
if (TSDB_CODE_SUCCESS == code) {
code = setColEqList(pJoin->pColEqCond, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, &pJoin->pEqLeft, &pJoin->pEqRight);
code = setColEqList(pJoin->pColEqCond, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, &pJoin->pEqLeft,
&pJoin->pEqRight);
}
}
@ -1143,15 +1152,15 @@ static int32_t createMergeJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
// TODO set from input blocks for group algo
/*
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColOnCond) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColOnCond, &pJoin->pColOnCond);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColOnCond,
&pJoin->pColOnCond);
}
*/
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColOnCond) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1,
pJoinLogicNode->pColOnCond, &pJoin->pColOnCond);
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pJoinLogicNode->pColOnCond,
&pJoin->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
}
@ -1200,7 +1209,8 @@ static int32_t extractHashJoinOpCols(int16_t lBlkId, int16_t rBlkId, SNode* pEq,
code = nodesListStrictAppend(pJoin->pOnRight, pL);
}
} else {
planError("Invalid join equal cond, lbid:%d, rbid:%d, oplid:%d, oprid:%d", lBlkId, rBlkId, pLeft->dataBlockId, pRight->dataBlockId);
planError("Invalid join equal cond, lbid:%d, rbid:%d, oplid:%d, oprid:%d", lBlkId, rBlkId, pLeft->dataBlockId,
pRight->dataBlockId);
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
@ -1236,7 +1246,8 @@ static int32_t extractHashJoinOnCols(int16_t lBlkId, int16_t rBlkId, SNode* pEq,
return code;
}
static int32_t createHashJoinColList(int16_t lBlkId, int16_t rBlkId, SNode* pEq1, SNode* pEq2, SNode* pEq3, SHashJoinPhysiNode* pJoin) {
static int32_t createHashJoinColList(int16_t lBlkId, int16_t rBlkId, SNode* pEq1, SNode* pEq2, SNode* pEq3,
SHashJoinPhysiNode* pJoin) {
int32_t code = TSDB_CODE_SUCCESS;
pJoin->pOnLeft = NULL;
code = nodesMakeList(&pJoin->pOnLeft);
@ -1356,8 +1367,8 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys
return code;
}
static int32_t setHashJoinPrimColEqCond(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkId, SHashJoinPhysiNode* pJoin) {
static int32_t setHashJoinPrimColEqCond(SNode* pEqCond, int16_t leftBlkId, int16_t rightBlkId,
SHashJoinPhysiNode* pJoin) {
int32_t code = 0;
if (QUERY_NODE_OPERATOR == nodeType(pEqCond)) {
SOperatorNode* pOp = (SOperatorNode*)pEqCond;
@ -1463,7 +1474,6 @@ static int32_t setHashJoinPrimColEqCond(SNode* pEqCond, int16_t leftBlkId, int16
return code;
}
static int32_t createHashJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
SPhysiNode** pPhyNode) {
SHashJoinPhysiNode* pJoin =
@ -1509,10 +1519,12 @@ static int32_t createHashJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil
}
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond, &pJoin->pColEqCond);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqCond,
&pJoin->pColEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqCond, &pJoin->pTagEqCond);
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqCond,
&pJoin->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, -1, pJoinLogicNode->pLeftOnCond, &pJoin->pLeftOnCond);
@ -1525,16 +1537,19 @@ static int32_t createHashJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil
}
SNode* pOnCond = (NULL != pJoinLogicNode->pColOnCond) ? pJoinLogicNode->pColOnCond : pJoinLogicNode->pTagOnCond;
if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) {
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pOnCond, &pJoin->pFullOnCond);
code =
setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pOnCond, &pJoin->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets, &pJoin->pTargets);
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets,
&pJoin->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = createHashJoinColList(pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin->pPrimKeyCond, pJoin->pColEqCond, pJoin->pTagEqCond, pJoin);
code = createHashJoinColList(pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin->pPrimKeyCond,
pJoin->pColEqCond, pJoin->pTagEqCond, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = sortHashJoinTargets(pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoin);
@ -1567,8 +1582,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
return TSDB_CODE_FAILED;
}
static int32_t createGroupCachePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SGroupCacheLogicNode* pLogicNode,
SPhysiNode** pPhyNode) {
static int32_t createGroupCachePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SGroupCacheLogicNode* pLogicNode, SPhysiNode** pPhyNode) {
SGroupCachePhysiNode* pGrpCache =
(SGroupCachePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pLogicNode, QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE);
if (NULL == pGrpCache) {
@ -1592,8 +1607,8 @@ static int32_t createGroupCachePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh
return code;
}
static int32_t updateDynQueryCtrlStbJoinInfo(SPhysiPlanContext* pCxt, SNodeList* pChildren, SDynQueryCtrlLogicNode* pLogicNode,
SDynQueryCtrlPhysiNode* pDynCtrl) {
static int32_t updateDynQueryCtrlStbJoinInfo(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SDynQueryCtrlLogicNode* pLogicNode, SDynQueryCtrlPhysiNode* pDynCtrl) {
SDataBlockDescNode* pPrevDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
SNodeList* pVgList = NULL;
SNodeList* pUidList = NULL;
@ -1623,8 +1638,8 @@ static int32_t updateDynQueryCtrlStbJoinInfo(SPhysiPlanContext* pCxt, SNodeList*
return code;
}
static int32_t createDynQueryCtrlPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SDynQueryCtrlLogicNode* pLogicNode,
SPhysiNode** pPhyNode) {
static int32_t createDynQueryCtrlPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SDynQueryCtrlLogicNode* pLogicNode, SPhysiNode** pPhyNode) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlPhysiNode* pDynCtrl =
(SDynQueryCtrlPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pLogicNode, QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL);
@ -2657,7 +2672,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge, int32_t i
return nodesListMakeStrictAppend(&pMerge->node.pChildren, (SNode*)pExchange);
}
static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SMergeLogicNode* pMergeLogicNode, SPhysiNode** pPhyNode) {
static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SMergeLogicNode* pMergeLogicNode,
SPhysiNode** pPhyNode) {
int32_t code = TSDB_CODE_SUCCESS;
SMergePhysiNode* pMerge =
(SMergePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE);
@ -2711,7 +2727,8 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildre
SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc;
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pMergeLogicNode->node.pTargets, &pMerge->pTargets);
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pMergeLogicNode->node.pTargets,
&pMerge->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pMerge->pTargets, pMerge->node.pOutputDataBlockDesc);
}
@ -3147,8 +3164,7 @@ static int32_t setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) {
}
if (pCxt->hasSysScan || !pCxt->hasScan) {
SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
if (NULL == taosArrayPush(pExecNodeList, &node))
code = terrno;
if (NULL == taosArrayPush(pExecNodeList, &node)) code = terrno;
}
return code;
}

51
source/os/src/osDir.c
View File

@ -223,7 +223,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
#endif
if (taosDirExist(temp)) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
if (checkAccess &&
taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0;
}
@ -277,7 +278,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
}
if (code < 0 && errno == EEXIST) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
if (checkAccess &&
taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0;
}
}
@ -582,3 +584,48 @@ void taosGetCwd(char *buf, int32_t len) {
tstrncpy(buf, "not implemented on windows", len);
#endif
}
int taosGetDirSize(const char *path, int64_t *size) {
int32_t code;
TdDirPtr pDir = taosOpenDir(path);
if (pDir == NULL) {
return code = terrno;
}
int32_t nBytes = 0;
char fullPath[1024] = {0};
int64_t totalSize = 0;
TdDirEntryPtr de = NULL;
while ((de = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
continue;
}
nBytes = snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= sizeof(fullPath)) {
TAOS_UNUSED(taosCloseDir(&pDir));
return TSDB_CODE_OUT_OF_RANGE;
}
int64_t subSize = 0;
if (taosIsDir(fullPath)) {
code = taosGetDirSize(fullPath, &subSize);
if (code != 0) {
TAOS_UNUSED(taosCloseDir(&pDir));
return code;
}
} else {
code = taosStatFile(fullPath, &subSize, NULL, NULL);
if (code != 0) {
TAOS_UNUSED(taosCloseDir(&pDir));
return code;
}
}
totalSize += subSize;
fullPath[0] = 0;
}
*size = totalSize;
TAOS_UNUSED(taosCloseDir(&pDir));
return 0;
}

37
tests/develop-test/2-query/table_count_scan.py
View File

@ -63,21 +63,22 @@ class TDTestCase:
tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);')
tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;')
tdSql.checkRows(3)
tdSql.checkData(0, 0, 34)
tdSql.checkData(0, 1, 'information_schema')
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 3)
tdSql.checkData(1, 1, 'tbl_count')
tdSql.checkData(1, 2, 'stb1')
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
#tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;')
#tdSql.checkRows(3)
#tdSql.checkData(0, 0, 35)
#tdSql.checkData(0, 1, 'information_schema')
#tdSql.checkData(0, 2, None)
#tdSql.checkData(1, 0, 3)
#tdSql.checkData(1, 1, 'tbl_count')
#tdSql.checkData(1, 2, 'stb1')
#tdSql.checkData(2, 0, 5)
#tdSql.checkData(2, 1, 'performance_schema')
#tdSql.checkData(2, 2, None)
tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;')
tdSql.checkRows(3)
tdSql.checkData(0, 0, 34)
tdSql.checkData(0, 0, 35)
tdSql.checkData(0, 1, 'information_schema')
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 5)
@ -93,7 +94,7 @@ class TDTestCase:
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(0, 0, 3)
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(2, 0, 34)
tdSql.checkData(2, 0, 35)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -106,7 +107,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 42)
tdSql.checkData(0, 0, 43)
tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);')
@ -189,7 +190,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 34)
tdSql.checkData(3, 0, 35)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -204,7 +205,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 34)
tdSql.checkData(3, 0, 35)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -215,7 +216,7 @@ class TDTestCase:
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(1, 0, 5)
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(2, 0, 34)
tdSql.checkData(2, 0, 35)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -228,7 +229,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 43)
tdSql.checkData(0, 0, 44)
tdSql.execute('drop database tbl_count')

3
tests/parallel_test/cases.task
View File

@ -236,6 +236,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py
@ -374,6 +375,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_show_table_distributed.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_show_disk_usage.py
,,n,system-test,python3 ./test.py -f 0-others/compatibility.py
,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
@ -1538,6 +1540,7 @@
,,y,script,./test.sh -f tsim/compute/stddev.sim
,,y,script,./test.sh -f tsim/compute/sum.sim
,,y,script,./test.sh -f tsim/compute/top.sim
,,y,script,./test.sh -f tsim/compute/disk_usage.sim
,,y,script,./test.sh -f tsim/field/2.sim
,,y,script,./test.sh -f tsim/field/3.sim
,,y,script,./test.sh -f tsim/field/4.sim

78
tests/script/tsim/compute/disk_usage.sim Normal file
View File

@ -0,0 +1,78 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c debugflag -v 131
system sh/exec.sh -n dnode1 -s start
sql connect
$dbPrefix = m_di_db
$tbPrefix = m_di_tb
$mtPrefix = m_di_mt
$ntPrefix = m_di_nt
$tbNum = 1
$rowNum = 2000
print =============== step1
$i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
$nt = $ntPrefix . $i
sql drop database $db -x step1
step1:
sql create database $db
sql use $db
sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int)
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using $mt tags( $i )
$x = 0
while $x < $rowNum
$cc = $x * 60000
$ms = 1601481600000 + $cc
sql insert into $tb values ($ms , $x )
$x = $x + 1
endw
$i = $i + 1
endw
sql create table $nt (ts timestamp, tbcol int)
$x = 0
while $x < $rowNum
$cc = $x * 60000
$ms = 1601481600000 + $cc
sql insert into $nt values ($ms , $x )
$x = $x + 1
endw
sql flush database $db
sql select * from information_schema.ins_disk_usage
sql select sum(vgroup_id) from information_schema.ins_disk_usage
sql select sum(wal) from information_schema.ins_disk_usage
sql select sum(data1) from information_schema.ins_disk_usage
sql select sum(data2) from information_schema.ins_disk_usage
sql select sum(data3) from information_schema.ins_disk_usage
sql select sum(cache_rdb) from information_schema.ins_disk_usage
sql select sum(table_meta) from information_schema.ins_disk_usage
sql select sum(s3) from information_schema.ins_disk_usage
sql select sum(raw_data) from information_schema.ins_disk_usage
print $data00
print $rawDataSize
print $rows
sql use $db
sql show disk_info
#sql drop database $db
#sql select * from information_schema.ins_disk_usage
#if $rows != 0 then
# return -1
#endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/tsim/query/sys_tbname.sim
View File

@ -58,7 +58,7 @@ endi
sql select tbname from information_schema.ins_tables;
print $rows $data00
if $rows != 43 then
if $rows != 44 then
return -1
endi
if $data00 != @ins_tables@ then

6
tests/script/tsim/query/tableCount.sim
View File

@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou
if $rows != 3 then
return -1
endi
if $data01 != 40 then
if $data01 != 41 then
return -1
endi
if $data11 != 10 then
@ -72,7 +72,7 @@ endi
if $data11 != 5 then
return -1
endi
if $data21 != 34 then
if $data21 != 35 then
return -1
endi
if $data31 != 5 then
@ -97,7 +97,7 @@ endi
if $data42 != 3 then
return -1
endi
if $data52 != 34 then
if $data52 != 35 then
return -1
endi
if $data62 != 5 then

4
tests/system-test/0-others/information_schema.py
View File

@ -61,7 +61,7 @@ class TDTestCase:
self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\
'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\
'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views',
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full"]
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usage"]
self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps']
def insert_data(self,column_dict,tbname,row_num):
insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str)
@ -222,7 +222,7 @@ class TDTestCase:
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdLog.info(len(tdSql.queryResult))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(282, 283))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(292, 293))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(56, len(tdSql.queryResult))

217
tests/system-test/0-others/test_show_disk_usage.py Normal file
View File

@ -0,0 +1,217 @@
from itertools import product
import taos
import random
import time
from taos.tmq import *
from util.cases import *
from util.common import *
from util.log import *
from util.sql import *
from util.sqlset import *
import os
import subprocess
def get_disk_usage(path):
try:
result = subprocess.run(['du', '-sb', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
# The output is in the format "size\tpath"
size = int(result.stdout.split()[0])
return size
else:
print(f"Error: {result.stderr}")
return None
except Exception as e:
print(f"Exception occurred: {e}")
return None
def list_directories_with_keyword(base_path, keyword):
matching_dirs = []
for dirpath, dirnames, filenames in os.walk(base_path):
for dirname in dirnames:
if keyword in dirname:
full_path = os.path.join(dirpath, dirname)
matching_dirs.append(full_path)
return matching_dirs
def calculate_directories_size(base_path, keyword):
matching_dirs = list_directories_with_keyword(base_path, keyword)
total_size = 0
for directory in matching_dirs:
printf("directory: %s" % directory)
size = get_disk_usage(directory)
if size is not None:
total_size += size
return int(total_size/1024)
class TDTestCase:
"""This test case is used to veirfy show db disk usage"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
# init the tdsql
tdSql.init(conn.cursor())
self.dbname = "db_disk_usage"
self.stname = "st"
self.ctnum = 100
self.row_num = 1000
self.row_data_size = self.ctnum * self.row_num * (8 + 4 + 4) # timestamp + int + float
self.other_dbname = "db_disk_usage_other"
self.other_stname = "st_other"
self.data_path = tdCom.getTaosdPath()
tdLog.debug("data_path: %s" % self.data_path)
# create database
tdSql.execute(f'create database if not exists {self.dbname};')
tdSql.execute(f'create database if not exists {self.other_dbname};')
tdSql.execute(f'create table {self.other_dbname}.{self.other_stname} (ts timestamp, id int, temperature float) tags (name binary(20));')
tdSql.execute(f'create database if not exists {self.other_dbname};')
tdSql.execute(f'use {self.dbname};')
# create super table
tdSql.execute(f'create table {self.dbname}.{self.stname} (ts timestamp, id int, temperature float) tags (name binary(20));')
# create child table
for i in range(self.ctnum):
tdSql.execute(f'create table ct_{str(i+1)} using {self.stname} tags ("name{str(i+1)}");')
# insert data
sql = f"insert into ct_{str(i+1)} values "
for j in range(self.row_num):
sql += f"(now+{j+1}s, {j+1}, {random.uniform(15, 30)}) "
sql += ";"
tdSql.execute(sql)
tdSql.execute(f"flush database {self.dbname};")
tdLog.debug("init finished")
def getWALSize(self):
return calculate_directories_size(self.data_path, "wal")
def getTSDBSize(self):
tsdbDirSize = calculate_directories_size(self.data_path, "tsdb")
cacheRdbSize = calculate_directories_size(self.data_path, "cache.rdb")
return tsdbDirSize - cacheRdbSize
def getTableMetaSize(self):
return calculate_directories_size(self.data_path, "meta")
def getCacheRDBSize(self):
return calculate_directories_size(self.data_path, "cache.rdb")
def checkRes(self, queryRes):
disk_occupied = 0
compress_radio = 0
for item in queryRes:
if "Disk_occupied=" in item[0]:
disk_occupied= int(item[0].split("=")[1].split(" ")[0].replace("[", "").replace("k]", ""))
#tdLog.debug("disk_occupied: %s" % disk_occupied)
elif "Compress_radio=" in item[0]:
value = item[0].split("=")[1].split(" ")[0].replace("[", "").replace("]", "")
if value != 'NULL':
compress_radio = float(value)
#tdLog.debug("compress_occupied: %s" % compress_radio)
return disk_occupied, compress_radio
def insertData(self):
tdSql.execute(f'use {self.other_dbname};')
# create super table
tdSql.execute(f'create table {self.other_dbname}.{self.stname} (ts timestamp, id int, temperature float) tags (name binary(20));')
# create child table
for i in range(self.ctnum):
tdSql.execute(f'create table ct_{str(i+1)} using {self.stname} tags ("name{str(i+1)}");')
sql = f"insert into ct_{str(i+1)} values "
for j in range(self.row_num):
sql += f"(now+{j+1}s, {j+1}, {random.uniform(15, 30)}) "
sql += ";"
tdSql.execute(sql)
tdSql.execute(f"flush database {self.other_dbname};")
tdLog.debug("init finished")
def value_check(self,base_value,check_value, threshold):
if abs(base_value-check_value) < threshold:
tdLog.info(f"checkEqual success, base_value={base_value},check_value={check_value}")
else :
tdLog.exit(f"checkEqual error, base_value=={base_value},check_value={check_value}")
def run(self):
tdSql.execute(f"flush database {self.dbname};")
tdSql.query(f"show disk_info")
tdLog.debug(tdSql.queryResult)
disk_occupied,compress_radio = self.checkRes(tdSql.queryResult)
tdLog.debug("disk_occupied: %s, compress_radio: %s" % (disk_occupied, compress_radio))
#mem_rows_num, stt_rows_num = self.checkRes(tdSql.queryResult)
#tdLog.debug("mem_rows_num: %s, stt_rows_num: %s" % (mem_rows_num, stt_rows_num))
tdSql.query(f"select sum(data1+data2+data3) from information_schema.ins_disk_usage where db_name='{self.dbname}';")
tdSql.checkData(0,0,disk_occupied)
tdSql.query(f"select sum(data1+data2+data3)/sum(raw_data) from information_schema.ins_disk_usage where db_name='{self.dbname}';")
#tdSql.checkData(0,0,compress_radio/100)
tdSql.query(f"select sum(wal) from information_schema.ins_disk_usage where db_name='{self.dbname}';")
tdSql.query(f"select sum(table_meta) from information_schema.ins_disk_usage where db_name='{self.dbname}';")
tdSql.query(f"select sum(cache_rdb) from information_schema.ins_disk_usage where db_name='{self.dbname}';")
tdSql.execute(f"use {self.other_dbname};")
tdSql.query(f"select sum(data1+data2+data3) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,0)
tdSql.query(f"select sum(wal) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,0)
tdSql.query(f"select sum(cache_rdb) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,12)
tdSql.query(f"select sum(table_meta) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,152)
tdSql.query(f"select sum(s3) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,0)
tdSql.error(f"select sum(s3) from information_schema.ins_disk_usage where db='{self.other_dbname}';")
tdSql.error(f"select sum(s3) from information_schema.ins_disk_usage where db1='{self.other_dbname}';")
self.insertData()
tdSql.execute(f"flush database {self.other_dbname};")
tdSql.query(f"show {self.other_dbname}.disk_info;")
disk_occupied,compress_radio = self.checkRes(tdSql.queryResult)
tdLog.debug("database: %s, disk_occupied: %s, compress_radio: %s" % (self.other_dbname,disk_occupied, compress_radio))
tdSql.query(f"select sum(data1+data2+data3) from information_schema.ins_disk_usage where db_name='{self.other_dbname}';")
tdSql.checkData(0,0,disk_occupied)
tdSql.query(f"select sum(wal) from information_schema.ins_disk_usage where db_name='{self.other_dbname}' or db_name='{self.dbname}';")
tdSql.checkRows(1)
iwal = tdSql.queryResult[0][0]
tdSql.query(f"select sum(table_meta) from information_schema.ins_disk_usage where db_name='{self.other_dbname}' or db_name='{self.dbname}';")
itableMeta = tdSql.queryResult[0][0]
tdSql.query(f"select sum(data1+data2+data3) from information_schema.ins_disk_usage where db_name='{self.other_dbname}' or db_name='{self.dbname}';")
itsdbSize = int(tdSql.queryResult[0][0])
tdSql.query(f"select sum(cache_rdb) from information_schema.ins_disk_usage where db_name='{self.other_dbname}' or db_name='{self.dbname}';")
icache = tdSql.queryResult[0][0]
walSize = self.getWALSize()
tableMetaSize = self.getTableMetaSize()
tsdbSize = self.getTSDBSize()
cacheRdbSize = self.getCacheRDBSize()
tdLog.debug("calc: walSize: %s, tableMetaSize: %s, tsdbSize: %s, cacheRdbSize: %s" % (iwal, itableMeta, itsdbSize, icache))
tdLog.debug("du: walSize: %s, tableMetaSize: %s, tsdbSize: %s, cacheRdbSize: %s" % (walSize, tableMetaSize, tsdbSize, cacheRdbSize))
self.value_check(icache, cacheRdbSize, 64)
self.value_check(itableMeta,tableMetaSize, 64)
self.value_check(itsdbSize, tsdbSize, 64)
self.value_check(iwal, walSize, 128)
#if abs(icache - cacheRdbSize) > 12:
# tdLog.error("cache_rdb size is not equal")
#if abs(walSize - iwal) > 12:
# tdLog.error("wal size is not equal")
#if abs(tableMetaSize - itableMeta) > 12k'k'k
# tdLog.error("table_meta size is not equal")
#if abs(tsdbSize - itsdbSize) > 12:
# tdLog.error("tsdb size is not equal")
def stop(self):
# remove the user
tdSql.execute(f'drop database {self.dbname};')
# close the connection
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())