diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 45f608b24e..f2fec58d8f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -53,12 +53,12 @@ typedef struct { typedef struct SColumnDataAgg { int16_t colId; - int64_t sum; - int64_t max; - int64_t min; int16_t maxIndex; int16_t minIndex; int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; } SColumnDataAgg; typedef struct SDataBlockInfo { diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index da724206d9..991cb55e50 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -135,8 +135,8 @@ typedef struct { #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 #define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) -#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) +#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS)) +#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8)) int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); @@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche } #endif // ----------------- Data column structure +// SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { int8_t type; // column type uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6ec7d8c75b..fc7e994cbb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -930,6 +930,21 @@ typedef struct { char data[]; } SRetrieveMetaTableRsp; +typedef struct SExplainExecInfo { + uint64_t startupCost; + uint64_t totalCost; + uint64_t numOfRows; + void *verboseInfo; +} SExplainExecInfo; + +typedef struct { + int32_t numOfPlans; + SExplainExecInfo *subplanInfo; +} SExplainRsp; + +int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); +int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); + typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port int32_t port; @@ -1067,6 +1082,7 @@ typedef struct SSubQueryMsg { uint64_t taskId; int64_t refId; int8_t taskType; + int8_t explain; uint32_t sqlLen; // the query sql, uint32_t phyLen; char msg[]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 36a489eb59..1d4667cda0 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -188,6 +188,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "vnode-query-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_EXPLAIN, "vnode-explain", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) diff --git a/include/common/trow.h b/include/common/trow.h index 01f4076382..7cde8c50c5 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -147,20 +147,20 @@ typedef struct { typedef struct { // basic info - int8_t rowType; - int16_t sver; - STSRow *pBuf; + int8_t rowType; + schema_ver_t sver; + STSRow *pBuf; // extended info - int32_t flen; - int16_t nBoundCols; - int16_t nCols; - int16_t nBitmaps; - int16_t nBoundBitmaps; - int32_t offset; - void *pBitmap; - void *pOffset; - int32_t extendedRowSize; + int32_t flen; + col_id_t nBoundCols; + col_id_t nCols; + col_id_t nBitmaps; + col_id_t nBoundBitmaps; + int32_t offset; + void *pBitmap; + void *pOffset; + int32_t extendedRowSize; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); } else { pBuilder->nBoundBitmaps = 0; } diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 1cb398fb85..37d688a0ef 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -146,6 +146,7 @@ typedef struct { #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) +#define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 63f305ab2d..83dccd0092 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -59,6 +59,7 @@ int32_t taosVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool #endif int32_t taosVariantTypeSetType(SVariant *pVariant, char type); +char * taosVariantGet(SVariant *pVar, int32_t type); #ifdef __cplusplus } diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 7e58d39692..fa229d7265 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -17,8 +17,14 @@ #include "tmsg.h" #include "plannodes.h" +typedef struct SExplainCtx SExplainCtx; + int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp); int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); +int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int32_t startTs); +int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp); +int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp); +void qExplainFreeCtx(SExplainCtx *pCtx); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d3cd828cf5..4d289147d0 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -174,6 +174,8 @@ void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); + #ifdef __cplusplus } #endif diff --git a/include/libs/function/function.h b/include/libs/function/function.h index fe9edc323d..d4307362b6 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -36,7 +36,7 @@ typedef struct SFuncExecEnv { typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); -typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); +typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -154,7 +154,9 @@ typedef struct SResultDataInfo { int32_t interBufSize; } SResultDataInfo; -#define GET_RES_INFO(ctx) ((ctx)->resultInfo) +#define GET_RES_INFO(ctx) ((ctx)->resultInfo) +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) +#define DATA_SET_FLAG ',' // to denote the output area has data, not null value typedef struct SInputColumnInfoData { int32_t totalRows; // total rows in current columnar data @@ -192,7 +194,8 @@ typedef struct SqlFunctionCtx { int32_t numOfParams; SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list - void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int32_t offset; SVariant tag; struct SResultRowEntryInfo *resultInfo; SSubsidiaryResInfo subsidiaryRes; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 0969fb203a..57a3862a9a 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -132,6 +132,7 @@ bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); +bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9668082696..28e5483552 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -163,6 +163,8 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, + QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, + QUERY_NODE_PHYSICAL_PLAN_PARTITION, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_SUBPLAN, @@ -213,16 +215,16 @@ typedef enum EDealRes { } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNodeptr a, const SNodeptr b); @@ -237,6 +239,8 @@ int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int3 int32_t nodesStringToList(const char* pStr, SNodeList** pList); int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); +char *nodesGetNameFromColumnNode(SNode *pNode); +int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); #ifdef __cplusplus } diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 37163f60dd..65f7e1ceb8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -104,6 +104,7 @@ typedef struct SWindowLogicNode { SFillNode* pFill; int64_t sessionGap; SNode* pTspk; + SNode* pStateExpr; } SWindowLogicNode; typedef struct SSortLogicNode { @@ -194,11 +195,20 @@ typedef struct SSystemTableScanPhysiNode { int32_t accountId; } SSystemTableScanPhysiNode; +typedef enum EScanRequired { + SCAN_REQUIRED_DATA_NO_NEEDED = 1, + SCAN_REQUIRED_DATA_STATIS_NEEDED, + SCAN_REQUIRED_DATA_ALL_NEEDED, + SCAN_REQUIRED_DATA_DISCARD, +} EScanRequired; + typedef struct STableScanPhysiNode { SScanPhysiNode scan; uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; double ratio; + EScanRequired scanRequired; + SNodeList* pScanReferFuncs; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; @@ -257,17 +267,33 @@ typedef struct SIntervalPhysiNode { SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SMultiTableIntervalPhysiNode { + SIntervalPhysiNode interval; + SNodeList* pPartitionKeys; +} SMultiTableIntervalPhysiNode; + typedef struct SSessionWinodwPhysiNode { SWinodwPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; +typedef struct SStateWinodwPhysiNode { + SWinodwPhysiNode window; + SNode* pStateKey; +} SStateWinodwPhysiNode; + typedef struct SSortPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode } SSortPhysiNode; +typedef struct SPartitionPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of partition_by_clause + SNodeList* pPartitionKeys; +} SPartitionPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; @@ -308,6 +334,7 @@ typedef enum EExplainMode { typedef struct SExplainInfo { EExplainMode mode; bool verbose; + double ratio; } SExplainInfo; typedef struct SQueryPlan { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index ca4646a5ca..f811b42f2f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -22,6 +22,7 @@ extern "C" { #include "nodes.h" #include "tmsg.h" +#include "tvariant.h" #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) #define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) @@ -79,6 +80,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool genByCalc; union { bool b; int64_t i; @@ -187,7 +189,7 @@ typedef struct SLimitNode { typedef struct SStateWindowNode { ENodeType type; // QUERY_NODE_STATE_WINDOW - SNode* pCol; + SNode* pExpr; } SStateWindowNode; typedef struct SSessionWindowNode { @@ -315,6 +317,8 @@ bool nodesIsTimelineQuery(const SNode* pQuery); void* nodesGetValueFromNode(SValueNode *pNode); char* nodesGetStrValueFromNode(SValueNode *pNode); +char *getFillModeString(EFillMode mode); +void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); #ifdef __cplusplus } diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index a93180800e..c81cb49b64 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -40,7 +40,7 @@ extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t op extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); -extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 16a6ae32cf..5ab4ead89c 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -71,7 +71,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, int64_t* pJob, const char* sql, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 157e0cb721..3e2f596784 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -58,6 +58,9 @@ typedef struct { void *pNode; } SNodeMsg; +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); +typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -74,10 +77,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + RpcCfp cfp; // call back to retrieve the client auth info, for server app only - int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + RpcAfp afp;; void *parent; } SRpcInit; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4e2cb7944a..e7f440e26f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -483,6 +483,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME TAOS_DEF_ERROR_CODE(0, 0x2617) #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618) +//planner +#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index be88afcf66..f1af9eca1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -480,9 +480,9 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MND_VGID -1 -#define QND_VGID 1 -#define VND_VGID 0 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE 1 +#define DEFAULT_HANDLE 0 #define MAX_NUM_STR_SIZE 40 diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index e49e89c309..44e43f81cf 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -76,6 +76,7 @@ void taosWLockLatch(SRWLatch *pLatch); void taosWUnLockLatch(SRWLatch *pLatch); void taosRLockLatch(SRWLatch *pLatch); void taosRUnLockLatch(SRWLatch *pLatch); +int32_t taosWTryLockLatch(SRWLatch *pLatch); // copy on read #define taosCorBeginRead(x) \ diff --git a/include/util/tprocess.h b/include/util/tprocess.h index bc0c8fe5a4..80c855b78c 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -24,11 +24,10 @@ extern "C" { typedef enum { PROC_QUEUE, PROC_REQ, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType; -typedef struct SProcQueue SProcQueue; -typedef struct SProcObj SProcObj; +typedef struct SProcObj SProcObj; typedef void *(*ProcMallocFp)(int32_t contLen); typedef void *(*ProcFreeFp)(void *pCont); -typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen, +typedef void *(*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen, ProcFuncType ftype); typedef struct { @@ -43,7 +42,7 @@ typedef struct { ProcMallocFp parentMallocBodyFp; ProcFreeFp parentFreeBodyFp; SShm shm; - void *pParent; + void *parent; const char *name; bool isChild; } SProcCfg; @@ -51,10 +50,13 @@ typedef struct { SProcObj *taosProcInit(const SProcCfg *pCfg); void taosProcCleanup(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc); -int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - ProcFuncType ftype); -void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - ProcFuncType ftype); + +int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, + void *handle, ProcFuncType ftype); +void taosProcRemoveHandle(SProcObj *pProc, void *handle); +void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)); +void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, + ProcFuncType ftype); #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 0749ce9e7d..ae1b34a3bd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -163,6 +163,7 @@ typedef struct SReqResultInfo { uint64_t totalRows; uint32_t current; bool completed; + int32_t precision; int32_t payloadLen; } SReqResultInfo; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 80559c025f..c3a17b6073 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -254,7 +254,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, &res); + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -825,6 +825,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); + pResultInfo->precision = pRsp->precision; // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0ca6169883..b2c32a4366 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - // TODO - return NULL; + SRequestObj* pRequest = (SRequestObj*) res; + if (pRequest == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + if (taos_is_update_query(res)) { + return NULL; + } + + return &pRequest->body.resInfo.row; } // todo intergrate with tDataTypes @@ -313,7 +322,14 @@ int taos_affected_rows(TAOS_RES *res) { return pResInfo->numOfRows; } -int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } +int taos_result_precision(TAOS_RES *res) { + SRequestObj* pRequest = (SRequestObj*) res; + if (pRequest == NULL) { + return TSDB_TIME_PRECISION_MILLI; + } + + return pRequest->body.resInfo.precision; +} int taos_select_db(TAOS *taos, const char *db) { STscObj *pObj = (STscObj *)taos; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 426a62433b..602ecdf6ab 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -79,6 +79,7 @@ struct tmq_t { tmq_commit_cb* commit_cb; int32_t nextTopicIdx; int8_t epStatus; + int32_t epSkipCnt; int32_t waitingRequest; int32_t readyRequest; SArray* clientTopics; // SArray @@ -107,6 +108,7 @@ typedef struct { // connection info int32_t vgId; int32_t vgStatus; + int64_t skipCnt; SEpSet epSet; } SMqClientVg; @@ -138,6 +140,7 @@ typedef struct { tmq_t* tmq; SMqClientVg* pVg; int32_t epoch; + int32_t vgId; tsem_t rspSem; tmq_message_t** msg; int32_t sync; @@ -313,6 +316,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->waitingRequest = 0; pTmq->readyRequest = 0; pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -348,6 +352,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->epoch = 0; pTmq->waitingRequest = 0; pTmq->readyRequest = 0; + pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -834,7 +840,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - tscWarn("msg discard, code:%x", code); + tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); goto CREATE_MSG_FAIL; } @@ -842,13 +848,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ - tsem_post(&tmq->rspSem); - tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + /*tsem_post(&tmq->rspSem);*/ + tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); return 0; } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } else { atomic_sub_fetch_32(&tmq->waitingRequest, 1); } @@ -892,29 +898,29 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset, + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset, pRsp->msg.rspOffset); pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return code; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ - tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch); bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { return false; @@ -932,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { taosHashClear(pHash); topic.topicName = strdup(pTopicEp->topic); + tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); if (vgNumCur == 0) break; for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -956,18 +964,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } - tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, + .skipCnt = 0, }; taosArrayPush(topic.vgs, &clientVg); set = true; @@ -984,9 +993,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; - tscDebug("consumer %ld recv ep", tmq->consumerId); if (code != 0) { - tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync); + tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); goto END; } @@ -995,6 +1003,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); + tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (head->epoch <= epoch) { goto END; } @@ -1019,7 +1028,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); taosWriteQitem(tmq->mqueue, pRsp); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ } END: @@ -1033,9 +1042,11 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { - tscDebug("consumer %ld skip ask ep", tmq->consumerId); - return 0; + int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); + tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); + if (epSkipCnt < 5000) return 0; } + atomic_store_32(&tmq->epSkipCnt, 0); int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { @@ -1221,24 +1232,34 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { - tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId); + int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); + tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt); continue; +#if 0 + if (skipCnt < 30000) { + continue; + } else { + tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId); + } +#endif } + atomic_store_64(&pVg->skipCnt, 0); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } pParam->tmq = tmq; pParam->pVg = pVg; + pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; pParam->sync = 0; @@ -1247,7 +1268,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { taosMemoryFree(pReq); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } @@ -1265,7 +1286,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); - tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset); + tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1379,7 +1400,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmqAskEp(tmq, false); tmqPollImpl(tmq, blocking_time); - tsem_wait(&tmq->rspSem); + /*tsem_wait(&tmq->rspSem);*/ rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 629dd90da3..772f9049e5 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -400,6 +400,7 @@ TEST(testCase, show_vgroup_Test) { taos_free_result(pRes); taos_close(pConn); } +#endif TEST(testCase, create_multiple_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -458,7 +459,7 @@ TEST(testCase, create_multiple_tables) { taos_free_result(pRes); - for (int32_t i = 0; i < 20; ++i) { + for (int32_t i = 0; i < 25000; ++i) { char sql[512] = {0}; snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, @@ -652,7 +653,6 @@ TEST(testCase, projection_query_stables) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1d7a4aeb8f..c1225012ac 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - spaceNeeded += TYPE_BYTES[pCol->type]; + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove + // the additional space #endif if (pCol->spaceSize < spaceNeeded) { @@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { } } #ifdef TD_SUPPORT_BITMAP + if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints); pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes); @@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) bool isNEleNull(SDataCol *pCol, int nEle) { if (isAllRowsNull(pCol)) return true; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; @@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { dataColSetNullAt(pCol, i); } } else { @@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { // char *tptr = (char *)(pCol->pData); VarDataOffsetT offset = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { pCol->dataOff[i] = offset; offset += varDataTLen(tptr); tptr = POINTER_SHIFT(tptr, varDataTLen(tptr)); @@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { tdFreeDataCols(pCols); return NULL; } +#if 0 // no need as calloc used int i; for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; @@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } +#endif } return pCols; @@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for (i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; ++i) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; + pCols->cols[i].pBitmap = NULL; pCols->cols[i].spaceSize = 0; } } +#if 0 + tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...) +#endif - tdResetDataCols(pCols); + pCols->numOfRows = 0; pCols->numOfCols = schemaNCols(pSchema); - for (i = 0; i < schemaNCols(pSchema); i++) { + for (i = 0; i < schemaNCols(pSchema); ++i) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } @@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { if (pCols->cols) { int maxCols = pCols->maxCols; - for (i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; ++i) { SDataCol *pCol = &pCols->cols[i]; taosMemoryFreeClear(pCol->pData); } @@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; - for (int i = 0; i < pCols->maxCols; i++) { + for (int i = 0; i < pCols->maxCols; ++i) { dataColReset(pCols->cols + i); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3a144a8a8e..b40378cbb3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -367,7 +367,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); @@ -376,7 +376,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols); pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); @@ -408,7 +408,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma); buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); @@ -2770,6 +2770,48 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) return 0; } +int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfPlans) < 0) return -1; + for (int32_t i = 0; i < pRsp->numOfPlans; ++i) { + SExplainExecInfo *info = &pRsp->subplanInfo[i]; + if (tEncodeU64(&encoder, info->startupCost) < 0) return -1; + if (tEncodeU64(&encoder, info->totalCost) < 0) return -1; + if (tEncodeU64(&encoder, info->numOfRows) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfPlans) < 0) return -1; + if (pRsp->numOfPlans > 0) { + pRsp->subplanInfo = taosMemoryMalloc(pRsp->numOfPlans * sizeof(SExplainExecInfo)); + if (pRsp->subplanInfo == NULL) return -1; + } + for (int32_t i = 0; i < pRsp->numOfPlans; ++i) { + if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].startupCost) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].numOfRows) < 0) return -1; + } + + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9ee2ec1300..0d5a874c1c 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); } } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 3995db89b6..77feb04f6c 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -1014,4 +1014,28 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} + +char * taosVariantGet(SVariant *pVar, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_TIMESTAMP: + return (char *)&pVar->i; + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_FLOAT: + return (char *)&pVar->d; + case TSDB_DATA_TYPE_BINARY: + return (char *)pVar->pz; + case TSDB_DATA_TYPE_NCHAR: + return (char *)pVar->ucs4; + default: + return NULL; + } + + return NULL; +} + diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 81c434eb27..0c9a3733a2 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,10 +1,10 @@ -aux_source_directory(dm/src DNODE_SRC) -aux_source_directory(qm/src DNODE_SRC) -aux_source_directory(bm/src DNODE_SRC) -aux_source_directory(sm/src DNODE_SRC) -aux_source_directory(vm/src DNODE_SRC) -aux_source_directory(mm/src DNODE_SRC) -aux_source_directory(main/src DNODE_SRC) +aux_source_directory(dm DNODE_SRC) +aux_source_directory(qm DNODE_SRC) +aux_source_directory(bm DNODE_SRC) +aux_source_directory(sm DNODE_SRC) +aux_source_directory(vm DNODE_SRC) +aux_source_directory(mm DNODE_SRC) +aux_source_directory(main DNODE_SRC) add_library(dnode STATIC ${DNODE_SRC}) target_link_libraries( dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor @@ -12,20 +12,14 @@ target_link_libraries( target_include_directories( dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -aux_source_directory(main/exe EXEC_SRC) +aux_source_directory(exe EXEC_SRC) add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/bm/src/bmMsg.c b/source/dnode/mgmt/bm/bmHandle.c similarity index 97% rename from source/dnode/mgmt/bm/src/bmMsg.c rename to source/dnode/mgmt/bm/bmHandle.c index c01d260c3f..2ae9b3817f 100644 --- a/source/dnode/mgmt/bm/src/bmMsg.c +++ b/source/dnode/mgmt/bm/bmHandle.c @@ -54,4 +54,4 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } } -void bmInitMsgHandles(SMgmtWrapper *pWrapper) {} +void bmInitMsgHandle(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/bmInt.c similarity index 97% rename from source/dnode/mgmt/bm/src/bmInt.c rename to source/dnode/mgmt/bm/bmInt.c index 4ca7afbb6d..990c7873a9 100644 --- a/source/dnode/mgmt/bm/src/bmInt.c +++ b/source/dnode/mgmt/bm/bmInt.c @@ -109,7 +109,7 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { return code; } -void bmGetMgmtFp(SMgmtWrapper *pWrapper) { +void bmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = bmOpen; mgmtFp.closeFp = bmClose; @@ -117,7 +117,7 @@ void bmGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.dropMsgFp = bmProcessDropReq; mgmtFp.requiredFp = bmRequire; - bmInitMsgHandles(pWrapper); + bmInitMsgHandle(pWrapper); pWrapper->name = "bnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/bmWorker.c similarity index 98% rename from source/dnode/mgmt/bm/src/bmWorker.c rename to source/dnode/mgmt/bm/bmWorker.c index 932c008e34..a5a97f6af0 100644 --- a/source/dnode/mgmt/bm/src/bmWorker.c +++ b/source/dnode/mgmt/bm/bmWorker.c @@ -18,7 +18,7 @@ static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rpcRsp); + tmsgSendRsp(&rpcRsp); dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); diff --git a/source/dnode/mgmt/bm/inc/bm.h b/source/dnode/mgmt/bm/inc/bm.h deleted file mode 100644 index 79cf76d113..0000000000 --- a/source/dnode/mgmt/bm/inc/bm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_H_ -#define _TD_DND_BNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void bmGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dm/src/dmFile.c b/source/dnode/mgmt/dm/dmFile.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmFile.c rename to source/dnode/mgmt/dm/dmFile.c diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/dmHandle.c similarity index 62% rename from source/dnode/mgmt/dm/src/dmMsg.c rename to source/dnode/mgmt/dm/dmHandle.c index 2167e16248..24aa50afb6 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/dmHandle.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" -#include "vm.h" void dmSendStatusReq(SDnodeMgmt *pMgmt) { SDnode *pDnode = pMgmt->pDnode; @@ -57,7 +56,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { pMgmt->statusSent = 1; dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); - dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + tmsgSendReq(&pMgmt->msgCb, &epSet, &rpcMsg); } static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { @@ -117,21 +118,99 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { return TSDB_CODE_OPS_NOT_SUPPORT; } -void dmInitMsgHandles(SMgmtWrapper *pWrapper) { + +static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dmProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dmProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dmProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dmProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dmProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dmProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dmProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dmProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} + +void dmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessStatusMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/dmInt.c similarity index 89% rename from source/dnode/mgmt/dm/src/dmInt.c rename to source/dnode/mgmt/dm/dmInt.c index 3c5f394d5e..c710af9006 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/dmInt.c @@ -78,7 +78,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) { return dmStartThread(pWrapper->pMgmt); } -int32_t dmInit(SMgmtWrapper *pWrapper) { +static int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); dInfo("dnode-mgmt start to init"); @@ -112,22 +112,19 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return -1; } - if (dndInitServer(pDnode) != 0) { - dError("failed to init trans server since %s", terrstr()); - return -1; - } - - if (dndInitClient(pDnode) != 0) { - dError("failed to init trans client since %s", terrstr()); + if (dndInitTrans(pDnode) != 0) { + dError("failed to init transport since %s", terrstr()); return -1; } pWrapper->pMgmt = pMgmt; + pMgmt->msgCb = dndCreateMsgcb(pWrapper); + dInfo("dnode-mgmt is initialized"); return 0; } -void dmCleanup(SMgmtWrapper *pWrapper) { +static void dmCleanup(SMgmtWrapper *pWrapper) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; @@ -151,25 +148,24 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); + dndCleanupTrans(pDnode); dInfo("dnode-mgmt is cleaned up"); } -int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { +static int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { *required = true; return 0; } -void dmGetMgmtFp(SMgmtWrapper *pWrapper) { +void dmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = dmInit; mgmtFp.closeFp = dmCleanup; mgmtFp.startFp = dmStart; mgmtFp.requiredFp = dmRequire; - dmInitMsgHandles(pWrapper); + dmInitMsgHandle(pWrapper); pWrapper->name = "dnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/main/src/dndMonitor.c b/source/dnode/mgmt/dm/dmMonitor.c similarity index 88% rename from source/dnode/mgmt/main/src/dndMonitor.c rename to source/dnode/mgmt/dm/dmMonitor.c index c01f840794..22d9086971 100644 --- a/source/dnode/mgmt/main/src/dndMonitor.c +++ b/source/dnode/mgmt/dm/dmMonitor.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" -static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { +static int32_t dmGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); pInfo->logdir.size = tsLogSpace.size; tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); @@ -30,14 +30,14 @@ static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; } -static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { +static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->protocol = 1; pInfo->dnode_id = pDnode->dnodeId; pInfo->cluster_id = pDnode->clusterId; tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); } -static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { +static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); taosGetCpuCores(&pInfo->cpu_cores); @@ -63,7 +63,7 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { } } -void dndSendMonitorReport(SDnode *pDnode) { +void dmSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); @@ -71,7 +71,7 @@ void dndSendMonitorReport(SDnode *pDnode) { if (pMonitor == NULL) return; SMonBasicInfo basicInfo = {0}; - dndGetMonitorBasicInfo(pDnode, &basicInfo); + dmGetMonitorBasicInfo(pDnode, &basicInfo); monSetBasicInfo(pMonitor, &basicInfo); SMonClusterInfo clusterInfo = {0}; @@ -89,11 +89,11 @@ void dndSendMonitorReport(SDnode *pDnode) { } SMonDnodeInfo dnodeInfo = {0}; - dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + dmGetMonitorDnodeInfo(pDnode, &dnodeInfo); monSetDnodeInfo(pMonitor, &dnodeInfo); SMonDiskInfo diskInfo = {0}; - if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { + if (dmGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { monSetDiskInfo(pMonitor, &diskInfo); } diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/dmWorker.c similarity index 74% rename from source/dnode/mgmt/dm/src/dmWorker.c rename to source/dnode/mgmt/dm/dmWorker.c index ca86f42608..5516b86dae 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/dmWorker.c @@ -14,16 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "bm.h" #include "dmInt.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; int64_t lastStatusTime = taosGetTimestampMs(); int64_t lastMonitorTime = lastStatusTime; @@ -37,8 +32,7 @@ static void *dmThreadRoutine(void *param) { } int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; + float statusInterval = (curTime - lastStatusTime) / 1000.0f; if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { dmSendStatusReq(pMgmt); lastStatusTime = curTime; @@ -46,32 +40,33 @@ static void *dmThreadRoutine(void *param) { float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; if (monitorInterval >= tsMonitorInterval) { - dndSendMonitorReport(pDnode); + dmSendMonitorReport(pDnode); lastMonitorTime = curTime; } } return TSDB_CODE_SUCCESS; } +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode queue", pMsg); switch (pRpc->msgType) { - case TDMT_DND_CREATE_MNODE: - case TDMT_DND_CREATE_QNODE: - case TDMT_DND_CREATE_SNODE: - case TDMT_DND_CREATE_BNODE: - case TDMT_DND_DROP_MNODE: - case TDMT_DND_DROP_QNODE: - case TDMT_DND_DROP_SNODE: - case TDMT_DND_DROP_BNODE: - code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); - break; case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -85,8 +80,8 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { code = dmProcessGrantRsp(pMgmt, pMsg); break; default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + code = dmProcessCDnodeReq(pMgmt->pDnode, pMsg); + break; } if (pRpc->msgType & 1u) { @@ -101,16 +96,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { } int32_t dmStartWorker(SDnodeMgmt *pMgmt) { - SSingleWorkerCfg mgmtCfg = { - .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { + SSingleWorkerCfg mcfg = {.min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mcfg) != 0) { dError("failed to start dnode mgmt worker since %s", terrstr()); return -1; } - SSingleWorkerCfg statusCfg = { - .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { + SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) { dError("failed to start dnode status worker since %s", terrstr()); return -1; } @@ -119,17 +112,6 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return 0; } -int32_t dmStartThread(SDnodeMgmt *pMgmt) { - pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - void dmStopWorker(SDnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->statusWorker); @@ -144,9 +126,15 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } + + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + +int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->statusWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); diff --git a/source/dnode/mgmt/dm/inc/dm.h b/source/dnode/mgmt/dm/inc/dm.h deleted file mode 100644 index 3984e6dbd4..0000000000 --- a/source/dnode/mgmt/dm/inc/dm.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDnodeMgmt SDnodeMgmt; - -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void dmInitMsgHandles(SMgmtWrapper *pWrapper); - -void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c similarity index 99% rename from source/dnode/mgmt/main/exe/dndMain.c rename to source/dnode/mgmt/exe/dndMain.c index c6a109d62a..48510e9622 100644 --- a/source/dnode/mgmt/main/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "dnd.h" +#include "dndInt.h" #include "tconfig.h" static struct { diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h similarity index 94% rename from source/dnode/mgmt/bm/inc/bmInt.h rename to source/dnode/mgmt/inc/bmInt.h index f19ba4e034..919b1d2c7c 100644 --- a/source/dnode/mgmt/bm/inc/bmInt.h +++ b/source/dnode/mgmt/inc/bmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_BNODE_INT_H_ #define _TD_DND_BNODE_INT_H_ -#include "bm.h" +#include "dndInt.h" #include "bnode.h" #ifdef __cplusplus @@ -35,8 +35,8 @@ typedef struct SBnodeMgmt { int32_t bmOpen(SMgmtWrapper *pWrapper); int32_t bmDrop(SMgmtWrapper *pWrapper); -// bmMsg.c -void bmInitMsgHandles(SMgmtWrapper *pWrapper); +// bmHandle.c +void bmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/inc/dmInt.h similarity index 86% rename from source/dnode/mgmt/dm/inc/dmInt.h rename to source/dnode/mgmt/inc/dmInt.h index 3036d1f5ad..5dcd1be47c 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/inc/dmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_DNODE_INT_H_ #define _TD_DND_DNODE_INT_H_ -#include "dm.h" +#include "dndInt.h" #ifdef __cplusplus extern "C" { @@ -33,6 +33,7 @@ typedef struct SDnodeMgmt { SRWLatch latch; SSingleWorker mgmtWorker; SSingleWorker statusWorker; + SMsgCb msgCb; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; @@ -43,18 +44,24 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt); int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); -// dmMsg.c +// dmHandle.c +void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg); + +// dmMonitor.c +void dmSendMonitorReport(SDnode *pDnode); // dmWorker.c +int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); -int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/inc/dndInt.h similarity index 70% rename from source/dnode/mgmt/main/inc/dnd.h rename to source/dnode/mgmt/inc/dndInt.h index f3a409f257..20e61c43dd 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/inc/dndInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_H_ -#define _TD_DND_H_ +#ifndef _TD_DND_INT_H_ +#define _TD_DND_INT_H_ #include "os.h" @@ -27,13 +27,13 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "tmsgcb.h" #include "tprocess.h" #include "tqueue.h" #include "trpc.h" #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tmsgcb.h" #include "dnode.h" #include "monitor.h" @@ -73,7 +73,7 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef struct SMsgHandle { - SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pQndWrapper; SMgmtWrapper *pMndWrapper; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -92,6 +92,7 @@ typedef struct SMgmtWrapper { char *path; int32_t refCount; SRWLatch latch; + ENodeType ntype; bool deployed; bool required; EProcType procType; @@ -134,44 +135,72 @@ typedef struct SDnode { SMgmtWrapper wrappers[NODE_MAX]; } SDnode; -// dndFile.h -int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); -int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); +// dndEnv.c +const char *dndStatStr(EDndStatus stat); +const char *dndNodeLogStr(ENodeType ntype); +const char *dndNodeProcStr(ENodeType ntype); +const char *dndEventStr(EDndEvent ev); -// dndInt.h +// dndExec.c +int32_t dndOpenNode(SMgmtWrapper *pWrapper); +void dndCloseNode(SMgmtWrapper *pWrapper); + +// dndFile.c +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); +TdFilePtr dndCheckRunning(const char *dataDir); +int32_t dndReadShmFile(SDnode *pDnode); +int32_t dndWriteShmFile(SDnode *pDnode); + +// dndInt.c EDndStatus dndGetStatus(SDnode *pDnode); void dndSetStatus(SDnode *pDnode, EDndStatus stat); void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); void dndReleaseWrapper(SMgmtWrapper *pWrapper); +void dndHandleEvent(SDnode *pDnode, EDndEvent event); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); -// dndMonitor.h -void dndSendMonitorReport(SDnode *pDnode); +// dndTransport.c +int32_t dndInitTrans(SDnode *pDnode); +void dndCleanupTrans(SDnode *pDnode); +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); +int32_t dndInitMsgHandle(SDnode *pDnode); -// dndMsg.h -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); +// mgmt +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void bmSetMgmtFp(SMgmtWrapper *pWrapper); +void qmSetMgmtFp(SMgmtWrapper *pMgmt); +void smSetMgmtFp(SMgmtWrapper *pWrapper); +void vmSetMgmtFp(SMgmtWrapper *pWrapper); +void mmSetMgmtFp(SMgmtWrapper *pMgmt); -// dndStr.h -const char *dndStatStr(EDndStatus stat); -const char *dndNodeLogStr(ENodeType ntype); -const char *dndNodeProcStr(ENodeType ntype); -const char *dndEventStr(EDndEvent ev); +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); -// dndTransport.h -int32_t dndInitServer(SDnode *pDnode); -void dndCleanupServer(SDnode *pDnode); -int32_t dndInitClient(SDnode *pDnode); -void dndCleanupClient(SDnode *pDnode); -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_H_*/ \ No newline at end of file +#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/inc/mmInt.h similarity index 96% rename from source/dnode/mgmt/mm/inc/mmInt.h rename to source/dnode/mgmt/inc/mmInt.h index 86cba97a33..d09d15255d 100644 --- a/source/dnode/mgmt/mm/inc/mmInt.h +++ b/source/dnode/mgmt/inc/mmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_MNODE_INT_H_ #define _TD_DND_MNODE_INT_H_ -#include "mm.h" +#include "dndInt.h" #include "mnode.h" #ifdef __cplusplus @@ -46,8 +46,8 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq); int32_t mmDrop(SMgmtWrapper *pWrapper); int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq); -// mmMsg.c -void mmInitMsgHandles(SMgmtWrapper *pWrapper); +// mmHandle.c +void mmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/inc/qmInt.h similarity index 95% rename from source/dnode/mgmt/qm/inc/qmInt.h rename to source/dnode/mgmt/inc/qmInt.h index 3e975663d3..02905413ab 100644 --- a/source/dnode/mgmt/qm/inc/qmInt.h +++ b/source/dnode/mgmt/inc/qmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_QNODE_INT_H_ #define _TD_DND_QNODE_INT_H_ -#include "qm.h" +#include "dndInt.h" #include "qnode.h" #ifdef __cplusplus @@ -36,8 +36,8 @@ typedef struct SQnodeMgmt { int32_t qmOpen(SMgmtWrapper *pWrapper); int32_t qmDrop(SMgmtWrapper *pWrapper); -// qmMsg.c -void qmInitMsgHandles(SMgmtWrapper *pWrapper); +// qmHandle.c +void qmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/inc/smInt.h similarity index 95% rename from source/dnode/mgmt/sm/inc/smInt.h rename to source/dnode/mgmt/inc/smInt.h index 9290384cab..285ec84942 100644 --- a/source/dnode/mgmt/sm/inc/smInt.h +++ b/source/dnode/mgmt/inc/smInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_SNODE_INT_H_ #define _TD_DND_SNODE_INT_H_ -#include "sm.h" +#include "dndInt.h" #include "snode.h" #ifdef __cplusplus @@ -38,8 +38,8 @@ typedef struct SSnodeMgmt { int32_t smOpen(SMgmtWrapper *pWrapper); int32_t smDrop(SMgmtWrapper *pWrapper); -// smMsg.c -void smInitMsgHandles(SMgmtWrapper *pWrapper); +// smHandle.c +void smInitMsgHandle(SMgmtWrapper *pWrapper); int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/inc/vmInt.h similarity index 97% rename from source/dnode/mgmt/vm/inc/vmInt.h rename to source/dnode/mgmt/inc/vmInt.h index 6722fe1d65..889ad7c164 100644 --- a/source/dnode/mgmt/vm/inc/vmInt.h +++ b/source/dnode/mgmt/inc/vmInt.h @@ -17,7 +17,7 @@ #define _TD_DND_VNODES_INT_H_ #include "sync.h" -#include "vm.h" +#include "dndInt.h" #include "vnode.h" #ifdef __cplusplus @@ -84,8 +84,8 @@ void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); -// vmMsg.c -void vmInitMsgHandles(SMgmtWrapper *pWrapper); +// vmHandle.c +void vmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/dndEnv.c similarity index 62% rename from source/dnode/mgmt/main/src/dndStr.c rename to source/dnode/mgmt/main/dndEnv.c index 8a5af68b4f..8bdc2867d2 100644 --- a/source/dnode/mgmt/main/src/dndStr.c +++ b/source/dnode/mgmt/main/dndEnv.c @@ -15,6 +15,48 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#include "wal.h" + +static int8_t once = DND_ENV_INIT; + +int32_t dndInit() { + dDebug("start to init dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + SMonCfg monCfg = {0}; + monCfg.maxLogs = tsMonitorMaxLogs; + monCfg.port = tsMonitorPort; + monCfg.server = tsMonitorFqdn; + monCfg.comp = tsMonitorComp; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + return -1; + } + + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + dDebug("start to cleanup dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + monCleanup(); + walCleanUp(); + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} const char *dndStatStr(EDndStatus status) { switch (status) { @@ -25,7 +67,7 @@ const char *dndStatStr(EDndStatus status) { case DND_STAT_STOPPED: return "stopped"; default: - return "unknown"; + return "UNKNOWN"; } } diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/dndExec.c similarity index 78% rename from source/dnode/mgmt/main/src/dndExec.c rename to source/dnode/mgmt/main/dndExec.c index 8837da6d48..d76ef37a5d 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/dndExec.c @@ -65,53 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, mgmt has been closed", pWrapper->name); } -static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - SRpcMsg *pRpc = &pMsg->rpcMsg; - pRpc->pCont = pCont; - dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); - - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - int32_t code = (*msgFp)(pWrapper, pMsg); - - if (code != 0) { - dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); - } - - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pCont); - } -} - -static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - pMsg->pCont = pCont; - dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); - - switch (ftype) { - case PROC_REGIST: - rpcRegisterBrokenLinkArg(pMsg); - break; - case PROC_RELEASE: - rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); - rpcFreeCont(pCont); - break; - case PROC_REQ: - dndSendReqToMnode(pWrapper, pMsg); - // dndSendReq(pWrapper, (const SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); - break; - case PROC_RSP: - dndSendRpcRsp(pWrapper, pMsg); - break; - default: - break; - } - taosMemoryFree(pMsg); -} static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { char tstr[8] = {0}; @@ -135,21 +88,9 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { return 0; } -static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .shm = pWrapper->shm, - .pParent = pWrapper, - .name = pWrapper->name}; - return cfg; +static void dndProcessProcHandle(void *handle) { + SRpcMsg rpcMsg = {.handle = handle, .code = TSDB_CODE_DND_OFFLINE}; + rpcSendResponse(&rpcMsg); } static int32_t dndRunInSingleProcess(SDnode *pDnode) { @@ -283,6 +224,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) { dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); + taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle); dndNewProc(pWrapper, n); } } diff --git a/source/dnode/mgmt/main/src/dndFile.c b/source/dnode/mgmt/main/dndFile.c similarity index 100% rename from source/dnode/mgmt/main/src/dndFile.c rename to source/dnode/mgmt/main/dndFile.c diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/dndInt.c similarity index 83% rename from source/dnode/mgmt/main/src/dndInt.c rename to source/dnode/mgmt/main/dndInt.c index 25e4d89c5b..e85fe8a9fc 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/dndInt.c @@ -82,12 +82,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { } dndSetStatus(pDnode, DND_STAT_INIT); - dmGetMgmtFp(&pDnode->wrappers[DNODE]); - mmGetMgmtFp(&pDnode->wrappers[MNODE]); - vmGetMgmtFp(&pDnode->wrappers[VNODES]); - qmGetMgmtFp(&pDnode->wrappers[QNODE]); - smGetMgmtFp(&pDnode->wrappers[SNODE]); - bmGetMgmtFp(&pDnode->wrappers[BNODE]); + dmSetMgmtFp(&pDnode->wrappers[DNODE]); + mmSetMgmtFp(&pDnode->wrappers[MNODE]); + vmSetMgmtFp(&pDnode->wrappers[VNODES]); + qmSetMgmtFp(&pDnode->wrappers[QNODE]); + smSetMgmtFp(&pDnode->wrappers[SNODE]); + bmSetMgmtFp(&pDnode->wrappers[BNODE]); for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; @@ -95,6 +95,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { pWrapper->path = strdup(path); pWrapper->shm.id = -1; pWrapper->pDnode = pDnode; + pWrapper->ntype = n; if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; @@ -203,3 +204,26 @@ void dndSetStatus(SDnode *pDnode, EDndStatus status) { pDnode->status = status; } } + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +static void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} \ No newline at end of file diff --git a/source/dnode/mgmt/main/dndTransport.c b/source/dnode/mgmt/main/dndTransport.c new file mode 100644 index 0000000000..22895dffcc --- /dev/null +++ b/source/dnode/mgmt/main/dndTransport.c @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" +#define INTERNAL_SECRET "_pwd" + +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); +} + +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } + + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + pMsg->clientIp = connInfo.clientIp; + pMsg->clientPort = connInfo.clientPort; + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + return 0; +} + +static void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = (*msgFp)(pWrapper, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and put into child queue, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle, + PROC_REQ); + } else { + dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + ASSERT(1); + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed in parent process", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + tmsgSendRsp(&rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pMsg->msgType; + bool isReq = msgType & 1u; + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + SMgmtWrapper *pWrapper = pHandle->pWrapper; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dndProcessStartupReq(pDnode, pMsg); + return; + } + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + return; + } + + if (isReq && pMsg->pCont == NULL) { + dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + if (pWrapper == NULL) { + dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + } + + if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = ntohl(pHead->vgId); + if (vgId == QNODE_HANDLE) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MNODE_HANDLE) { + pWrapper = pHandle->pMndWrapper; + } else { + } + } + + dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +} + +static int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcCfp)dndProcessMsg; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = INTERNAL_USER; + rpcInit.ckey = INTERNAL_CKEY; + rpcInit.spi = 1; + rpcInit.parent = pDnode; + + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init dnode rpc client"); + return -1; + } + + dDebug("dnode rpc client is initialized"); + return 0; +} + +static void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dDebug("dnode rpc client is closed"); + } +} + +static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { + SEpSet epSet = {0}; + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); +} + +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, + char *ckey) { + int32_t code = 0; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + if (strcmp(user, INTERNAL_USER) == 0) { + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + } else { + code = -1; + } + + if (code == 0) { + memcpy(secret, pass, TSDB_PASSWORD_LEN); + *spi = 1; + *encrypt = 0; + *ckey = 0; + } + + return code; +} + +static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (dndGetHideUserAuth(pDnode, user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); + + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); + } else { + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +static int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->serverPort; + rpcInit.label = "DND"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = (RpcCfp)dndProcessMsg; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = (RpcAfp)dndRetrieveUserAuthInfo; + rpcInit.parent = pDnode; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init dnode rpc server"); + return -1; + } + + dDebug("dnode rpc server is initialized"); + return 0; +} + +static void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + dDebug("dnode rpc server is closed"); + } +} + +int32_t dndInitTrans(SDnode *pDnode) { + if (dndInitServer(pDnode) != 0) return -1; + if (dndInitClient(pDnode) != 0) return -1; + return 0; +} + +void dndCleanupTrans(SDnode *pDnode) { + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); +} + +int32_t dndInitMsgHandle(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + + for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { + NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + int8_t vgId = pWrapper->msgVgIds[msgIndex]; + if (msgFp == NULL) continue; + + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; + if (vgId == QNODE_HANDLE) { + if (pHandle->pQndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pQndWrapper = pWrapper; + } else if (vgId == MNODE_HANDLE) { + if (pHandle->pMndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pMndWrapper = pWrapper; + } else { + if (pHandle->pWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pWrapper = pWrapper; + } + } + } + + return 0; +} + +static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (pMgmt->clientRpc == NULL) { + terrno = TSDB_CODE_DND_OFFLINE; + return -1; + } + + rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); + return 0; +} + +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { + if (pRsp->code == TSDB_CODE_APP_NOT_READY) { + if (pWrapper->ntype == MNODE) { + dmSendRedirectRsp(pWrapper->pMgmt, pRsp); + return; + } + } + + rpcSendResponse(pRsp); +} + +static int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (dndGetStatus(pWrapper->pDnode) != DND_STAT_RUNNING) { + terrno = TSDB_CODE_DND_OFFLINE; + dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); + return -1; + } + + if (pWrapper->procType != PROC_CHILD) { + return dndSendRpcReq(&pWrapper->pDnode->trans, pEpSet, pReq); + } else { + char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet)); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memcpy(pHead, pReq, sizeof(SRpcMsg)); + memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); + taosProcPutToParentQ(pWrapper->pProc, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen, + PROC_REQ); + taosMemoryFree(pHead); + return 0; + } +} + +static void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { + if (pWrapper->procType != PROC_CHILD) { + dndSendRpcRsp(pWrapper, pRsp); + } else { + taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP); + } +} + +static void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { + if (pWrapper->procType != PROC_CHILD) { + rpcRegisterBrokenLinkArg(pMsg); + } else { + taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGIST); + } +} + +static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type) { + if (pWrapper->procType != PROC_CHILD) { + rpcReleaseHandle(handle, type); + } else { + SRpcMsg msg = {.handle = handle, .code = type}; + taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE); + } +} + +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { + SMsgCb msgCb = { + .pWrapper = pWrapper, + .sendReqFp = dndSendReq, + .sendRspFp = dndSendRsp, + .registerBrokenLinkArgFp = dndRegisterBrokenLinkArg, + .releaseHandleFp = dndReleaseHandle, + }; + return msgCb; +} + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + pMsg->pCont = pCont; + dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); + + switch (ftype) { + case PROC_REGIST: + rpcRegisterBrokenLinkArg(pMsg); + break; + case PROC_RELEASE: + taosProcRemoveHandle(pWrapper->pProc, pMsg->handle); + rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); + rpcFreeCont(pCont); + break; + case PROC_REQ: + dndSendRpcReq(&pWrapper->pDnode->trans, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + break; + case PROC_RSP: + taosProcRemoveHandle(pWrapper->pProc, pMsg->handle); + dndSendRpcRsp(pWrapper, pMsg); + break; + default: + break; + } + taosMemoryFree(pMsg); +} + +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { + SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->shm, + .parent = pWrapper, + .name = pWrapper->name}; + return cfg; +} \ No newline at end of file diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h deleted file mode 100644 index 0c4ada1df3..0000000000 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_INT_H_ -#define _TD_DND_INT_H_ - -#include "dnd.h" - -#include "bm.h" -#include "dm.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// dndEnv.h -int32_t dndInit(); -void dndCleanup(); - -// dndExec.h -int32_t dndOpenNode(SMgmtWrapper *pWrapper); -void dndCloseNode(SMgmtWrapper *pWrapper); -int32_t dndRun(SDnode *pDnode); - -// dndInt.c -SDnode *dndCreate(const SDnodeOpt *pOption); -void dndClose(SDnode *pDnode); -void dndHandleEvent(SDnode *pDnode, EDndEvent event); - -// dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); - -// dndTransport.c -int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); - -// dndFile.c -TdFilePtr dndCheckRunning(const char *dataDir); -int32_t dndReadShmFile(SDnode *pDnode); -int32_t dndWriteShmFile(SDnode *pDnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c deleted file mode 100644 index 9f75594335..0000000000 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" -#include "wal.h" - -static int8_t once = DND_ENV_INIT; - -int32_t dndInit() { - dDebug("start to init dnode env"); - if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { - terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); - return -1; - } - - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - - SMonCfg monCfg = {0}; - monCfg.maxLogs = tsMonitorMaxLogs; - monCfg.port = tsMonitorPort; - monCfg.server = tsMonitorFqdn; - monCfg.comp = tsMonitorComp; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - return -1; - } - - dInfo("dnode env is initialized"); - return 0; -} - -void dndCleanup() { - dDebug("start to cleanup dnode env"); - if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); - return; - } - - monCleanup(); - walCleanUp(); - taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); -} diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c deleted file mode 100644 index bb0f8ccb89..0000000000 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); - dndReleaseWrapper(pWrapper); - } -} - -static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - if (msgFp == NULL) { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - return msgFp; -} - -static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { - SRpcConnInfo connInfo = {0}; - if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); - return -1; - } - - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - pMsg->clientIp = connInfo.clientIp; - pMsg->clientPort = connInfo.clientPort; - memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; -} - -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - int32_t code = -1; - SNodeMsg *pMsg = NULL; - NodeMsgFp msgFp = NULL; - - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - - if (dndMarkWrapper(pWrapper) != 0) goto _OVER; - if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; - if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - - if (pWrapper->procType == PROC_SINGLE) { - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); - code = (*msgFp)(pWrapper, pMsg); - } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); - } else { - dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - ASSERT(1); - } - -_OVER: - if (code == 0) { - if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed in parent process", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { - dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); - } - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - - dndReleaseWrapper(pWrapper); -} - -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper != NULL) { - dndReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - dError("failed to create node since %s", terrstr()); - return -1; - } - - pWrapper = &pDnode->wrappers[ntype]; - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been opened", pWrapper->name); - pWrapper->deployed = true; - } - - return code; -} - -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - dError("failed to drop node since %s", terrstr()); - return -1; - } - - taosWLockLatch(&pWrapper->latch); - pWrapper->deployed = false; - - int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); - if (code != 0) { - pWrapper->deployed = true; - dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); - } else { - pWrapper->deployed = false; - dDebug("node:%s, has been dropped", pWrapper->name); - } - - taosWUnLockLatch(&pWrapper->latch); - dndReleaseWrapper(pWrapper); - return code; -} - -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { - switch (pMsg->rpcMsg.msgType) { - case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); - case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - return -1; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c deleted file mode 100644 index d463f2ba7a..0000000000 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" - -static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMsgHead *pHead = pMsg->pCont; - int32_t vgId = htonl(pHead->vgId); - - SMgmtWrapper *pWrapper = pHandle->pWrapper; - if (vgId == QND_VGID) { - pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { - pWrapper = pHandle->pMndWrapper; - } - - dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, - pMsg->handle, pMsg->ahandle, vgId); - dndProcessRpcMsg(pWrapper, pMsg, pEpSet); -} - -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pRsp->msgType; - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), - pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); - dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); - } - } else { - dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - } -} - -int32_t dndInitClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; - rpcInit.ckey = INTERNAL_CKEY; - rpcInit.spi = 1; - rpcInit.parent = pDnode; - - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - rpcInit.secret = pass; - - pMgmt->clientRpc = rpcOpen(&rpcInit); - if (pMgmt->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; - } - - dDebug("dnode rpc client is initialized"); - return 0; -} - -void dndCleanupClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - if (pMgmt->clientRpc) { - rpcClose(pMgmt->clientRpc); - pMgmt->clientRpc = NULL; - dDebug("dnode rpc client is closed"); - } -} - -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pReq->msgType; - - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, - pReq->handle, pReq->ahandle); - dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); - } - } else { - dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); - dndReleaseWrapper(pWrapper); - } - - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); -} - -static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = 0; - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - - if (strcmp(user, INTERNAL_USER) == 0) { - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - } else { - code = -1; - } - - if (code == 0) { - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - } - - return code; -} - -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { - dndReleaseWrapper(pWrapper); - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - dndReleaseWrapper(pWrapper); - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -int32_t dndInitServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->serverPort; - rpcInit.label = "DND"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; - rpcInit.parent = pDnode; - - pMgmt->serverRpc = rpcOpen(&rpcInit); - if (pMgmt->serverRpc == NULL) { - dError("failed to init dnode rpc server"); - return -1; - } - - dDebug("dnode rpc server is initialized"); - return 0; -} - -void dndCleanupServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - if (pMgmt->serverRpc) { - rpcClose(pMgmt->serverRpc); - pMgmt->serverRpc = NULL; - dDebug("dnode rpc server is closed"); - } -} - -int32_t dndInitMsgHandle(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - for (ENodeType n = 0; n < NODE_MAX; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - - for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { - NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; - int32_t vgId = pWrapper->msgVgIds[msgIndex]; - if (msgFp == NULL) continue; - - // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); - - SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (vgId == QND_VGID) { - if (pHandle->pQndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pQndWrapper = pWrapper; - } else if (vgId == MND_VGID) { - if (pHandle->pMndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pMndWrapper = pWrapper; - } else { - if (pHandle->pWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pWrapper = pWrapper; - } - } - } - - return 0; -} - -static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) { - if (pMgmt->clientRpc == NULL) { - terrno = TSDB_CODE_DND_OFFLINE; - return -1; - } - - rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); - return 0; -} - -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - STransMgmt *pTrans = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper2 = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper2 != NULL) { - dmGetMnodeEpSet(pWrapper2->pMgmt, &epSet); - dndReleaseWrapper(pWrapper2); - } - return dndSendRpcReq(pTrans, &epSet, pReq); -} - -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { - if (pRsp->code == TSDB_CODE_APP_NOT_READY) { - SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - if (pDnodeWrapper != NULL) { - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); - dndReleaseWrapper(pDnodeWrapper); - } else { - rpcSendResponse(pRsp); - } - } else { - rpcSendResponse(pRsp); - } -} - -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - terrno = TSDB_CODE_DND_OFFLINE; - dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); - return -1; - } - - if (pWrapper->procType != PROC_CHILD) { - return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); - } else { - int32_t headLen = sizeof(SRpcMsg) + sizeof(SEpSet); - char *pHead = taosMemoryMalloc(headLen); - if (pHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - memcpy(pHead, pReq, sizeof(SRpcMsg)); - memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); - - taosProcPutToParentQ(pWrapper->pProc, pReq, headLen, pReq->pCont, pReq->contLen, PROC_REQ); - taosMemoryFree(pHead); - return 0; - } -} - -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { - if (pWrapper->procType != PROC_CHILD) { - dndSendRpcRsp(pWrapper, pRsp); - } else { - taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP); - } -} - -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { - if (pWrapper->procType != PROC_CHILD) { - rpcRegisterBrokenLinkArg(pMsg); - } else { - taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGIST); - } -} - -static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type) { - if (pWrapper->procType != PROC_CHILD) { - rpcReleaseHandle(handle, type); - } else { - SRpcMsg msg = {.handle = handle, .code = type}; - taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE); - } -} - -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { - SMsgCb msgCb = { - .pWrapper = pWrapper, - .sendReqFp = dndSendReq, - .sendRspFp = dndSendRsp, - .registerBrokenLinkArgFp = dndRegisterBrokenLinkArg, - .releaseHandleFp = dndReleaseHandle, - }; - return msgCb; -} \ No newline at end of file diff --git a/source/dnode/mgmt/mm/inc/mm.h b/source/dnode/mgmt/mm/inc/mm.h deleted file mode 100644 index 6ed6c42d93..0000000000 --- a/source/dnode/mgmt/mm/inc/mm.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void mmGetMgmtFp(SMgmtWrapper *pMgmt); - -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mm/src/mmFile.c b/source/dnode/mgmt/mm/mmFile.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmFile.c rename to source/dnode/mgmt/mm/mmFile.c diff --git a/source/dnode/mgmt/mm/src/mmMsg.c b/source/dnode/mgmt/mm/mmHandle.c similarity index 77% rename from source/dnode/mgmt/mm/src/mmMsg.c rename to source/dnode/mgmt/mm/mmHandle.c index 6afcd249b3..acf83d4ba8 100644 --- a/source/dnode/mgmt/mm/src/mmMsg.c +++ b/source/dnode/mgmt/mm/mmHandle.c @@ -73,93 +73,93 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { } } -void mmInitMsgHandles(SMgmtWrapper *pWrapper) { +void mmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); } diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/mmInt.c similarity index 94% rename from source/dnode/mgmt/mm/src/mmInt.c rename to source/dnode/mgmt/mm/mmInt.c index 301ca598e6..cd6ea1499f 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/mmInt.c @@ -227,7 +227,7 @@ static int32_t mmStart(SMgmtWrapper *pWrapper) { return mndStart(pMgmt->pMnode); } -void mmGetMgmtFp(SMgmtWrapper *pWrapper) { +void mmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = mmOpen; mgmtFp.closeFp = mmClose; @@ -236,19 +236,11 @@ void mmGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.dropMsgFp = mmProcessDropReq; mgmtFp.requiredFp = mmRequire; - mmInitMsgHandles(pWrapper); + mmInitMsgHandle(pWrapper); pWrapper->name = "mnode"; pWrapper->fp = mgmtFp; } -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - - int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { SMnodeMgmt *pMgmt = pWrapper->pMgmt; diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/mmWorker.c similarity index 98% rename from source/dnode/mgmt/mm/src/mmWorker.c rename to source/dnode/mgmt/mm/mmWorker.c index c4aafe05e4..cde4eb853a 100644 --- a/source/dnode/mgmt/mm/src/mmWorker.c +++ b/source/dnode/mgmt/mm/mmWorker.c @@ -37,7 +37,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); } SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } @@ -60,7 +60,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { if (pRpc->handle != NULL && code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } diff --git a/source/dnode/mgmt/qm/inc/qm.h b/source/dnode/mgmt/qm/inc/qm.h deleted file mode 100644 index e28ea3e948..0000000000 --- a/source/dnode/mgmt/qm/inc/qm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void qmGetMgmtFp(SMgmtWrapper *pMgmt); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qm/src/qmMsg.c b/source/dnode/mgmt/qm/qmHandle.c similarity index 91% rename from source/dnode/mgmt/qm/src/qmMsg.c rename to source/dnode/mgmt/qm/qmHandle.c index da5ba6472a..77a9db1175 100644 --- a/source/dnode/mgmt/qm/src/qmMsg.c +++ b/source/dnode/mgmt/qm/qmHandle.c @@ -54,16 +54,16 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } } -void qmInitMsgHandles(SMgmtWrapper *pWrapper) { +void qmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QNODE_HANDLE); } diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/qmInt.c similarity index 97% rename from source/dnode/mgmt/qm/src/qmInt.c rename to source/dnode/mgmt/qm/qmInt.c index 8079859b96..585a7fb183 100644 --- a/source/dnode/mgmt/qm/src/qmInt.c +++ b/source/dnode/mgmt/qm/qmInt.c @@ -112,7 +112,7 @@ int32_t qmOpen(SMgmtWrapper *pWrapper) { return code; } -void qmGetMgmtFp(SMgmtWrapper *pWrapper) { +void qmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = qmOpen; mgmtFp.closeFp = qmClose; @@ -120,7 +120,7 @@ void qmGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.dropMsgFp = qmProcessDropReq; mgmtFp.requiredFp = qmRequire; - qmInitMsgHandles(pWrapper); + qmInitMsgHandle(pWrapper); pWrapper->name = "qnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c similarity index 99% rename from source/dnode/mgmt/qm/src/qmWorker.c rename to source/dnode/mgmt/qm/qmWorker.c index 14efb311b1..5ceb9dbf88 100644 --- a/source/dnode/mgmt/qm/src/qmWorker.c +++ b/source/dnode/mgmt/qm/qmWorker.c @@ -18,7 +18,7 @@ static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/sm/inc/sm.h b/source/dnode/mgmt/sm/inc/sm.h deleted file mode 100644 index 82a52e5d1f..0000000000 --- a/source/dnode/mgmt/sm/inc/sm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void smGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/sm/src/smMsg.c b/source/dnode/mgmt/sm/smHandle.c similarity index 95% rename from source/dnode/mgmt/sm/src/smMsg.c rename to source/dnode/mgmt/sm/smHandle.c index c522ef7fc3..a1fa41a686 100644 --- a/source/dnode/mgmt/sm/src/smMsg.c +++ b/source/dnode/mgmt/sm/smHandle.c @@ -54,8 +54,8 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } } -void smInitMsgHandles(SMgmtWrapper *pWrapper) { +void smInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/smInt.c similarity index 97% rename from source/dnode/mgmt/sm/src/smInt.c rename to source/dnode/mgmt/sm/smInt.c index 6d2e2aaefd..ef4e95d915 100644 --- a/source/dnode/mgmt/sm/src/smInt.c +++ b/source/dnode/mgmt/sm/smInt.c @@ -109,7 +109,7 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { return code; } -void smGetMgmtFp(SMgmtWrapper *pWrapper) { +void smSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = smOpen; mgmtFp.closeFp = smClose; @@ -117,7 +117,7 @@ void smGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.dropMsgFp = smProcessDropReq; mgmtFp.requiredFp = smRequire; - smInitMsgHandles(pWrapper); + smInitMsgHandle(pWrapper); pWrapper->name = "snode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/sm/src/smWorker.c b/source/dnode/mgmt/sm/smWorker.c similarity index 100% rename from source/dnode/mgmt/sm/src/smWorker.c rename to source/dnode/mgmt/sm/smWorker.c diff --git a/source/dnode/mgmt/vm/inc/vm.h b/source/dnode/mgmt/vm/inc/vm.h deleted file mode 100644 index 60d9cfc3a1..0000000000 --- a/source/dnode/mgmt/vm/inc/vm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -void vmGetMgmtFp(SMgmtWrapper *pWrapper); - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vm/src/vmFile.c b/source/dnode/mgmt/vm/vmFile.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmFile.c rename to source/dnode/mgmt/vm/vmFile.c diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/vmHandle.c similarity index 86% rename from source/dnode/mgmt/vm/src/vmMsg.c rename to source/dnode/mgmt/vm/vmHandle.c index c3ad74d246..bcb9ef9e5a 100644 --- a/source/dnode/mgmt/vm/src/vmMsg.c +++ b/source/dnode/mgmt/vm/vmHandle.c @@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; pCfg->walCfg.level = pCreate->walLevel; @@ -70,6 +71,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); if (pVnode != NULL) { + tFreeSCreateVnodeReq(&createReq); dDebug("vgId:%d, already exist", createReq.vgId); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; @@ -88,12 +90,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); return -1; } int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); @@ -103,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { code = vmWriteVnodesToFile(pMgmt); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; @@ -233,51 +238,51 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return 0; } -void vmInitMsgHandles(SMgmtWrapper *pWrapper) { +void vmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/vmInt.c similarity index 99% rename from source/dnode/mgmt/vm/src/vmInt.c rename to source/dnode/mgmt/vm/vmInt.c index d9ef7b5ae6..b3390ba31c 100644 --- a/source/dnode/mgmt/vm/src/vmInt.c +++ b/source/dnode/mgmt/vm/vmInt.c @@ -333,13 +333,13 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { return 0; } -void vmGetMgmtFp(SMgmtWrapper *pWrapper) { +void vmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = vmInit; mgmtFp.closeFp = vmCleanup; mgmtFp.requiredFp = vmRequire; - vmInitMsgHandles(pWrapper); + vmInitMsgHandle(pWrapper); pWrapper->name = "vnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/vmWorker.c similarity index 99% rename from source/dnode/mgmt/vm/src/vmWorker.c rename to source/dnode/mgmt/vm/vmWorker.c index 4be6311cf8..c228e6e7dd 100644 --- a/source/dnode/mgmt/vm/src/vmWorker.c +++ b/source/dnode/mgmt/vm/vmWorker.c @@ -18,7 +18,7 @@ static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { @@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; - dndSendRsp(pVnode->pWrapper, pRsp); + tmsgSendRsp(pRsp); taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c4c93b7fc9..6080ec7710 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -160,10 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); - pOldConsumer->epoch++; - - // TODO handle update /*taosWLockLatch(&pOldConsumer->lock);*/ + atomic_add_fetch_32(&pOldConsumer->epoch, 1); /*taosWUnLockLatch(&pOldConsumer->lock);*/ return 0; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 462f0eb85a..44c547bec3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, + sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) goto DB_ENCODE_OVER; int32_t dataPos = 0; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 40b4f60bd4..d184e354c4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -52,7 +52,7 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) { } int32_t mndInitQuery(SMnode *pMnode) { - if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { + if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 4562d9e5d3..df7946a0c1 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib consumerEp.consumerId = -1; consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; + + mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId); + int32_t msgLen; if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 211163ce35..57d98396ea 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -212,19 +212,24 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } + //TODO add lock ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + int32_t serverEpoch = pConsumer->epoch; // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mTrace("try to get sub ep, old val: %d", hbStatus); + mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ strcpy(rsp.cgroup, pReq->cgroup); - if (epoch != pConsumer->epoch) { - mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); + if (epoch != serverEpoch) { + mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, serverEpoch); + mDebug("consumer %ld try r lock", consumerId); + taosRLockLatch(&pConsumer->lock); + mDebug("consumer %ld r locked", consumerId); SArray *pTopics = pConsumer->currentTopics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -238,7 +243,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); if (consumerId == pSubConsumer->consumerId) { int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - mInfo("topic %s has %d vg", topicName, pConsumer->epoch); + mInfo("topic %s has %d vg", topicName, serverEpoch); SMqSubTopicEp topicEp; strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); @@ -264,6 +269,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { } mndReleaseSubscribe(pMnode, pSub); } + taosRUnLockLatch(&pConsumer->lock); + mDebug("consumer %ld r unlock", consumerId); } int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -272,7 +279,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { return -1; } ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + ((SMqRspHead *)buf)->epoch = serverEpoch; ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -395,7 +402,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); taosMemoryFreeClear(pRebSub->key); - mInfo("mq rebalance subscription: %s", pSub->key); + mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg)); // remove lost consumer for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { @@ -442,6 +449,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { } SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); + mDebug("consumer %ld try w lock", pRebConsumer->consumerId); + taosWLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w locked", pRebConsumer->consumerId); int32_t status = atomic_load_32(&pRebConsumer->status); if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || @@ -462,6 +472,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); mndTransAppendCommitlog(pTrans, pConsumerRaw); } + taosWUnLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w unlock", pRebConsumer->consumerId); mndReleaseConsumer(pMnode, pRebConsumer); } diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 1e4a7e5a93..38d6d879fa 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -55,13 +55,14 @@ typedef struct STsdbCfg { int8_t precision; int8_t update; int8_t compression; - uint64_t lruCacheSize; int32_t daysPerFile; int32_t minRowsPerFileBlock; int32_t maxRowsPerFileBlock; int32_t keep; int32_t keep1; int32_t keep2; + uint64_t lruCacheSize; + SArray *retentions; } STsdbCfg; // query condition to build multi-table data block iterator @@ -171,9 +172,9 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); -int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo); +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo); -bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); +bool isTsdbCacheLastRow(tsdbReaderT* pReader); /** * diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 17c220a35a..90a877bb83 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -112,10 +112,10 @@ typedef struct { #else typedef struct { int16_t colId; - uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint8_t reserve : 7; - uint8_t type; - int32_t len; + uint16_t type : 6; + uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) + uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t len : 31; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d5e9b55a71..2a8875fefe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish @@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; + int32_t tBitmapsLen = 0; if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tBitmaps = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tlen += tBitmaps; @@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif // move bitmap parts ahead // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); } #endif - void *tptr; + void *tptr, *bptr; // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); @@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF // Compress or just copy if (pCfg->compression) { +#if 0 flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); +#endif + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + if (tBitmaps > 0) { + bptr = POINTER_SHIFT(pBlockData, lsize + flen); + tBitmapsLen = + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, + pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); + TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); + flen += tBitmapsLen; + } } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); + if (tBitmaps > 0) { + bptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(bptr, pDataCol->pBitmap, tBitmaps); + tBitmapsLen = tBitmaps; + flen += tBitmapsLen; + } } // Add checksum ASSERT(flen > 0); + ASSERT(tBitmapsLen <= 1024); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; + pBlockCol->len = flen; // data + bitmaps + pBlockCol->blen = tBitmapsLen; ++tcol; } else { keyLen = flen; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4a3e764b13..144aa7fcdc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3046,8 +3046,8 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) { // return code; //} -bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle) { - return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; +bool isTsdbCacheLastRow(tsdbReaderT* pReader) { + return ((STsdbReadHandle *)pReader)->cachelastrow > TSDB_CACHED_TYPE_NONE; } int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 304b3286fe..8cb90d76a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, int numOfColIds); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); @@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; continue; } @@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat TD_SET_COL_ROWS_NORM(pDataCol); } - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } if (tcolId == pDataCol->colId) { if (pBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), - (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen, + pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints, + TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return 0; } -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize) { +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) { if (!taosCheckChecksumWhole((uint8_t *)content, len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; @@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 if (comp) { // Need to decompress int tlen = - (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps, + (*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); + tsdbError( + "Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + (int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } pDataCol->len = tlen; + + if (numOfBitmaps > 0) { + tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps, + pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError( + "Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d " + "bufferSize:%d", + bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + // pDataCol->blen = tlen; + } } else { // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); + pDataCol->len = len - bitmapLen - sizeof(TSCKSUM); memcpy(pDataCol->pData, content, pDataCol->len); + if (numOfBitmaps > 0) { + // pDataCol->blen = bitmapLen; + memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen); + } } +#if 0 if (lenOfBitmaps > 0) { pDataCol->len -= lenOfBitmaps; @@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } else if (IS_VAR_DATA_TYPE(pDataCol->type)) { dataColSetOffset(pDataCol, numOfRows); } - +#endif + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } return 0; } @@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb * pRepo = TSDB_READ_REPO(pReadh); + STsdb *pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); - int32_t tBitmaps = 0; + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if (!TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; @@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, - tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm, + pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), pBlockCol->colId, offset); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 771baca2ab..23e4f2b24f 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -26,38 +26,54 @@ extern "C" { #define EXPLAIN_MAX_GROUP_NUM 100 //newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d" -#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" -#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" -#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" -#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" -#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" -#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s" +#define EXPLAIN_PROJECTION_FORMAT "Projection" +#define EXPLAIN_JOIN_FORMAT "%s" +#define EXPLAIN_AGG_FORMAT "Aggragate" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1" +#define EXPLAIN_SORT_FORMAT "Sort" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s" +#define EXPLAIN_SESSION_FORMAT "Session" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " #define EXPLAIN_FILL_FORMAT "Fill: %s" #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" +#define EXPLAIN_OUTPUT_FORMAT "Output: " +#define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c" +#define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64 //append area -#define EXPLAIN_GROUPS_FORMAT " groups=%d" -#define EXPLAIN_WIDTH_FORMAT " width=%d" -#define EXPLAIN_LOOPS_FORMAT " loops=%d" -#define EXPLAIN_REVERSE_FORMAT " reverse=%d" +#define EXPLAIN_LEFT_PARENTHESIS_FORMAT " (" +#define EXPLAIN_RIGHT_PARENTHESIS_FORMAT ")" +#define EXPLAIN_BLANK_FORMAT " " +#define EXPLAIN_COST_FORMAT "cost=%.2f..%.2f" +#define EXPLAIN_ROWS_FORMAT "rows=%" PRIu64 +#define EXPLAIN_COLUMNS_FORMAT "columns=%d" +#define EXPLAIN_WIDTH_FORMAT "width=%d" +#define EXPLAIN_GROUPS_FORMAT "groups=%d" +#define EXPLAIN_WIDTH_FORMAT "width=%d" +#define EXPLAIN_LOOPS_FORMAT "loops=%d" +#define EXPLAIN_REVERSE_FORMAT "reverse=%d" +#define EXPLAIN_FUNCTIONS_FORMAT "functions=%d" +#define EXPLAIN_EXECINFO_FORMAT "cost=%" PRIu64 "..%" PRIu64 " rows=%" PRIu64 typedef struct SExplainGroup { int32_t nodeNum; + int32_t physiPlanExecNum; + int32_t physiPlanNum; + int32_t physiPlanExecIdx; + SRWLatch lock; SSubplan *plan; - void *execInfo; //TODO + SArray *nodeExecInfo; //Array } SExplainGroup; typedef struct SExplainResNode { - SNodeList* pChildren; - SPhysiNode* pNode; - void* pExecInfo; + SNodeList* pChildren; + SPhysiNode* pNode; + SArray* pExecInfo; // Array } SExplainResNode; typedef struct SQueryExplainRowInfo { @@ -67,11 +83,21 @@ typedef struct SQueryExplainRowInfo { } SQueryExplainRowInfo; typedef struct SExplainCtx { - int32_t totalSize; - bool verbose; - char *tbuf; - SArray *rows; - SHashObj *groupHash; + EExplainMode mode; + double ratio; + bool verbose; + + SRWLatch lock; + int32_t rootGroupId; + int32_t dataSize; + bool execDone; + int64_t reqStartTs; + int64_t jobStartTs; + int64_t jobDoneTs; + char *tbuf; + SArray *rows; + int32_t groupDoneNum; + SHashObj *groupHash; // Hash } SExplainCtx; #define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 847a863e76..2a36113749 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -17,44 +17,61 @@ #include "plannodes.h" #include "commandInt.h" -int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); -int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); +int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); +int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); -void qFreeExplainResTree(SExplainResNode *res) { - if (NULL == res) { +void qExplainFreeResNode(SExplainResNode *resNode) { + if (NULL == resNode) { return; } - taosMemoryFreeClear(res->pExecInfo); + taosMemoryFreeClear(resNode->pExecInfo); SNode* node = NULL; - FOREACH(node, res->pChildren) { - qFreeExplainResTree((SExplainResNode *)node); + FOREACH(node, resNode->pChildren) { + qExplainFreeResNode((SExplainResNode *)node); } - nodesClearList(res->pChildren); + nodesClearList(resNode->pChildren); - taosMemoryFreeClear(res); + taosMemoryFreeClear(resNode); } -void qFreeExplainCtx(void *ctx) { - if (NULL == ctx) { +void qExplainFreeCtx(SExplainCtx *pCtx) { + if (NULL == pCtx) { return; } - SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowSize = taosArrayGetSize(pCtx->rows); for (int32_t i = 0; i < rowSize; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); taosMemoryFreeClear(row->buf); } + if (EXPLAIN_MODE_ANALYZE == pCtx->mode && pCtx->groupHash) { + void *pIter = taosHashIterate(pCtx->groupHash, NULL); + while (pIter) { + SExplainGroup *group = (SExplainGroup *)pIter; + if (NULL == group->nodeExecInfo) { + continue; + } + + int32_t num = taosArrayGetSize(group->nodeExecInfo); + for (int32_t i = 0; i < num; ++i) { + SExplainRsp *rsp = taosArrayGet(group->nodeExecInfo, i); + taosMemoryFreeClear(rsp->subplanInfo); + } + + pIter = taosHashIterate(pCtx->groupHash, pIter); + } + } + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } -int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) { +int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { @@ -74,7 +91,9 @@ int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) { QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + ctx->mode = mode; ctx->verbose = verbose; + ctx->ratio = ratio; ctx->tbuf = tbuf; ctx->rows = rows; ctx->groupHash = groupHash; @@ -92,35 +111,7 @@ _return: QRY_RET(code); } - -char *qFillModeString(EFillMode mode) { - switch (mode) { - case FILL_MODE_NONE: - return "none"; - case FILL_MODE_VALUE: - return "value"; - case FILL_MODE_PREV: - return "prev"; - case FILL_MODE_NULL: - return "null"; - case FILL_MODE_LINEAR: - return "linear"; - case FILL_MODE_NEXT: - return "next"; - default: - return "unknown"; - } -} - -char *qGetNameFromColumnNode(SNode *pNode) { - if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) { - return "NULL"; - } - - return ((SColumnNode *)pNode)->colName; -} - -int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { +int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; @@ -192,52 +183,119 @@ int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeLis SNode* node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { - QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); } return TSDB_CODE_SUCCESS; } -int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { +int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group) { + *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); + if (NULL == (*pExecInfo)) { + qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + SExplainRsp *rsp = NULL; + for (int32_t i = 0; i < group->nodeNum; ++i) { + rsp = taosArrayGet(group->nodeExecInfo, i); + if (group->physiPlanExecIdx >= rsp->numOfPlans) { + qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); + return TSDB_CODE_QRY_APP_ERROR; + } + + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); + } + + ++group->physiPlanExecIdx; + + return TSDB_CODE_SUCCESS; +} + +int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) { if (NULL == pNode) { - *pRes = NULL; + *pResNode = NULL; qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - - SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode)); - if (NULL == res) { + + SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); + if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } int32_t code = 0; - res->pNode = pNode; - res->pExecInfo = pExecInfo; - QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren)); + resNode->pNode = pNode; - *pRes = res; + if (group->nodeExecInfo) { + QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(&resNode->pExecInfo, group)); + } + + QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); + + ++group->physiPlanNum; + + *pResNode = resNode; return TSDB_CODE_SUCCESS; _return: - qFreeExplainResTree(res); + qExplainFreeResNode(resNode); QRY_RET(code); } -int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { +int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = *len; + int32_t nodeNum = taosArrayGetSize(pExecInfo); + SExplainExecInfo maxExecInfo = {0}; - EXPLAIN_ROW_APPEND("(exec info here)"); + for (int32_t i = 0; i < nodeNum; ++i) { + SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); + if (execInfo->startupCost > maxExecInfo.startupCost) { + maxExecInfo.startupCost = execInfo->startupCost; + } + if (execInfo->totalCost > maxExecInfo.totalCost) { + maxExecInfo.totalCost = execInfo->totalCost; + } + if (execInfo->numOfRows > maxExecInfo.numOfRows) { + maxExecInfo.numOfRows = execInfo->numOfRows; + } + } + EXPLAIN_ROW_APPEND(EXPLAIN_EXECINFO_FORMAT, maxExecInfo.startupCost, maxExecInfo.totalCost, maxExecInfo.numOfRows); + *len = tlen; return TSDB_CODE_SUCCESS; } +int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { + int32_t tlen = 0; + bool gotVerbose = false; + int32_t nodeNum = taosArrayGetSize(pExecInfo); + SExplainExecInfo maxExecInfo = {0}; + + for (int32_t i = 0; i < nodeNum; ++i) { + SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); + if (execInfo->verboseInfo) { + gotVerbose = true; + } + } + + if (gotVerbose) { + EXPLAIN_ROW_APPEND("exec verbose info"); + } + + *len = tlen; + + return TSDB_CODE_SUCCESS; +} + + int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); @@ -249,7 +307,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; row.len = len; - ctx->totalSize += len; + ctx->dataSize += len; if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); @@ -275,39 +333,77 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTagScanNode->pScanCols->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); if (pTagScanNode->reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); } + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + if (tlen) { + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); if (pTblScanNode->scan.reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); } + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -327,18 +423,33 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); if (pSTblScanNode->scan.reverse) { + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); } + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -355,14 +466,27 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pPrjNode->pProjections->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -374,14 +498,27 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType)); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -398,19 +535,31 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length); - if (pAggNode->pGroupKeys) { - EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); - } - EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); - + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pAggNode->pAggFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->totalRowSize); + if (pAggNode->pGroupKeys) { + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); } + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -428,14 +577,25 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -444,19 +604,33 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } } - QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1)); + QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSortNode->pSortKeys->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -468,20 +642,34 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, - pIntNode->offset, getPrecisionUnit(pIntNode->precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, - pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->pTspk)); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), + pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(pIntNode->precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, getFillModeString(pIntNode->pFill->mode)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -496,18 +684,36 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i break; } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ - SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); - } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pSessNode->window.pFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + if (verbose) { - if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_WINDOW_FORMAT, pSessNode->gap); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pSessNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -540,7 +746,7 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 return TSDB_CODE_SUCCESS; } -int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) { +int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; int32_t code = 0; SExplainCtx *ctx = (SExplainCtx *)pCtx; @@ -551,19 +757,24 @@ int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node)); + QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); + + if ((EXPLAIN_MODE_ANALYZE == ctx->mode) && (group->physiPlanNum != group->physiPlanExecNum)) { + qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, groupId); + QRY_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); _return: - qFreeExplainResTree(node); + qExplainFreeResNode(node); QRY_RET(code); } -int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { +int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { @@ -572,7 +783,7 @@ int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -582,7 +793,7 @@ int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = htonl(pCtx->totalSize); + *(int32_t *)rsp->data = htonl(pCtx->dataSize); int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); char *data = (char *)(offset + rowNum); @@ -604,13 +815,13 @@ int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } -int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { + +int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; SExplainGroup *pGroup = NULL; - void *pCtx = NULL; - int32_t rootGroupId = 0; + SExplainCtx *ctx = NULL; if (pDag->numOfSubplans <= 0) { qError("invalid subplan num:%d", pDag->numOfSubplans); @@ -629,7 +840,7 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); + QRY_ERR_JRET(qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -653,7 +864,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { continue; } - SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; + SExplainGroup group = {0}; + group.nodeNum = 1; + group.plan = plan; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -666,22 +880,130 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - rootGroupId = plan->id.groupId; + ctx->rootGroupId = plan->id.groupId; } qDebug("level %d group handled, taskNum:%d", i, taskNum); } - QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); - - QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp)); + *pCtx = ctx; + + return TSDB_CODE_SUCCESS; _return: - qFreeExplainCtx(pCtx); + qExplainFreeCtx(ctx); QRY_RET(code); } +int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { + QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0)); + + QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { + SExplainResNode *node = NULL; + int32_t code = 0; + bool groupDone = false; + SExplainCtx *ctx = (SExplainCtx *)pCtx; + + SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); + if (NULL == group) { + qError("group %d not in groupHash", groupId); + taosMemoryFreeClear(pRspMsg->subplanInfo); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + taosWLockLatch(&group->lock); + if (NULL == group->nodeExecInfo) { + group->nodeExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainRsp)); + if (NULL == group->nodeExecInfo) { + qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); + taosMemoryFreeClear(pRspMsg->subplanInfo); + taosWUnLockLatch(&group->lock); + + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + group->physiPlanExecNum = pRspMsg->numOfPlans; + } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { + qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), group->nodeNum); + taosMemoryFreeClear(pRspMsg->subplanInfo); + taosWUnLockLatch(&group->lock); + + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (group->physiPlanExecNum != pRspMsg->numOfPlans) { + qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, groupId); + taosMemoryFreeClear(pRspMsg->subplanInfo); + taosWUnLockLatch(&group->lock); + + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + taosArrayPush(group->nodeExecInfo, pRspMsg); + groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); + + taosWUnLockLatch(&group->lock); + + if (groupDone && (taosHashGetSize(pCtx->groupHash) == atomic_add_fetch_32(&pCtx->groupDoneNum, 1))) { + if (atomic_load_8((int8_t *)&pCtx->execDone)) { + if (0 == taosWTryLockLatch(&pCtx->lock)) { + QRY_ERR_RET(qExplainGenerateRsp(pCtx, pRsp)); + // LEAVE LOCK THERE + } + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { + int32_t code = 0; + SExplainCtx *pCtx = NULL; + + QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); + + QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); + +_return: + + qExplainFreeCtx(pCtx); + + QRY_RET(code); +} + +int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int32_t startTs) { + QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx)); + + (*pCtx)->reqStartTs = startTs; + (*pCtx)->jobStartTs = taosGetTimestampMs(); + + return TSDB_CODE_SUCCESS; +} + +int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { + int32_t code = 0; + pCtx->jobDoneTs = taosGetTimestampMs(); + + atomic_store_8((int8_t *)&pCtx->execDone, true); + + if (taosHashGetSize(pCtx->groupHash) == atomic_load_32(&pCtx->groupDoneNum)) { + if (0 == taosWTryLockLatch(&pCtx->lock)) { + QRY_ERR_RET(qExplainGenerateRsp(pCtx, pRsp)); + // LEAVE LOCK THERE + } + } + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 12a78134c3..f947acf822 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,17 +1,12 @@ aux_source_directory(src EXECUTOR_SRC) #add_library(executor ${EXECUTOR_SRC}) - -#target_link_libraries( -# executor -# PRIVATE os util common function parser planner qcom tsdb -#) - add_library(executor STATIC ${EXECUTOR_SRC}) #set_target_properties(executor PROPERTIES # IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) + target_link_libraries(executor PRIVATE os util common function parser planner qcom vnode scalar nodes ) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5a3e25bd68..48e5dbeeb1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -36,6 +36,7 @@ extern "C" { #include "thash.h" #include "tlockfree.h" #include "tpagedbuf.h" +#include "tmsg.h" struct SColumnFilterElem; @@ -165,7 +166,7 @@ typedef struct STaskCostInfo { typedef struct SOperatorCostInfo { uint64_t openCost; - uint64_t execCost; + uint64_t totalCost; } SOperatorCostInfo; typedef struct SOrder { @@ -238,6 +239,7 @@ typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup); typedef void (*__optr_close_fn_t)(void* param, int32_t num); +typedef int32_t (*__optr_get_explain_fn_t)(struct SOperatorInfo* pOptr, void **pOptrExplain); typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id @@ -306,26 +308,27 @@ enum { }; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char* name; // name, used to show the query execution plan - void* info; // extension attribution - SExprInfo* pExpr; - STaskRuntimeEnv* pRuntimeEnv; // todo remove it - SExecTaskInfo* pTaskInfo; - SOperatorCostInfo cost; - SResultInfo resultInfo; - struct SOperatorInfo** pDownstream; // downstram pointer list - int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t getStreamResFn; // execute the aggregate in the stream model. - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char* name; // name, used to show the query execution plan + void* info; // extension attribution + SExprInfo* pExpr; + STaskRuntimeEnv* pRuntimeEnv; // todo remove it + SExecTaskInfo* pTaskInfo; + SOperatorCostInfo cost; + SResultInfo resultInfo; + struct SOperatorInfo** pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t getStreamResFn; // execute the aggregate in the stream model. + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_get_explain_fn_t getExplainFn; } SOperatorInfo; typedef struct { @@ -385,6 +388,12 @@ typedef struct SExchangeInfo { SLoadRemoteDataInfo loadInfo; } SExchangeInfo; +typedef struct SColMatchInfo { + int32_t colId; + int32_t targetSlotId; + bool output; +} SColMatchInfo; + typedef struct STableScanInfo { void* dataReader; int32_t numOfBlocks; // extract basic running information. @@ -497,8 +506,9 @@ typedef struct SAggOperatorInfo { typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; + SAggSupporter aggSup; SSDataBlock *existDataBlock; - int32_t threshold; + SArray *pPseudoColInfo; SLimit limit; int64_t curOffset; int64_t curOutput; @@ -544,6 +554,7 @@ typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; SArray* pGroupCols; SArray* pGroupColVals; // current group column values, SArray + SNode* pCondition; bool isInit; // denote if current val is initialized or not char* keyBuf; // group by keys for hash int32_t groupKeyLen; // total group by column width @@ -623,13 +634,31 @@ typedef struct SDistinctOperatorInfo { SHashObj* pSet; SSDataBlock* pRes; bool recordNullVal; // has already record the null value, no need to try again - int64_t threshold; - int64_t outputCapacity; - int32_t totalBytes; +// int64_t threshold; // todo remove it +// int64_t outputCapacity;// todo remove it +// int32_t totalBytes; // todo remove it + SResultInfo resInfo; char* buf; SArray* pDistinctDataInfo; } SDistinctOperatorInfo; +int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); +void operatorDummyCloseFn(void* param, int32_t numOfCols); +int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); +int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, + int32_t numOfRows, SSDataBlock* pResultBlock, const char* pkey); +void toSDatablock(SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); +void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); +int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, + int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); +void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput); +int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, + char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, + uint64_t* total, SArray* pColList); +void doSetOperatorCompleted(SOperatorInfo* pOperator); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); + SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SNode* pCondition, SExecTaskInfo* pTaskInfo); @@ -645,14 +674,14 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, - SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); + SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, @@ -700,6 +729,7 @@ int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model); +int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo **pRes, int32_t *capacity, int32_t *resNum); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 6d78fa835a..d7c99d54df 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -229,3 +229,12 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) { queryCostStatis(pTaskInfo); // print the query cost summary doDestroyTask(pTaskInfo); } + +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes) { + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)tinfo; + int32_t capacity = 0; + + return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); +} + + diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c418d78540..f3f52455f0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,12 +13,11 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include +#include "filter.h" +#include "functionMgt.h" +#include "function.h" +#include "querynodes.h" +#include "tname.h" #include "os.h" #include "parser.h" @@ -29,7 +28,6 @@ #include "tsort.h" #include "ttime.h" -#include "../../function/inc/taggfunction.h" #include "executorimpl.h" #include "function.h" #include "query.h" @@ -47,15 +45,12 @@ #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SDATA_BLOCK_INITIALIZER \ (SDataBlockInfo) { {0}, 0 } #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MULTI_KEY_DELIM "-" - enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, @@ -67,11 +62,6 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; -typedef struct SColMatchInfo { - int32_t colId; - int32_t targetSlotId; - bool output; -} SColMatchInfo; #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { @@ -215,7 +205,6 @@ static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); @@ -230,28 +219,26 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); -static void doSetOperatorCompleted(SOperatorInfo* pOperator) { +void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; if (pOperator->pTaskInfo != NULL) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } } + #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -static int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { +int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } -static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} +void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); -static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, - int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, - SAggSupporter* pAggSup); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow* win); @@ -988,7 +975,7 @@ static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, b ts[4] = pWin->ekey + delta; // window end key } -static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, +void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pWin->skey; @@ -1260,8 +1247,17 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction } } +static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) { + size_t num = (pPseudoList != NULL)? taosArrayGetSize(pPseudoList):0; + for (int32_t i = 0; i < num; ++i) { + pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i); + } +} + static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput) { + int32_t numOfOutput, SArray* pPseudoList) { + setPseudoOutputColInfo(pResult, pCtx, pPseudoList); + for (int32_t k = 0; k < numOfOutput; ++k) { if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k); @@ -1280,19 +1276,44 @@ static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSData taosArrayDestroy(pBlockList); } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { - ASSERT(!fmIsAggFunc(pCtx->functionId)); + ASSERT(!fmIsAggFunc(pCtx[k].functionId)); - SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); - taosArrayPush(pBlockList, &pSrcBlock); + if (fmIsPseudoColumnFunc(pCtx[k].functionId)) { + // do nothing + } else if (fmIsNonstandardSQLFunc(pCtx[k].functionId)) { + // todo set the correct timestamp column + pCtx[k].input.pPTS = taosArrayGet(pSrcBlock->pDataBlock, 1); - SScalarParam dest = {0}; - dest.columnData = taosArrayGet(pResult->pDataBlock, k); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[k]); + pCtx[k].fpSet.init(&pCtx[k], pResInfo); - scalarCalculate((SNode *)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); - pResult->info.rows = dest.numOfRows; + pCtx[k].pOutput = taosArrayGet(pResult->pDataBlock, k); + pCtx[k].offset = pResult->info.rows; // set the start offset - taosArrayDestroy(pBlockList); + if (taosArrayGetSize(pPseudoList) > 0) { + int32_t* outputColIndex = taosArrayGet(pPseudoList, 0); + pCtx[k].pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput; + } + int32_t numOfRows = pCtx[k].fpSet.process(&pCtx[k]); + pResult->info.rows += numOfRows; + } else { + SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pBlockList, &pSrcBlock); + + SScalarParam dest = {0}; + dest.columnData = taosArrayGet(pResult->pDataBlock, k); + + scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + pResult->info.rows = dest.numOfRows; + taosArrayDestroy(pBlockList); + } + } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { + SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k); + for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) { + colDataAppend(pColInfoData, i, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].type), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); + } + pResult->info.rows = pSrcBlock->info.rows; } else { ASSERT(0); } @@ -1687,179 +1708,6 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } -static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, - int32_t numOfGroupCols) { - SColumnDataAgg* pColAgg = NULL; - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); - if (pBlock->pBlockAgg != NULL) { - pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? - } - - bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); - - SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); - if (pkey->isNull && isNull) { - continue; - } - - if (isNull || pkey->isNull) { - return false; - } - - char* val = colDataGetData(pColInfoData, rowIndex); - - if (IS_VAR_DATA_TYPE(pkey->type)) { - int32_t len = varDataLen(val); - if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { - continue; - } else { - return false; - } - } else { - if (memcmp(pkey->pData, val, pkey->bytes) != 0) { - return false; - } - } - } - - return true; -} - -static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { - SColumnDataAgg* pColAgg = NULL; - - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); - - if (pBlock->pBlockAgg != NULL) { - pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? - } - - SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); - if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { - pkey->isNull = true; - } else { - char* val = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pkey->type)) { - memcpy(pkey->pData, val, varDataTLen(val)); - } else { - memcpy(pkey->pData, val, pkey->bytes); - } - } - } -} - -static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVals) { - ASSERT(pKey != NULL); - size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); - - char* isNull = (char*)pKey; - char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); - if (pkey->isNull) { - isNull[i] = 1; - continue; - } - - isNull[i] = 0; - if (IS_VAR_DATA_TYPE(pkey->type)) { - varDataCopy(pStart, pkey->pData); - pStart += varDataTLen(pkey->pData); - ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); - } else { - memcpy(pStart, pkey->pData, pkey->bytes); - pStart += pkey->bytes; - } - } - - *length = (pStart - (char*)pKey); - return 0; -} - -// assign the group keys or user input constant values if required -static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { - for (int32_t i = 0; i < numOfOutput; ++i) { - if (pCtx[i].functionId == -1) { - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); - - SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; - if (!colDataIsNull(pColInfoData, totalRows, rowIndex, NULL)) { - char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); - char* data = colDataGetData(pColInfoData, rowIndex); - - // set result exists, todo refactor - memcpy(dest, data, pColInfoData->info.bytes); - pEntryInfo->hasResult = DATA_SET_FLAG; - pEntryInfo->numOfRes = 1; - } - } - } -} - -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SGroupbyOperatorInfo* pInfo = pOperator->info; - - SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; - int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); - // if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - // qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); - // return; - // } - - int32_t len = 0; - STimeWindow w = TSWINDOW_INITIALIZER; - - int32_t num = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (!pInfo->isInit) { - keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); - pInfo->isInit = true; - num++; - continue; - } - - bool equal = groupKeyCompare(pInfo, pBlock, j, numOfGroupCols); - if (equal) { - num++; - continue; - } - - /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - int32_t rowIndex = j - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); - - // assign the group keys or user input constant values if required - doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); - keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); - num = 1; - } - - if (num > 0) { - /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); - int32_t ret = - setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, - 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - int32_t rowIndex = pBlock->info.rows - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); - doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); - } -} - static void doKeepTuple(SSessionAggOperatorInfo* pInfo, int64_t ts) { pInfo->curWindow.ekey = ts; pInfo->prevTs = ts; @@ -1953,7 +1801,7 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } } -static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, +int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup) { SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; @@ -2087,7 +1935,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num SFuncExecEnv env = {0}; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; - if (fmIsAggFunc(pCtx->functionId)) { + if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) { fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); } else { @@ -2097,7 +1945,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num } } pCtx->resDataInfo.interBufSize = env.calcMemSize; - } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) { + } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || pExpr->pExpr->nodeType == QUERY_NODE_VALUE) { pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element. } @@ -2105,12 +1953,12 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->ptsOutputBuf = NULL; + pCtx->pTsOutput = NULL;//taosArrayInit(4, POINTER_BYTES); pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; - pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; + pCtx->resDataInfo.type = pFunct->resSchema.type; + pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + pCtx->end.key = INT64_MIN; #if 0 for (int32_t j = 0; j < pCtx->numOfParams; ++j) { // int16_t type = pFunct->param[j].nType; @@ -2228,43 +2076,6 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; } -// static bool isFixedOutputQuery(STaskAttr* pQueryAttr) { -// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { -// return false; -// } -// -// // Note:top/bottom query is fixed output query -// if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) { -// return true; -// } -// -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// SExprBasicInfo *pExpr = &pQueryAttr->pExpr1[i].base; -// -// if (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_TS_DUMMY) { -// continue; -// } -// -// if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) { -// return true; -// } -// } -// -// return false; -// } - -// todo refactor with isLastRowQuery -// bool isPointInterpoQuery(STaskAttr *pQueryAttr) { -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; -// if (functionId == FUNCTION_INTERP) { -// return true; -// } -// } -// -// return false; -//} - static bool isCachedLastQuery(STaskAttr* pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -2313,40 +2124,6 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t ke } } -/* - * todo add more parameters to check soon.. - */ -bool colIdCheck(STaskAttr* pQueryAttr, uint64_t qId) { - // load data column information is incorrect - for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { - if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { - // qError("QInfo:0x%"PRIx64" invalid data load column for query", qId); - return false; - } - } - - return true; -} - -// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which -// the scan order is not matter -static bool onlyOneQueryType(STaskAttr* pQueryAttr, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - - if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || - functionId == FUNCTION_TAG_DUMMY) { - continue; - } - - if (functionId != functId && functionId != functIdDst) { - return false; - } - } - - return true; -} - static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { bool hasFirstLastFunc = false; bool hasOtherFunc = false; @@ -2823,84 +2600,6 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData return status; } -void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { - // set the initial static data value filter expression - for (int32_t i = 0; i < numOfFilterCols; ++i) { - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); - - if (pFilterInfo[i].info.colId == pColInfo->info.colId) { - pFilterInfo[i].pData = pColInfo->pData; - break; - } - } - } -} - -int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { - STaskCostInfo* pCost = &pTaskInfo->cost; - - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; - - pCost->totalCheckedRows += pBlock->info.rows; - pCost->loadBlocks += 1; - - *status = BLK_DATA_ALL_NEEDED; - - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (pCols == NULL) { - return terrno; - } - - int32_t numOfCols = pBlock->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - - ASSERT(pColMatchInfo->colId == p->info.colId); - taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); - } - - if (pTableScanInfo->pFilterNode != NULL) { - SFilterInfo* filter = NULL; - int32_t code = filterInitFromNode((SNode*)pTableScanInfo->pFilterNode, &filter, 0); - - SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; - code = filterSetDataFromSlotId(filter, ¶m1); - - int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); - - SSDataBlock* px = createOneDataBlock(pBlock); - blockDataEnsureCapacity(px, pBlock->info.rows); - - int32_t numOfRow = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); - - numOfRow = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; - } - - colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); - numOfRow += 1; - } - *pSrc = *pDst; - } - - pBlock->info.rows = numOfRow; - } - - return TSDB_CODE_SUCCESS; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; @@ -3275,35 +2974,6 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) } } -static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { -#if 0 - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); - SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pCheckInfo); - - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle -// STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); -// pTableKeyInfo->lastKey = pCheckInfo->lastKey; -// -// assert(pCheckInfo->pTable == pTableKeyInfo->pTable); - } - } -#endif -} - -void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t i = 0; i < numOfOutput; ++i) { - SWITCH_ORDER(pCtx[i].order); - } -} - void initResultRow(SResultRow* pResultRow) { pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); } @@ -3340,7 +3010,7 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t // set the timestamp output buffer for top/bottom/diff query // int32_t fid = pCtx[i].functionId; // if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { - // if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + // if (i > 0) pCtx[i].pTsOutput = pCtx[i-1].pOutput; // } } @@ -3377,7 +3047,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOf if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { - if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i - 1].pOutput; +// if (i > 0) pBInfo->pCtx[i].pTsOutput = pBInfo->pCtx[i - 1].pOutput; } } } @@ -3415,7 +3085,7 @@ void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (isRowEntryInitialized(pResInfo) || pCtx[j].functionId == -1) { + if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || fmIsScalarFunc(pCtx[j].functionId)) { continue; } @@ -3433,27 +3103,6 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } -static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - // if (pRuntimeEnv->pTsBuf) { - // SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - // assert(ret); - // } - - // reverse order time range - SET_REVERSE_SCAN_FLAG(pTableScanInfo); - // setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED); - - switchCtxOrder(pCtx, numOfOutput); - - SWITCH_ORDER(pTableScanInfo->order); - setupQueryRangeForReverseScan(pTableScanInfo); - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; -} - void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t j = 0; j < numOfOutput; ++j) { if (pCtx[j].functionId == -1) { @@ -3602,7 +3251,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pRes } if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; +// if (i > 0) pCtx[i].pTsOutput = pCtx[i - 1].pOutput; } // if (!pResInfo->initialized) { @@ -3630,6 +3279,44 @@ void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, } } +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { + if (pFilterNode == NULL) { + return; + } + + SFilterInfo* filter = NULL; + int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + + SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m1); + + int8_t* rowRes = NULL; + bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); + + SSDataBlock* px = createOneDataBlock(pBlock); + blockDataEnsureCapacity(px, pBlock->info.rows); + + int32_t numOfRow = 0; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + + numOfRow = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } + + colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); + numOfRow += 1; + } + + *pSrc = *pDst; + } + + pBlock->info.rows = numOfRow; +} + void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3855,7 +3542,7 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResI return 0; } -static void toSDatablock(SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, +void toSDatablock(SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); @@ -4442,237 +4129,6 @@ static void doCloseAllTimeWindow(STaskRuntimeEnv* pRuntimeEnv) { } } -static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { - STableScanInfo* pTableScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SSDataBlock* pBlock = &pTableScanInfo->block; - STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; - - *newgroup = false; - - while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { - if (isTaskKilled(pOperator->pTaskInfo)) { - longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); - - // todo opt - // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - // STableQueryInfo** pTableQueryInfo = - // (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - // if (pTableQueryInfo == NULL) { - // break; - // } - // - // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); - // } - - // this function never returns error? - uint32_t status = BLK_DATA_ALL_NEEDED; - int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); - // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pTaskInfo->env, code); - } - - // current block is ignored according to filter result by block statistics data, continue load the next block - if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { - continue; - } - - return pBlock; - } - - return NULL; -} - -static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { - STableScanInfo* pTableScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->dataReader == NULL) { - return NULL; - } - - SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; - *newgroup = false; - - while (pTableScanInfo->current < pTableScanInfo->times) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); - if (p != NULL) { - return p; - } - - if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 /* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { - return NULL; - } else { - break; - } - } - - // do prepare for the next round table scan operation - // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->scanFlag = REPEAT_SCAN; - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = 0; - } - - qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); - } - - SSDataBlock* p = NULL; - // todo refactor - if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - - p = doTableScanImpl(pOperator, newgroup); - } - - return p; -} - -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableScanInfo* pTableScanInfo = pOperator->info; - *newgroup = false; - - STableBlockDistInfo tableBlockDist = {0}; - tableBlockDist.numOfTables = 1; // TODO set the correct number of tables - - int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++numRowSteps; - } - - tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); - - tableBlockDist.maxRows = INT_MIN; - tableBlockDist.minRows = INT_MAX; - - tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); - - SSDataBlock* pBlock = &pTableScanInfo->block; - pBlock->info.rows = 1; - pBlock->info.numOfCols = 1; - -// SBufferWriter bw = tbufInitWriter(NULL, false); -// blockDistInfoToBinary(&tableBlockDist, &bw); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - -// int32_t len = (int32_t) tbufTell(&bw); -// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); -// *(int32_t*) pColInfo->pData = len; -// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); -// -// tbufCloseWriter(&bw); - -// SArray* g = GET_TABLEGROUP(pOperator->, 0); -// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); - - pOperator->status = OP_EXEC_DONE; - return pBlock; -} - -static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { - size_t total = taosArrayGetSize(pInfo->pBlockLists); - - pInfo->validBlockIndex = 0; - for (int32_t i = 0; i < total; ++i) { - SSDataBlock* p = taosArrayGetP(pInfo->pBlockLists, i); - blockDataDestroy(p); - } - taosArrayClear(pInfo->pBlockLists); -} - -static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) { - // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamBlockScanInfo* pInfo = pOperator->info; - - pTaskInfo->code = pOperator->_openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { - size_t total = taosArrayGetSize(pInfo->pBlockLists); - if (pInfo->validBlockIndex >= total) { - doClearBufferedBlocks(pInfo); - return NULL; - } - - int32_t current = pInfo->validBlockIndex++; - return taosArrayGetP(pInfo->pBlockLists, current); - } else { - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - blockDataCleanup(pInfo->pRes); - - while (tqNextDataBlock(pInfo->readerHandle)) { - pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - terrno = pTaskInfo->code; - return NULL; - } - - if (pBlockInfo->rows == 0) { - return NULL; - } - - SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); - - int32_t numOfCols = pInfo->pRes->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - - ASSERT(pColMatchInfo->colId == p->info.colId); - taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p); - } - - if (pInfo->pRes->pDataBlock == NULL) { - // TODO add log - pTaskInfo->code = terrno; - return NULL; - } - - break; - } - - // record the scan action. - pInfo->numOfExec++; - pInfo->numOfRows += pBlockInfo->rows; - - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } -} - int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*)param; if (code == TSDB_CODE_SUCCESS) { @@ -4688,7 +4144,6 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) pSourceDataInfo->status = EX_SOURCE_DATA_READY; tsem_post(&pSourceDataInfo->pEx->ready); - return TSDB_CODE_SUCCESS; } static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { @@ -4759,7 +4214,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf } // TODO if only one or two columnss required, how to extract data? -static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, +int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { blockDataEnsureCapacity(pRes, numOfRows); @@ -5218,490 +4673,6 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, - int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, - SNode* pCondition, SExecTaskInfo* pTaskInfo) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->block.pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData idata = {0}; - taosArrayPush(pInfo->block.pDataBlock, &idata); - } - - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - - pInfo->dataReader = pTsdbReadHandle; - pInfo->times = 1; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; - pInfo->prevGroupId = -1; - pRuntimeEnv->enableGroupData = true; - - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = doTableScanImpl; - - return pOperator; -} - -SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } - - pInfo->dataReader = dataReader; - pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; - infoData.info.bytes = 1024; - infoData.info.colId = 0; - taosArrayPush(pInfo->block.pDataBlock, &infoData); - - pOperator->name = "DataBlockInfoScanOperator"; - // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doBlockInfoScan; - - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; - - _error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; -} - -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { - SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - int32_t numOfOutput = taosArrayGetSize(pColList); - - SArray* pColIds = taosArrayInit(4, sizeof(int16_t)); - for(int32_t i = 0; i < numOfOutput; ++i) { - int16_t* id = taosArrayGet(pColList, i); - taosArrayPush(pColIds, id); - } - - pInfo->pColMatchInfo = pColList; - - // set the extract column id to streamHandle - tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds); - int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); - if (code != 0) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; - } - - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); - if (pInfo->pBlockLists == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; - } - - pInfo->readerHandle = streamReadHandle; - pInfo->pRes = pResBlock; - - pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - -static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { - SOperatorInfo* operator=(SOperatorInfo*) param; - SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; - if (TSDB_CODE_SUCCESS == code) { - pScanResInfo->pRsp = pMsg->pData; - - SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); - } else { - operator->pTaskInfo->code = code; - } - - tsem_post(&pScanResInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { - if (pInfo->pCondition == NULL) { - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; - } - - SFilterInfo* filter = NULL; - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); - - SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; - code = filterSetDataFromSlotId(filter, ¶m1); - - int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); - - SSDataBlock* px = createOneDataBlock(pInfo->pRes); - blockDataEnsureCapacity(px, pInfo->pRes->info.rows); - - // TODO refactor - int32_t numOfRow = 0; - for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { - SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); - - numOfRow = 0; - for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; - } - - colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); - numOfRow += 1; - } - } - - px->info.rows = numOfRow; - pInfo->pRes = px; - - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; -} - -EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { - int32_t code = TSDB_CODE_SUCCESS; - ENodeType nType = nodeType(pNode); - - switch (nType) { - case QUERY_NODE_OPERATOR: { - SOperatorNode* node = (SOperatorNode*)pNode; - - if (OP_TYPE_EQUAL == node->opType) { - *(int32_t*)pContext = 1; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - - return DEAL_RES_IGNORE_CHILD; - } - case QUERY_NODE_COLUMN: { - if (1 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SColumnNode* node = (SColumnNode*)pNode; - if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { - *(int32_t*)pContext = 2; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_CONTINUE; - } - case QUERY_NODE_VALUE: { - if (2 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SValueNode* node = (SValueNode*)pNode; - char* dbName = nodesGetValueFromNode(node); - strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); - *((char*)pContext + varDataLen(dbName)) = 0; - return DEAL_RES_ERROR; // stop walk - } - default: - break; - } - - return DEAL_RES_CONTINUE; -} - -void getDBNameFromCondition(SNode* pCondition, char* dbName) { - if (NULL == pCondition) { - return; - } - - nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); -} - -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { - // build message and send to mnode to fetch the content of system tables. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - // retrieve local table list info from vnode - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); - } - - blockDataCleanup(pInfo->pRes); - - int32_t tableNameSlotId = 1; - SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); - - char* name = NULL; - int32_t numOfRows = 0; - - char n[TSDB_TABLE_NAME_LEN] = {0}; - while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { - STR_TO_VARSTR(n, name); - colDataAppend(pTableNameCol, numOfRows, n, false); - numOfRows += 1; - if (numOfRows >= pInfo->capacity) { - break; - } - - for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { - if (i == tableNameSlotId) { - continue; - } - - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); - int64_t tmp = 0; - char t[10] = {0}; - STR_TO_VARSTR(t, "_"); //TODO - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - colDataAppend(pColInfoData, numOfRows, t, false); - } else { - colDataAppend(pColInfoData, numOfRows, (char*)&tmp, false); - } - } - } - - pInfo->loadInfo.totalRows += numOfRows; - pInfo->pRes->info.rows = numOfRows; - - // pInfo->elapsedTime; - // pInfo->totalBytes; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - int64_t startTs = taosGetTimestampUs(); - - pInfo->req.type = pInfo->type; - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - if (pInfo->showRewrite) { - char dbName[TSDB_DB_NAME_LEN] = {0}; - getDBNameFromCondition(pInfo->pCondition, dbName); - sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); - } - - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadSysTableContentCb; - - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - - if (pTaskInfo->code) { - return NULL; - } - - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; - - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - } - - if (pRsp->numOfRows == 0) { - // qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" - // try next", - // GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, - // pDataInfo->totalRows, pExchangeInfo->totalRows); - return NULL; - } - - SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; - setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, - pOperator->numOfOutput, startTs, NULL, pInfo->scanCols); - - return doFilterResult(pInfo); - } - - return NULL; -} - -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { - SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->accountId = accountId; - pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; - - // TODO remove it - int32_t tableType = 0; - const char* name = tNameGetTableName(pName); - if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_USER; - } else if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MODULE; - } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_QNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_FUNC; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_INDEX; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STREAMS; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TABLE; - } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_VGROUP; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_DIST; - } else { - ASSERT(0); - } - - tNameAssign(&pInfo->name, pName); - pInfo->type = tableType; - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { - pInfo->readHandle = pSysTableReadHandle; - blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); - } else { - tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = epset; - -#if 1 - { // todo refactor - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "DB-META"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char*)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6"; - - pInfo->pTransporter = rpcOpen(&rpcInit); - if (pInfo->pTransporter == NULL) { - return NULL; // todo - } - } -#endif - } - - pOperator->name = "SysTableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->getNextFn = doSysTableScan; - pOperator->closeFn = destroySysTableScannerOperatorInfo; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey); static void cleanupAggSup(SAggSupporter* pAggSup); @@ -6199,7 +5170,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (pBlock == NULL) { break; } - // if (pAggInfo->current != NULL) { // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // } @@ -6478,12 +5448,12 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, pProjectInfo->pPseudoColInfo); + // todo extract method if (pProjectInfo->curOffset < pInfo->pRes->info.rows && pProjectInfo->curOffset > 0) { blockDataTrimFirstNRows(pInfo->pRes, pProjectInfo->curOffset); pProjectInfo->curOffset = 0; - break; } else if (pProjectInfo->curOffset >= pInfo->pRes->info.rows) { pProjectInfo->curOffset -= pInfo->pRes->info.rows; blockDataCleanup(pInfo->pRes); @@ -6973,60 +5943,6 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SGroupbyOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, - pInfo->binfo.rowCellInfoOffset); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - return pInfo->binfo.pRes; - } - - int32_t order = TSDB_ORDER_ASC; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - doHashGroupbyAgg(pOperator, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->binfo.resultRowInfo); - - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - // if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); - // } else { - // updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, - // pInfo->binfo.rowCellInfoOffset); - // } - - blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); - toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, - pInfo->binfo.rowCellInfoOffset); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->binfo.pRes; -} static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, bool* newgroup, SExecTaskInfo* pTaskInfo) { @@ -7197,7 +6113,7 @@ static void cleanupAggSup(SAggSupporter* pAggSup) { destroyDiskbasedBuf(pAggSup->pResultBuf); } -static int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, +int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, int32_t numOfRows, SSDataBlock* pResultBlock, const char* pkey) { pBasicInfo->pCtx = createSqlFunctionCtx_rv(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; @@ -7279,7 +6195,7 @@ _error: return NULL; } -static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { +void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); @@ -7323,14 +6239,6 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(pInfo->p); } -void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - taosMemoryFreeClear(pInfo->keyBuf); - taosArrayDestroy(pInfo->pGroupCols); - taosArrayDestroy(pInfo->pGroupColVals); -} - static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); @@ -7348,24 +6256,6 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pSortInfo); } -static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*)param; - taosHashCleanup(pInfo->pSet); - taosMemoryFreeClear(pInfo->buf); - taosArrayDestroy(pInfo->pDistinctDataInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput) { - SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; - tsem_destroy(&pInfo->ready); - blockDataDestroy(pInfo->pRes); - - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { - metaCloseTbCursor(pInfo->pCur); - } -} - void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { SExchangeInfo* pExInfo = (SExchangeInfo*)param; taosArrayDestroy(pExInfo->pSources); @@ -7416,6 +6306,17 @@ _error: return NULL; } +static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { + SArray* pList = taosArrayInit(4, sizeof(int32_t)); + for(int32_t i = 0; i < numOfCols; ++i) { + if (fmIsPseudoColumnFunc(pCtx[i].functionId)) { + taosArrayPush(pList, &i); + } + } + + return pList; +} + SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); @@ -7427,12 +6328,12 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pInfo->limit = *pLimit; pInfo->curOffset = pLimit->offset; pInfo->binfo.pRes = pResBlock; - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, num, &pInfo->binfo.rowCellInfoOffset); - if (pInfo->binfo.pCtx == NULL) { - goto _error; - } - // initResultRowInfo(&pBInfo->resultRowInfo, 8); - // setFunctionResultOutput(pBInfo, MAIN_SCAN); + + int32_t numOfCols = num; + int32_t numOfRows = 4096; + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); + setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols); pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; @@ -7662,77 +6563,6 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun return pOperator; } -static int32_t initGroupOptrInfo(SGroupbyOperatorInfo* pInfo, SArray* pGroupColList) { - pInfo->pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); - if (pInfo->pGroupColVals == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pGroupColList, i); - pInfo->groupKeyLen += pCol->bytes; - - struct SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); - if (key.pData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush(pInfo->pGroupColVals, &key); - } - - int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; - pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); - - if (pInfo->keyBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, - const STableGroupInfo* pTableGroupInfo) { - SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->pGroupCols = pGroupColList; - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - int32_t code = initGroupOptrInfo(pInfo, pGroupColList); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = hashGroupbyAggregate; - pOperator->closeFn = destroyGroupbyOperatorInfo; - - code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, (int64_t*)fillVal); @@ -7998,146 +6828,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo return pOperator; } -static bool initMultiDistinctInfo(SDistinctOperatorInfo* pInfo, SOperatorInfo* pOperator, SSDataBlock* pBlock) { - if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { - // distinct info already inited - return true; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - // pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); - assert(i < numOfBlock); - for (int j = 0; j < numOfBlock; j++) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); - if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) { - SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; - taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); - } - } - } - pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes); - return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; -} -static void buildMultiDistinctKey(SDistinctOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { - char* p = pInfo->buf; - memset(p, 0, pInfo->totalBytes); - - for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { - SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo*)taosArrayGet(pInfo->pDistinctDataInfo, i); - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); - char* val = ((char*)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; - if (isNull(val, pDistDataInfo->type)) { - p += pDistDataInfo->bytes; - continue; - } - if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { - memcpy(p, varDataVal(val), varDataLen(val)); - p += varDataLen(val); - } else { - memcpy(p, val, pDistDataInfo->bytes); - p += pDistDataInfo->bytes; - } - memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); - p += strlen(MULTI_KEY_DELIM); - } -} - -static SSDataBlock* hashDistinct(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SDistinctOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->pRes; - - pRes->info.rows = 0; - SSDataBlock* pBlock = NULL; - - while (1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { - doSetOperatorCompleted(pOperator); - break; - } - // ensure result output buf - if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { - int32_t newSize = pRes->info.rows + pBlock->info.rows; - for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); - if (tmp == NULL) { - return NULL; - } else { - pResultColInfoData->pData = tmp; - } - } - pInfo->outputCapacity = newSize; - } - - for (int32_t i = 0; i < pBlock->info.rows; i++) { - buildMultiDistinctKey(pInfo, pBlock, i); - if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { - int32_t dummy; - taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); - for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); // src - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist - - char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; - char* start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; - memcpy(start, val, pDistDataInfo->bytes); - } - pRes->info.rows += 1; - } - } - - if (pRes->info.rows >= pInfo->threshold) { - break; - } - } - return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; -} - -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo)); - pInfo->totalBytes = 0; - pInfo->buf = NULL; - pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold - pInfo->outputCapacity = 4096; - pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); - pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - // pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "DistinctOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Distinct; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = hashDistinct; - pOperator->pExpr = pExpr; - pOperator->closeFn = destroyDistinctOperatorInfo; - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; -} static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { int32_t j = 0; @@ -8317,6 +7008,17 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* // pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; // pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); + } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_VALUE) { + pExp->pExpr->nodeType = QUERY_NODE_VALUE; + SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pValNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; + valueNodeToVariant(pValNode, &pExp->base.pParam[0].param); } else { ASSERT(0); } @@ -8360,7 +7062,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pColList, pScanPhyNode->node.pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { @@ -8370,8 +7071,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc); @@ -8428,7 +7128,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pAggNode->pGroupKeys != NULL) { SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); - return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); + return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pTaskInfo, NULL); } else { return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); } @@ -8520,7 +7220,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa continue; } - cond.colList[j].type = pColNode->node.resType.type; + cond.colList[j].type = pColNode->node.resType.type; cond.colList[j].bytes = pColNode->node.resType.bytes; cond.colList[j].colId = pColNode->colId; j += 1; @@ -8883,3 +7583,42 @@ void releaseQueryBuf(size_t numOfTables) { // restore value is not enough buffer available atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } + +int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo **pRes, int32_t *capacity, int32_t *resNum) { + if (*resNum >= *capacity) { + *capacity += 10; + + *pRes = taosMemoryRealloc(*pRes, (*capacity) * sizeof(SExplainExecInfo)); + if (NULL == *pRes) { + qError("malloc %d failed", (*capacity) * (int32_t)sizeof(SExplainExecInfo)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + } + + (*pRes)[*resNum].numOfRows = operatorInfo->resultInfo.totalRows; + (*pRes)[*resNum].startupCost = operatorInfo->cost.openCost; + (*pRes)[*resNum].totalCost = operatorInfo->cost.totalCost; + + if (operatorInfo->getExplainFn) { + int32_t code = (*operatorInfo->getExplainFn)(operatorInfo, &(*pRes)->verboseInfo); + if (code) { + qError("operator getExplainFn failed, error:%s", tstrerror(code)); + return code; + } + } + + ++(*resNum); + + int32_t code = 0; + for (int32_t i = 0; i < operatorInfo->numOfDownstream; ++i) { + code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pRes, capacity, resNum); + if (code) { + taosMemoryFreeClear(*pRes); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c new file mode 100644 index 0000000000..c22b58a0d8 --- /dev/null +++ b/source/libs/executor/src/groupoperator.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "function.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "executorimpl.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" + +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + taosMemoryFreeClear(pInfo->keyBuf); + taosArrayDestroy(pInfo->pGroupCols); + taosArrayDestroy(pInfo->pGroupColVals); +} + +static int32_t initGroupOptrInfo(SGroupbyOperatorInfo* pInfo, SArray* pGroupColList) { + pInfo->pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pGroupColVals == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pGroupColList, i); + pInfo->groupKeyLen += pCol->bytes; + + struct SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pInfo->pGroupColVals, &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); + + if (pInfo->keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, + int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); + + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (pkey->isNull && isNull) { + continue; + } + + if (isNull || pkey->isNull) { + return false; + } + + char* val = colDataGetData(pColInfoData, rowIndex); + + if (IS_VAR_DATA_TYPE(pkey->type)) { + int32_t len = varDataLen(val); + if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { + continue; + } else { + return false; + } + } else { + if (memcmp(pkey->pData, val, pkey->bytes) != 0) { + return false; + } + } + } + + return true; +} + +static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; + } else { + char* val = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + } + } +} + +static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVals) { + ASSERT(pKey != NULL); + size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); + + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); + if (pkey->isNull) { + isNull[i] = 1; + continue; + } + + isNull[i] = 0; + if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; + } + } + + *length = (pStart - (char*)pKey); + return 0; +} + +// assign the group keys or user input constant values if required +static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { + for (int32_t i = 0; i < numOfOutput; ++i) { + if (pCtx[i].functionId == -1) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); + + SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; + if (!colDataIsNull(pColInfoData, totalRows, rowIndex, NULL)) { + char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); + char* data = colDataGetData(pColInfoData, rowIndex); + + // set result exists, todo refactor + memcpy(dest, data, pColInfoData->info.bytes); + pEntryInfo->hasResult = DATA_SET_FLAG; + pEntryInfo->numOfRes = 1; + } + } + } +} + +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SGroupbyOperatorInfo* pInfo = pOperator->info; + + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; + int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); + // if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + // qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); + // return; + // } + + int32_t len = 0; + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (!pInfo->isInit) { + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); + pInfo->isInit = true; + num++; + continue; + } + + bool equal = groupKeyCompare(pInfo, pBlock, j, numOfGroupCols); + if (equal) { + num++; + continue; + } + + /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + int32_t rowIndex = j - num; + doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + + // assign the group keys or user input constant values if required + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); + num = 1; + } + + if (num > 0) { + /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = + setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, + 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + int32_t rowIndex = pBlock->info.rows - num; + doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + } +} + +static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SGroupbyOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->binfo.pRes; + + if (pOperator->status == OP_RES_TO_RETURN) { + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); + if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + doHashGroupbyAgg(pOperator, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->binfo.resultRowInfo); + + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, + &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + // if (!stableQuery) { // finalize include the update of result rows + // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // } else { + // updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, + // pInfo->binfo.rowCellInfoOffset); + // } + + blockDataEnsureCapacity(pRes, pInfo->binfo.capacity); + initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + + while(1) { + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, + pInfo->binfo.rowCellInfoOffset); + doFilter(pInfo->pCondition, pRes); + + bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); + if (!hasRemain) { + pOperator->status = OP_EXEC_DONE; + break; + } + + if (pRes->info.rows > 0) { + break; + } + } + + return pInfo->binfo.pRes; +} + +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo, + const STableGroupInfo* pTableGroupInfo) { + SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pGroupCols = pGroupColList; + pInfo->pCondition = pCondition; + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + int32_t code = initGroupOptrInfo(pInfo, pGroupColList); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + // pOperator->operatorType = OP_Groupby; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = hashGroupbyAggregate; + pOperator->closeFn = destroyGroupbyOperatorInfo; + + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; +} + +#define MULTI_KEY_DELIM "-" + +static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { + SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*)param; + taosHashCleanup(pInfo->pSet); + taosMemoryFreeClear(pInfo->buf); + taosArrayDestroy(pInfo->pDistinctDataInfo); + pInfo->pRes = blockDataDestroy(pInfo->pRes); +} + +static void buildMultiDistinctKey(SDistinctOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { + char* p = pInfo->buf; +// memset(p, 0, pInfo->totalBytes); + + for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { + SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo*)taosArrayGet(pInfo->pDistinctDataInfo, i); + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); + + char* val = ((char*)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; + if (isNull(val, pDistDataInfo->type)) { + p += pDistDataInfo->bytes; + continue; + } + if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { + memcpy(p, varDataVal(val), varDataLen(val)); + p += varDataLen(val); + } else { + memcpy(p, val, pDistDataInfo->bytes); + p += pDistDataInfo->bytes; + } + memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); + p += strlen(MULTI_KEY_DELIM); + } +} + +static bool initMultiDistinctInfo(SDistinctOperatorInfo* pInfo, SOperatorInfo* pOperator) { + for (int i = 0; i < pOperator->numOfOutput; i++) { + // pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; + } +#if 0 + for (int i = 0; i < pOperator->numOfOutput; i++) { + int numOfCols = (int)(taosArrayGetSize(pBlock->pDataBlock)); + assert(i < numOfCols); + + for (int j = 0; j < numOfCols; j++) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); + if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) { + SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; + taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); + } + } + } +#endif + +// pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); +// pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes); + return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; +} + +static SSDataBlock* hashDistinct(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SDistinctOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->pRes; + + pRes->info.rows = 0; + SSDataBlock* pBlock = NULL; + + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + while (1) { + publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pDownstream->getNextFn(pDownstream, newgroup); + publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + // ensure result output buf + if (pRes->info.rows + pBlock->info.rows > pInfo->resInfo.capacity) { + int32_t newSize = pRes->info.rows + pBlock->info.rows; + for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); + +// char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); +// if (tmp == NULL) { +// return NULL; +// } else { +// pResultColInfoData->pData = tmp; +// } + } + pInfo->resInfo.capacity = newSize; + } + + for (int32_t i = 0; i < pBlock->info.rows; i++) { + buildMultiDistinctKey(pInfo, pBlock, i); + if (taosHashGet(pInfo->pSet, pInfo->buf, 0) == NULL) { + taosHashPut(pInfo->pSet, pInfo->buf, 0, NULL, 0); + + for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); // src + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist + + char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; + char* start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; + memcpy(start, val, pDistDataInfo->bytes); + } + + pRes->info.rows += 1; + } + } + + if (pRes->info.rows >= pInfo->resInfo.threshold) { + break; + } + } + + return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; +} + +SOperatorInfo* createDistinctOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { + SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pOperator->resultInfo.capacity = 4096; // todo extract function. + +// pInfo->totalBytes = 0; + pInfo->buf = NULL; + + pInfo->pDistinctDataInfo = taosArrayInit(numOfCols, sizeof(SDistinctDataInfo)); + initMultiDistinctInfo(pInfo, pOperator); + + pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + + pOperator->name = "DistinctOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; +// pOperator->operatorType = DISTINCT; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->getNextFn = hashDistinct; + pOperator->closeFn = destroyDistinctOperatorInfo; + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c new file mode 100644 index 0000000000..f39a18da68 --- /dev/null +++ b/source/libs/executor/src/scanoperator.c @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tglobal.h" +#include "filter.h" +#include "function.h" +#include "os.h" +#include "querynodes.h" +#include "tname.h" +#include "vnode.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "executorimpl.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "tsdb.h" +#include "ttypes.h" + +#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) + + +void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SWITCH_ORDER(pCtx[i].order); + } +} + +static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { +#if 0 + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pCheckInfo); + + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle +// STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); +// pTableKeyInfo->lastKey = pCheckInfo->lastKey; +// +// assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } +#endif +} + +int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { + STaskCostInfo* pCost = &pTaskInfo->cost; + + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; + + pCost->totalCheckedRows += pBlock->info.rows; + pCost->loadBlocks += 1; + + *status = BLK_DATA_ALL_NEEDED; + + SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + if (pCols == NULL) { + return terrno; + } + + int32_t numOfCols = pBlock->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + + ASSERT(pColMatchInfo->colId == p->info.colId); + taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); + } + + doFilter(pTableScanInfo->pFilterNode, pBlock); + return TSDB_CODE_SUCCESS; +} + +static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { + // reverse order time range + SET_REVERSE_SCAN_FLAG(pTableScanInfo); + + switchCtxOrder(pCtx, numOfOutput); + SWITCH_ORDER(pTableScanInfo->order); + setupQueryRangeForReverseScan(pTableScanInfo); + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; +} + +static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { + STableScanInfo* pTableScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SSDataBlock* pBlock = &pTableScanInfo->block; + STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; + + *newgroup = false; + + while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { + if (isTaskKilled(pOperator->pTaskInfo)) { + longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + + pTableScanInfo->numOfBlocks += 1; + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); + + // todo opt + // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { + // STableQueryInfo** pTableQueryInfo = + // (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); + // if (pTableQueryInfo == NULL) { + // break; + // } + // + // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); + // } + + // this function never returns error? + uint32_t status = BLK_DATA_ALL_NEEDED; + int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); + // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pOperator->pTaskInfo->env, code); + } + + // current block is ignored according to filter result by block statistics data, continue load the next block + if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { + continue; + } + + return pBlock; + } + + return NULL; +} + +static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { + STableScanInfo* pTableScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + // The read handle is not initialized yet, since no qualified tables exists + if (pTableScanInfo->dataReader == NULL) { + return NULL; + } + + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; + *newgroup = false; + + while (pTableScanInfo->current < pTableScanInfo->times) { + SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + if (p != NULL) { + return p; + } + + if (++pTableScanInfo->current >= pTableScanInfo->times) { + if (pTableScanInfo->reverseTimes <= 0 /* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { + return NULL; + } else { + break; + } + } + + // do prepare for the next round table scan operation + // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); + // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); + + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTableScanInfo->scanFlag = REPEAT_SCAN; + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = 0; + } + + qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + } + + SSDataBlock* p = NULL; + // todo refactor + if (pTableScanInfo->reverseTimes > 0) { + setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); + // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); + // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); + + qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } + + p = doTableScanImpl(pOperator, newgroup); + } + + return p; +} + +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, + int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, + SNode* pCondition, SExecTaskInfo* pTaskInfo) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->block.pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {0}; + taosArrayPush(pInfo->block.pDataBlock, &idata); + } + + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pTsdbReadHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + pOperator->name = "TableScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; +} + +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + + pInfo->dataReader = pTsdbReadHandle; + pInfo->times = 1; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; + pInfo->prevGroupId = -1; + pRuntimeEnv->enableGroupData = true; + + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->getNextFn = doTableScanImpl; + + return pOperator; +} + +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableScanInfo* pTableScanInfo = pOperator->info; + *newgroup = false; + + STableBlockDistInfo tableBlockDist = {0}; + tableBlockDist.numOfTables = 1; // TODO set the correct number of tables + + int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++numRowSteps; + } + + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; + + tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); + + SSDataBlock* pBlock = &pTableScanInfo->block; + pBlock->info.rows = 1; + pBlock->info.numOfCols = 1; + +// SBufferWriter bw = tbufInitWriter(NULL, false); +// blockDistInfoToBinary(&tableBlockDist, &bw); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + +// int32_t len = (int32_t) tbufTell(&bw); +// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); +// *(int32_t*) pColInfo->pData = len; +// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); +// +// tbufCloseWriter(&bw); + +// SArray* g = GET_TABLEGROUP(pOperator->, 0); +// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); + + pOperator->status = OP_EXEC_DONE; + return pBlock; +} + +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + + pInfo->dataReader = dataReader; + pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.bytes = 1024; + infoData.info.colId = 0; + taosArrayPush(pInfo->block.pDataBlock, &infoData); + + pOperator->name = "DataBlockInfoScanOperator"; + // pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doBlockInfoScan; + + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; +} + +static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { + size_t total = taosArrayGetSize(pInfo->pBlockLists); + + pInfo->validBlockIndex = 0; + for (int32_t i = 0; i < total; ++i) { + SSDataBlock* p = taosArrayGetP(pInfo->pBlockLists, i); + blockDataDestroy(p); + } + taosArrayClear(pInfo->pBlockLists); +} + +static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) { + // NOTE: this operator does never check if current status is done or not + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamBlockScanInfo* pInfo = pOperator->info; + + pTaskInfo->code = pOperator->_openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + size_t total = taosArrayGetSize(pInfo->pBlockLists); + if (pInfo->validBlockIndex >= total) { + doClearBufferedBlocks(pInfo); + return NULL; + } + + int32_t current = pInfo->validBlockIndex++; + return taosArrayGetP(pInfo->pBlockLists, current); + } else { + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + blockDataCleanup(pInfo->pRes); + + while (tqNextDataBlock(pInfo->readerHandle)) { + pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + terrno = pTaskInfo->code; + return NULL; + } + + if (pBlockInfo->rows == 0) { + return NULL; + } + + SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); + + int32_t numOfCols = pInfo->pRes->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + + ASSERT(pColMatchInfo->colId == p->info.colId); + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p); + } + + if (pInfo->pRes->pDataBlock == NULL) { + // TODO add log + pTaskInfo->code = terrno; + return NULL; + } + + break; + } + + // record the scan action. + pInfo->numOfExec++; + pInfo->numOfRows += pBlockInfo->rows; + + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } +} + +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { + SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + int32_t numOfOutput = taosArrayGetSize(pColList); + + SArray* pColIds = taosArrayInit(4, sizeof(int16_t)); + for(int32_t i = 0; i < numOfOutput; ++i) { + int16_t* id = taosArrayGet(pColList, i); + taosArrayPush(pColIds, id); + } + + pInfo->pColMatchInfo = pColList; + + // set the extract column id to streamHandle + tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds); + int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); + if (code != 0) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; + } + + pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + if (pInfo->pBlockLists == NULL) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; + } + + pInfo->readerHandle = streamReadHandle; + pInfo->pRes = pResBlock; + + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doStreamBlockScan; + pOperator->closeFn = operatorDummyCloseFn; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; +} + +static void destroySysScanOperator(void* param, int32_t numOfOutput) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + metaCloseTbCursor(pInfo->pCur); + } +} + +EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode* node = (SOperatorNode*)pNode; + + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t*)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode* node = (SColumnNode*)pNode; + if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { + *(int32_t*)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode* node = (SValueNode*)pNode; + char* dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char*)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_ERROR; // stop walk + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + +void getDBNameFromCondition(SNode* pCondition, char* dbName) { + if (NULL == pCondition) { + return; + } + + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); +} + +static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { + SOperatorInfo* operator=(SOperatorInfo*) param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; + if (TSDB_CODE_SUCCESS == code) { + pScanResInfo->pRsp = pMsg->pData; + + SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; + } + + tsem_post(&pScanResInfo->ready); +} + +static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { + if (pInfo->pCondition == NULL) { + return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; + } + + SFilterInfo* filter = NULL; + int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); + + SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m1); + + int8_t* rowRes = NULL; + bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + + SSDataBlock* px = createOneDataBlock(pInfo->pRes); + blockDataEnsureCapacity(px, pInfo->pRes->info.rows); + + // TODO refactor + int32_t numOfRow = 0; + for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { + SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); + + numOfRow = 0; + for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } + + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); + numOfRow += 1; + } + } + + px->info.rows = numOfRow; + pInfo->pRes = px; + + return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; +} + +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { + // build message and send to mnode to fetch the content of system tables. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + // retrieve local table list info from vnode + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); + } + + blockDataCleanup(pInfo->pRes); + + int32_t tableNameSlotId = 1; + SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); + + char* name = NULL; + int32_t numOfRows = 0; + + char n[TSDB_TABLE_NAME_LEN] = {0}; + while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { + STR_TO_VARSTR(n, name); + colDataAppend(pTableNameCol, numOfRows, n, false); + numOfRows += 1; + if (numOfRows >= pInfo->capacity) { + break; + } + + for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { + if (i == tableNameSlotId) { + continue; + } + + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); + int64_t tmp = 0; + char t[10] = {0}; + STR_TO_VARSTR(t, "_"); //TODO + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + colDataAppend(pColInfoData, numOfRows, t, false); + } else { + colDataAppend(pColInfoData, numOfRows, (char*)&tmp, false); + } + } + } + + pInfo->loadInfo.totalRows += numOfRows; + pInfo->pRes->info.rows = numOfRows; + + // pInfo->elapsedTime; + // pInfo->totalBytes; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { // load the meta from mnode of the given epset + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int64_t startTs = taosGetTimestampUs(); + + pInfo->req.type = pInfo->type; + strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + if (pInfo->showRewrite) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } + + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; + pMsgSendInfo->fp = loadSysTableContentCb; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); + + if (pTaskInfo->code) { + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + } + + if (pRsp->numOfRows == 0) { + // qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" + // try next", + // GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, + // pDataInfo->totalRows, pExchangeInfo->totalRows); + return NULL; + } + + SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; + setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, + pOperator->numOfOutput, startTs, NULL, pInfo->scanCols); + + return doFilterResult(pInfo); + } + + return NULL; +} + +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, + SNode* pCondition, SEpSet epset, SArray* colList, + SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->accountId = accountId; + pInfo->showRewrite = showRewrite; + pInfo->pRes = pResBlock; + pInfo->capacity = 4096; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; + + // TODO remove it + int32_t tableType = 0; + const char* name = tNameGetTableName(pName); + if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_DB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_USER; + } else if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_DNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_MNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_MODULE; + } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_QNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_FUNC; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) { + // tableType = TSDB_MGMT_TABLE_INDEX; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_STB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_STREAMS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_VGROUP; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) { + // tableType = TSDB_MGMT_TABLE_DIST; + } else { + ASSERT(0); + } + + tNameAssign(&pInfo->name, pName); + pInfo->type = tableType; + if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + pInfo->readHandle = pSysTableReadHandle; + blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = epset; + +#if 1 + { // todo refactor + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "DB-META"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = qProcessFetchRsp; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char*)"root"; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6"; + + pInfo->pTransporter = rpcOpen(&rpcInit); + if (pInfo->pTransporter == NULL) { + return NULL; // todo + } + } +#endif + } + + pOperator->name = "SysTableScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->getNextFn = doSysTableScan; + pOperator->closeFn = destroySysScanOperator; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; +} diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index eb6a5dcabe..fe370d93b2 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -37,7 +37,7 @@ typedef struct SHNode { char data[]; } SHNode; -typedef struct SSHashObj { +struct SSHashObj { SHNode **hashList; size_t capacity; // number of slots int64_t size; // number of elements in hash table @@ -45,7 +45,7 @@ typedef struct SSHashObj { _equal_fn_t equalFp; // equal function int32_t keyLen; int32_t dataLen; -} SSHashObj; +}; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { int32_t len = MIN(length, HASH_MAX_CAPACITY); @@ -107,7 +107,7 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pDat return pNewNode; } -void taosHashTableResize(SSHashObj *pHashObj) { +static void taosHashTableResize(SSHashObj *pHashObj) { if (!HASH_NEED_RESIZE(pHashObj)) { return; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index ab7bfc7767..607bd279c1 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -26,31 +26,34 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); void functionFinalize(SqlFunctionCtx *pCtx); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -void countFunction(SqlFunctionCtx *pCtx); +int32_t countFunction(SqlFunctionCtx *pCtx); bool getSumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -void sumFunction(SqlFunctionCtx *pCtx); +int32_t sumFunction(SqlFunctionCtx *pCtx); bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -void minFunction(SqlFunctionCtx* pCtx); -void maxFunction(SqlFunctionCtx *pCtx); +int32_t minFunction(SqlFunctionCtx* pCtx); +int32_t maxFunction(SqlFunctionCtx *pCtx); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -void stddevFunction(SqlFunctionCtx* pCtx); +int32_t stddevFunction(SqlFunctionCtx* pCtx); void stddevFinalize(SqlFunctionCtx* pCtx); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -void percentileFunction(SqlFunctionCtx *pCtx); +int32_t percentileFunction(SqlFunctionCtx *pCtx); +void percentileFinalize(SqlFunctionCtx* pCtx); + +bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); +int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -void firstFunction(SqlFunctionCtx *pCtx); -void lastFunction(SqlFunctionCtx *pCtx); - -void valFunction(SqlFunctionCtx *pCtx); +int32_t firstFunction(SqlFunctionCtx *pCtx); +int32_t lastFunction(SqlFunctionCtx *pCtx); #ifdef __cplusplus } diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 906d4f63fb..0697d309ac 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -52,8 +52,6 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) - typedef struct STwaInfo { int8_t hasResult; // flag to denote has value double dOutput; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0048cc6f29..c1cdcbcb0c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -63,74 +63,74 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = functionFinalize }, { - .name = "stddev", - .type = FUNCTION_TYPE_STDDEV, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getStddevFuncEnv, - .initFunc = stddevFunctionSetup, - .processFunc = stddevFunction, - .finalizeFunc = stddevFinalize + .name = "stddev", + .type = FUNCTION_TYPE_STDDEV, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize }, { - .name = "percentile", - .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "percentile", + .type = FUNCTION_TYPE_PERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getPercentileFuncEnv, + .initFunc = percentileFunctionSetup, + .processFunc = percentileFunction, + .finalizeFunc = percentileFinalize }, { - .name = "apercentile", - .type = FUNCTION_TYPE_APERCENTILE, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "apercentile", + .type = FUNCTION_TYPE_APERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize }, { - .name = "top", - .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "top", + .type = FUNCTION_TYPE_TOP, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize }, { - .name = "bottom", - .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "bottom", + .type = FUNCTION_TYPE_BOTTOM, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize }, { - .name = "spread", - .type = FUNCTION_TYPE_SPREAD, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "spread", + .type = FUNCTION_TYPE_SPREAD, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize }, { - .name = "last_row", - .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC, - .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .name = "last_row", + .type = FUNCTION_TYPE_LAST_ROW, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize }, { .name = "first", @@ -152,6 +152,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = lastFunction, .finalizeFunc = functionFinalize }, + { + .name = "diff", + .type = FUNCTION_TYPE_DIFF, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getDiffFuncEnv, + .initFunc = diffFunctionSetup, + .processFunc = diffFunction, + .finalizeFunc = functionFinalize + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, @@ -377,7 +387,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_ROWTS, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, .checkFunc = stubCheckAndGetResultType, - .getEnvFunc = NULL, + .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, .sprocessFunc = NULL, .finalizeFunc = NULL @@ -459,9 +469,11 @@ const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFun int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { switch(pFunc->funcType) { case FUNCTION_TYPE_WDURATION: - case FUNCTION_TYPE_COUNT: + case FUNCTION_TYPE_COUNT: { pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; break; + } + case FUNCTION_TYPE_SUM: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraType = pParam->node.resType.type; @@ -480,6 +492,8 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType }; break; } + + case FUNCTION_TYPE_DIFF: case FUNCTION_TYPE_FIRST: case FUNCTION_TYPE_LAST: case FUNCTION_TYPE_MIN: @@ -490,10 +504,11 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_QENDTS: + case FUNCTION_TYPE_ROWTS: case FUNCTION_TYPE_QSTARTTS: - case FUNCTION_TYPE_WENDTS: - case FUNCTION_TYPE_WSTARTTS: { + case FUNCTION_TYPE_QENDTS: + case FUNCTION_TYPE_WSTARTTS: + case FUNCTION_TYPE_WENDTS:{ pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; break; } @@ -508,6 +523,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_PERCENTILE: case FUNCTION_TYPE_STDDEV: case FUNCTION_TYPE_SIN: case FUNCTION_TYPE_COS: @@ -531,36 +547,35 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_CONCAT: case FUNCTION_TYPE_CONCAT_WS: { int32_t paraType, paraBytes = 0; + bool typeSet = false; for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); - paraBytes += pParam->node.resType.bytes; - paraType = pParam->node.resType.type; + if (pParam->node.type == QUERY_NODE_COLUMN) { + if (typeSet == false) { + paraType = pParam->node.resType.type; + typeSet = true; + } else { + //columns have to be the same type + if (paraType != pParam->node.resType.type) { + return TSDB_CODE_FAILED; + } + } + paraBytes += pParam->node.resType.bytes; + } + } + + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + if (pParam->node.type == QUERY_NODE_VALUE) { + if (paraType == TSDB_DATA_TYPE_NCHAR) { + paraBytes += pParam->node.resType.bytes * TSDB_NCHAR_SIZE; + } else { + paraBytes += pParam->node.resType.bytes; + } + } } pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; break; - //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; - //int32_t firstParamIndex = 0; - //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { - // firstParamIndex = 1; - // SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); - // sepBytes = pSep->node.resType.type; - //} - //for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { - // SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); - // int32_t paraType = pParam->node.resType.type; - // if (i == firstParamIndex) { - // paraTypeFirst = paraType; - // } - // if (paraType != paraTypeFirst) { - // return TSDB_CODE_FAILED; - // } - // //TODO: for constants also needs numOfRows - // totalBytes += pParam->node.resType.bytes; - //} - ////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. - //totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; - //pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; - //break; } case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: @@ -574,7 +589,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_ROWTS: case FUNCTION_TYPE_TBNAME: { // todo break; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0dc2989f77..fb6abe3c5d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -65,7 +65,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ -void countFunction(SqlFunctionCtx *pCtx) { +int32_t countFunction(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; /* @@ -111,7 +111,7 @@ void countFunction(SqlFunctionCtx *pCtx) { } \ } while (0) -void sumFunction(SqlFunctionCtx *pCtx) { +int32_t sumFunction(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -432,12 +432,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { return numOfElems; } -void minFunction(SqlFunctionCtx *pCtx) { +int32_t minFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); } -void maxFunction(SqlFunctionCtx *pCtx) { +int32_t maxFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 0); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); } @@ -475,12 +475,11 @@ bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) return true; } -void stddevFunction(SqlFunctionCtx* pCtx) { +int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -601,6 +600,7 @@ void stddevFinalize(SqlFunctionCtx* pCtx) { } typedef struct SPercentileInfo { + double result; tMemBucket *pMemBucket; int32_t stage; double minval; @@ -627,19 +627,24 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI return true; } -void percentileFunction(SqlFunctionCtx *pCtx) { +int32_t percentileFunction(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; -#if 0 - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + + SColumnInfoData *pCol = pInput->pData[0]; + int32_t type = pCol->info.type; + + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { pInfo->stage += 1; // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; - return; + return 0; } else { pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); } @@ -647,19 +652,17 @@ void percentileFunction(SqlFunctionCtx *pCtx) { // the first stage, only acquire the min/max value if (pInfo->stage == 0) { - if (pCtx->preAggVals.isSet) { + if (pCtx->input.colDataAggIsSet) { double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); - tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); - } else { - assert(true); + if (IS_SIGNED_NUMERIC_TYPE(type)) { + tmin = (double)GET_INT64_VAL(&pAgg->min); + tmax = (double)GET_INT64_VAL(&pAgg->max); + } else if (IS_FLOAT_TYPE(type)) { + tmin = GET_DOUBLE_VAL(&pAgg->min); + tmax = GET_DOUBLE_VAL(&pAgg->max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + tmin = (double)GET_UINT64_VAL(&pAgg->min); + tmax = (double)GET_UINT64_VAL(&pAgg->max); } if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { @@ -670,17 +673,19 @@ void percentileFunction(SqlFunctionCtx *pCtx) { SET_DOUBLE_VAL(&pInfo->maxval, tmax); } - pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); + pInfo->numOfElems += (pInput->numOfRows - pAgg->numOfNull); } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + // check the valid data one by one + int32_t start = pInput->startRowIndex; + for (int32_t i = start; i < pInput->numOfRows + start; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } + char *data = colDataGetData(pCol, i); + double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, data); - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { SET_DOUBLE_VAL(&pInfo->minval, v); } @@ -693,24 +698,40 @@ void percentileFunction(SqlFunctionCtx *pCtx) { } } - return; + return 0; } // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + int32_t start = pInput->startRowIndex; + for (int32_t i = start; i < pInput->numOfRows + start; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } + char *data = colDataGetData(pCol, i); + notNullElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); } - SET_VAL(pCtx, notNullElems, 1); + SET_VAL(pResInfo, notNullElems, 1); pResInfo->hasResult = DATA_SET_FLAG; -#endif +} +// TODO set the correct parameter. +void percentileFinalize(SqlFunctionCtx* pCtx) { + double v = 50;//pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); + + tMemBucket * pMemBucket = ppInfo->pMemBucket; + if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null + SET_DOUBLE_VAL(&ppInfo->result, getPercentile(pMemBucket, v)); + } + + tMemBucketDestroy(pMemBucket); + functionFinalize(pCtx); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -721,9 +742,9 @@ bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { // TODO fix this // This ordinary first function only handle the data block in ascending order -void firstFunction(SqlFunctionCtx *pCtx) { +int32_t firstFunction(SqlFunctionCtx *pCtx) { if (pCtx->order == TSDB_ORDER_DESC) { - return; + return 0; } int32_t numOfElems = 0; @@ -737,7 +758,7 @@ void firstFunction(SqlFunctionCtx *pCtx) { // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); - return; + return 0; } // Check for the first not null data @@ -764,9 +785,9 @@ void firstFunction(SqlFunctionCtx *pCtx) { SET_VAL(pResInfo, numOfElems, 1); } -void lastFunction(SqlFunctionCtx *pCtx) { +int32_t lastFunction(SqlFunctionCtx *pCtx) { if (pCtx->order != TSDB_ORDER_DESC) { - return; + return 0; } int32_t numOfElems = 0; @@ -775,13 +796,12 @@ void lastFunction(SqlFunctionCtx *pCtx) { char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; // All null data column, return directly. if (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) { ASSERT(pInputCol->hasNull == true); - return; + return 0; } if (pCtx->order == TSDB_ORDER_DESC) { @@ -826,10 +846,242 @@ void lastFunction(SqlFunctionCtx *pCtx) { SET_VAL(pResInfo, numOfElems, 1); } -void valFunction(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); +typedef struct SDiffInfo { + bool hasPrev; + bool includeNull; + bool ignoreNegative; + bool firstOutput; + union { int64_t i64; double d64;} prev; +} SDiffInfo; - SColumnInfoData* pInputCol = pCtx->input.pData[0]; - memcpy(buf, pInputCol->pData, pInputCol->info.bytes); +bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SDiffInfo); + return true; } + +bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { + if (!functionSetup(pCtx, pResInfo)) { + return false; + } + + SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); + pDiffInfo->hasPrev = false; + pDiffInfo->prev.i64 = 0; + pDiffInfo->ignoreNegative = false; // TODO set correct param + pDiffInfo->includeNull = false; + pDiffInfo->firstOutput = false; + return true; +} + +int32_t diffFunction(SqlFunctionCtx *pCtx) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SDiffInfo *pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + bool isFirstBlock = (pDiffInfo->hasPrev == false); + int32_t numOfElems = 0; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); +// int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; + + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + int32_t startOffset = pCtx->offset; + switch (pInputCol->info.type) { + case TSDB_DATA_TYPE_INT: { + SColumnInfoData *pOutput = (SColumnInfoData *)pCtx->pOutput; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + + int32_t pos = startOffset + (isFirstBlock? (numOfElems-1):numOfElems); + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (pDiffInfo->includeNull) { + colDataSetNull_f(pOutput->nullbitmap, pos); + if (tsList != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + + numOfElems += 1; + } + continue; + } + + int32_t v = *(int32_t*) colDataGetData(pInputCol, i); + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); + } + + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + } + + pDiffInfo->prev.i64 = v; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + SColumnInfoData *pOutput = (SColumnInfoData *)pCtx->pOutput; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + int32_t v = 0; + if (pDiffInfo->hasPrev) { + v = *(int64_t*) colDataGetData(pInputCol, i); + int64_t delta = (int64_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (pDiffInfo->ignoreNegative) { + continue; + } + +// *(pOutput++) = delta; +// *pTimestamp = (tsList != NULL)? tsList[i]:0; +// +// pOutput += 1; +// pTimestamp += 1; + } + + pDiffInfo->prev.i64 = v; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } +#if 0 + case TSDB_DATA_TYPE_DOUBLE: { + double *pData = (double *)data; + double *pOutput = (double *)pCtx->pOutput; + + for (; i < pCtx->size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { + continue; + } + if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) { + continue; + } + + if (pDiffInfo->hasPrev) { // initial value is not set yet + SET_DOUBLE_VAL(pOutput, pData[i] - pDiffInfo->d64Prev); // direct previous may be null + *pTimestamp = (tsList != NULL)? tsList[i]:0; + pOutput += 1; + pTimestamp += 1; + } + + pDiffInfo->d64Prev = pData[i]; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *pData = (float *)data; + float *pOutput = (float *)pCtx->pOutput; + + for (; i < pCtx->size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { + continue; + } + if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) { + continue; + } + + if (pDiffInfo->hasPrev) { // initial value is not set yet + *pOutput = (float)(pData[i] - pDiffInfo->d64Prev); // direct previous may be null + *pTimestamp = (tsList != NULL)? tsList[i]:0; + pOutput += 1; + pTimestamp += 1; + } + + pDiffInfo->d64Prev = pData[i]; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *pData = (int16_t *)data; + int16_t *pOutput = (int16_t *)pCtx->pOutput; + + for (; i < pCtx->size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { + continue; + } + if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) { + continue; + } + + if (pDiffInfo->hasPrev) { // initial value is not set yet + *pOutput = (int16_t)(pData[i] - pDiffInfo->i64Prev); // direct previous may be null + *pTimestamp = (tsList != NULL)? tsList[i]:0; + pOutput += 1; + pTimestamp += 1; + } + + pDiffInfo->i64Prev = pData[i]; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: { + int8_t *pData = (int8_t *)data; + int8_t *pOutput = (int8_t *)pCtx->pOutput; + + for (; i < pCtx->size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { + continue; + } + if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) { + continue; + } + + if (pDiffInfo->hasPrev) { // initial value is not set yet + *pOutput = (int8_t)(pData[i] - pDiffInfo->i64Prev); // direct previous may be null + *pTimestamp = (tsList != NULL)? tsList[i]:0; + pOutput += 1; + pTimestamp += 1; + } + + pDiffInfo->i64Prev = pData[i]; + pDiffInfo->hasPrev = true; + numOfElems++; + } + break; + } +#endif + default: + break; +// qError("error input type"); + } + + // initial value is not set yet + if (!pDiffInfo->hasPrev || numOfElems <= 0) { + /* + * 1. current block and blocks before are full of null + * 2. current block may be null value + */ + assert(pCtx->hasNull); + } else { +// for (int t = 0; t < pCtx->tagInfo.numOfTagCols; ++t) { +// SqlFunctionCtx* tagCtx = pCtx->tagInfo.pTagCtxList[t]; +// if (tagCtx->functionId == TSDB_FUNC_TAG_DUMMY) { +// aAggs[TSDB_FUNC_TAGPRJ].xFunction(tagCtx); +// } +// } + + int32_t forwardStep = (isFirstBlock) ? numOfElems - 1 : numOfElems; + return forwardStep; + } +} + diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 4034a0eb0f..c50dea5a9d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsPseudoColumnFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); +} + bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } @@ -112,6 +116,11 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); } +bool fmIsNonstandardSQLFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); +} + + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 9174420ff4..789bfb61ee 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -1902,10 +1902,10 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { } // set the output timestamp of each record. - TSKEY *output = pCtx->ptsOutputBuf; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->timestamp; - } +// TSKEY *output = pCtx->pTsOutput; +// for (int32_t i = 0; i < len; ++i, output += step) { +// *output = tvp[i]->timestamp; +// } // set the corresponding tag data for each record // todo check malloc failure @@ -2687,7 +2687,7 @@ static void deriv_function(SqlFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY *pTimestamp = pCtx->ptsOutputBuf; + TSKEY *pTimestamp = NULL;//pCtx->pTsOutput; TSKEY *tsList = GET_TS_LIST(pCtx); double *pOutput = (double *)pCtx->pOutput; @@ -2867,7 +2867,7 @@ static void deriv_function(SqlFunctionCtx *pCtx) { } else { \ *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i)); \ *(type *)(&(ctx)->param[1].i) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ + *(int64_t *)(ctx)->pTsOutput = GET_TS_DATA(ctx, index); \ } \ } while (0); @@ -2881,7 +2881,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* pTimestamp = NULL;//pCtx->pTsOutput; TSKEY* tsList = GET_TS_LIST(pCtx); switch (pCtx->inputType) { diff --git a/source/libs/nodes/inc/nodesUtil.h b/source/libs/nodes/inc/nodesUtil.h index 976044c16f..c6233ba980 100644 --- a/source/libs/nodes/inc/nodesUtil.h +++ b/source/libs/nodes/inc/nodesUtil.h @@ -31,6 +31,7 @@ extern "C" { #define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + #ifdef __cplusplus } #endif diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f8bc99b975..12fe645473 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); - COPY_SCALAR_FIELD(isDuration); - COPY_SCALAR_FIELD(translate); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - COPY_SCALAR_FIELD(datum.b); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - COPY_SCALAR_FIELD(datum.i); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - COPY_SCALAR_FIELD(datum.u); - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - COPY_SCALAR_FIELD(datum.d); - break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -306,6 +283,12 @@ static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { return (SNode*)pDst; } +static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_LIST_FIELD(pPartitionKeys); + return (SNode*)pDst; +} + static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); @@ -390,6 +373,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return logicPartitionCopy((const SPartitionLogicNode*)pNode, (SPartitionLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 5337965164..6cb1955307 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -198,6 +198,10 @@ const char* nodesNodeName(ENodeType type) { return "PhysiInterval"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return "PhysiSessionWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + return "PhysiStateWindow"; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return "PhysiPartition"; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -1147,6 +1151,61 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkStateWindowPhysiPlanStateKey = "StateKey"; + +static int32_t physiStateWindowNodeToJson(const void* pObj, SJson* pJson) { + const SStateWinodwPhysiNode* pNode = (const SStateWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStateWindowPhysiPlanStateKey, nodeToJson, pNode->pStateKey); + } + + return code; +} + +static int32_t jsonToPhysiStateWindowNode(const SJson* pJson, void* pObj) { + SStateWinodwPhysiNode* pNode = (SStateWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStateWindowPhysiPlanStateKey, &pNode->pStateKey); + } + + return code; +} + +static const char* jkPartitionPhysiPlanExprs = "Exprs"; +static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys"; + +static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) { + const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkPartitionPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkPartitionPhysiPlanPartitionKeys, pNode->pPartitionKeys); + } + + return code; +} + +static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) { + SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkPartitionPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkPartitionPhysiPlanPartitionKeys, &pNode->pPartitionKeys); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -1491,6 +1550,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkValueGenByCalc = "GenByCalc"; static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; @@ -1544,6 +1604,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -1614,6 +1677,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -2413,6 +2479,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return physiSessionWindowNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + return physiStateWindowNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return physiPartitionNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -2505,6 +2575,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return jsonToPhysiSessionWindowNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + return jsonToPhysiStateWindowNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index bbd0473edd..e59a76ca3c 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -78,7 +78,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); break; case QUERY_NODE_STATE_WINDOW: - res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); + res = walkNode(((SStateWindowNode*)pNode)->pExpr, order, walker, pContext); break; case QUERY_NODE_SESSION_WINDOW: { SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; @@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { +void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } @@ -212,7 +212,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext); break; case QUERY_NODE_STATE_WINDOW: - res = rewriteNode(&(((SStateWindowNode*)pNode)->pCol), order, rewriter, pContext); + res = rewriteNode(&(((SStateWindowNode*)pNode)->pExpr), order, rewriter, pContext); break; case QUERY_NODE_SESSION_WINDOW: res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext); @@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe return DEAL_RES_CONTINUE; } -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); } -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); } @@ -290,21 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa switch (clause) { case SQL_CLAUSE_FROM: - nodesWalkNode(pSelect->pFromTable, walker, pContext); - nodesWalkNode(pSelect->pWhere, walker, pContext); + nodesWalkExpr(pSelect->pFromTable, walker, pContext); + nodesWalkExpr(pSelect->pWhere, walker, pContext); case SQL_CLAUSE_WHERE: - nodesWalkList(pSelect->pPartitionByList, walker, pContext); + nodesWalkExprs(pSelect->pPartitionByList, walker, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesWalkNode(pSelect->pWindow, walker, pContext); + nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: - nodesWalkList(pSelect->pGroupByList, walker, pContext); + nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: - nodesWalkNode(pSelect->pHaving, walker, pContext); + nodesWalkExpr(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pOrderByList, walker, pContext); - case SQL_CLAUSE_ORDER_BY: - nodesWalkList(pSelect->pProjectionList, walker, pContext); + nodesWalkExprs(pSelect->pProjectionList, walker, pContext); case SQL_CLAUSE_SELECT: + nodesWalkExprs(pSelect->pOrderByList, walker, pContext); default: break; } @@ -319,20 +318,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit switch (clause) { case SQL_CLAUSE_FROM: - nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext); - nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext); case SQL_CLAUSE_WHERE: - nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: - nodesRewriteList(pSelect->pGroupByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: - nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); case SQL_CLAUSE_HAVING: - nodesRewriteList(pSelect->pProjectionList, rewriter, pContext); + nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); case SQL_CLAUSE_SELECT: - nodesRewriteList(pSelect->pOrderByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 90f06c6c9c..eaee6ef4da 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -169,6 +169,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SWindowLogicNode)); case QUERY_NODE_LOGIC_PLAN_SORT: return makeNode(type, sizeof(SSortLogicNode)); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return makeNode(type, sizeof(SPartitionLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: @@ -197,6 +199,10 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return makeNode(type, sizeof(SSessionWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + return makeNode(type, sizeof(SStateWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return makeNode(type, sizeof(SPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -302,7 +308,7 @@ void nodesDestroyNode(SNodeptr pNode) { case QUERY_NODE_LIMIT: // no pointer field break; case QUERY_NODE_STATE_WINDOW: - nodesDestroyNode(((SStateWindowNode*)pNode)->pCol); + nodesDestroyNode(((SStateWindowNode*)pNode)->pExpr); break; case QUERY_NODE_SESSION_WINDOW: { SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; @@ -1041,3 +1047,96 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_SUCCESS; } + + +char *getFillModeString(EFillMode mode) { + switch (mode) { + case FILL_MODE_NONE: + return "none"; + case FILL_MODE_VALUE: + return "value"; + case FILL_MODE_PREV: + return "prev"; + case FILL_MODE_NULL: + return "null"; + case FILL_MODE_LINEAR: + return "linear"; + case FILL_MODE_NEXT: + return "next"; + default: + return "unknown"; + } +} + +char *nodesGetNameFromColumnNode(SNode *pNode) { + if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) { + return "NULL"; + } + + return ((SColumnNode *)pNode)->colName; +} + +int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { + if (NULL == pSlots || pSlots->length <= 0) { + return 0; + } + + SNode* pNode = NULL; + int32_t num = 0; + FOREACH(pNode, pSlots) { + if (QUERY_NODE_SLOT_DESC != pNode->type) { + continue; + } + + SSlotDescNode *descNode = (SSlotDescNode *)pNode; + if (descNode->output) { + ++num; + } + } + + return num; +} + + +void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { + pVal->nType = pNode->node.resType.type; + pVal->nLen = pNode->node.resType.bytes; + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + pVal->i = pNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + pVal->i = pNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + pVal->u = pNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + pVal->d = pNode->datum.d; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + pVal->pz = pNode->datum.p; + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } +} + + + diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp index fd4a9e8c20..fdfb9522db 100644 --- a/source/libs/nodes/test/nodesTest.cpp +++ b/source/libs/nodes/test/nodesTest.cpp @@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) { EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); EDealRes res = DEAL_RES_CONTINUE; - nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res); EXPECT_EQ(res, DEAL_RES_CONTINUE); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5d02868657..c3157480a9 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function transport qcom + PRIVATE os util nodes catalog function scalar transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 1fce11acdf..2fe325f1c5 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -103,7 +103,7 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); -SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 9bad3e9eb9..ea01caf9d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -23,9 +23,10 @@ extern "C" { #include "parser.h" int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 4ae33e5c5c..c30a75aa66 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -716,7 +716,7 @@ partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). twindow_clause_opt(A) ::= . { A = NULL; } twindow_clause_opt(A) ::= SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } -twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP expression(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } twindow_clause_opt(A) ::= diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 18a0ae8cfa..ebcecfe3cb 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -243,27 +243,27 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } - char val[20] = {0}; - int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val)); - char* pStart = val; - char* pEnd = val + len; - int32_t sepOrder = 1; - while (1) { - char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':'); - SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { - pCxt->valid = false; - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); - return pOptions; + if (pVal->n > 2) { + char* pStart = pVal->z + 1; + char* pEnd = pVal->z + pVal->n - 1; + int32_t sepOrder = 1; + while (1) { + char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':'); + SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { + pCxt->valid = false; + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); + return pOptions; + } + if (NULL == pPos) { + break; + } + pStart = pPos + 1; } - if (NULL == pPos) { - break; - } - pStart = pPos + 1; } - if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val); + if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z); pCxt->valid = false; } @@ -730,10 +730,10 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap return (SNode*)session; } -SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); CHECK_OUT_OF_MEM(state); - state->pCol = pCol; + state->pExpr = pExpr; return (SNode*)state; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5c38ccaff8..76af6b7ac7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*); extern void ParseFree(void*, FFree); extern void ParseTrace(FILE*, char*); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c new file mode 100644 index 0000000000..46cdcc4243 --- /dev/null +++ b/source/libs/parser/src/parCalcConst.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "functionMgt.h" +#include "parInt.h" +#include "scalar.h" + +typedef struct SCalcConstContext { + int32_t code; +} SCalcConstContext; + +static int32_t calcConstQuery(SNode* pStmt); + +static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { + SNode* pNew = NULL; + pCxt->code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS != pCxt->code) { + return DEAL_RES_ERROR; + } + ((SValueNode*)pNew)->genByCalc = true; + ((SValueNode*)pNew)->translate = true; + *pNode = pNew; + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { + SOperatorNode* pOp = *pNode; + if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); + } + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { + SFunctionNode* pFunc = *pNode; + if (fmIsPseudoColumnFunc(pFunc->funcId)) { + return DEAL_RES_CONTINUE; + } + SNode* pParam = NULL; + FOREACH(pParam, pFunc->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) { + SLogicConditionNode* pCond = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pCond->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { + SCalcConstContext* pCxt = pContext; + pCxt->code = calcConstQuery((*pNode)->pSubquery); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes calcConst(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + return calcConstOperator((SOperatorNode**)pNode, pContext); + case QUERY_NODE_FUNCTION: + return calcConstFunction((SFunctionNode**)pNode, pContext); + case QUERY_NODE_LOGIC_CONDITION: + return calcConstLogicCond((SLogicConditionNode**)pNode, pContext); + case QUERY_NODE_TEMP_TABLE: + return calcConstSubquery((STempTableNode**)pNode, pContext); + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t calcConstSelect(SSelectStmt* pSelect) { + SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; + nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); + } + return cxt.code; +} + +static int32_t calcConstQuery(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return calcConstSelect((SSelectStmt*)pStmt); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { + return calcConstQuery(pQuery->pRoot); +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 773060beab..97fce407b6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { - va_list vArgList; - va_start(vArgList, errCode); - generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList); - va_end(vArgList); - pCxt->errCode = errCode; - return DEAL_RES_ERROR; -} +#define generateDealNodeErrMsg(pCxt, code, ...) \ + ({ \ + generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ + pCxt->errCode = code; \ + DEAL_RES_ERROR; \ + }) static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -440,6 +438,13 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (nodesIsUnaryOp(pOp)) { + if (OP_TYPE_MINUS == pOp->opType) { + if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } return DEAL_RES_CONTINUE; } SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; @@ -510,12 +515,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt); return pCxt->errCode; } static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { - nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt); return pCxt->errCode; } @@ -570,7 +575,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -578,7 +583,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -605,9 +610,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; - nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (cxt.existAggFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); @@ -938,7 +943,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe SNode* pNode = NULL; int32_t index = 0; FOREACH(pNode, pRetentions) { - if (0 == index % 2) { + if (0 == ((index++) & 1)) { pFreq = (SValueNode*)pNode; } else { pKeep = (SValueNode*)pNode; @@ -951,6 +956,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe taosArrayPush(pReq->pRetensions, &retention); } } + pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); } return TSDB_CODE_SUCCESS; } @@ -2336,46 +2342,6 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, } } -static void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { - pVal->nType = pNode->node.resType.type; - pVal->nLen = pNode->node.resType.bytes; - switch (pNode->node.resType.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - pVal->i = pNode->datum.b; - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - pVal->i = pNode->datum.i; - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - pVal->u = pNode->datum.u; - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - pVal->d = pNode->datum.d; - break; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: - pVal->pz = pNode->datum.p; - break; - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - // todo - default: - break; - } -} - static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, SKVRowBuilder* pBuilder) { if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; @@ -2635,7 +2601,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = { .pParseCxt = pParseCxt, .errCode = TSDB_CODE_SUCCESS, diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d410aa2e17..ebe76cc129 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) { } static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { - int32_t code = doParse(pCxt, pQuery); + int32_t code = parse(pCxt, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = doTranslate(pCxt, *pQuery); + code = translate(pCxt, *pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = calculateConstant(pCxt, *pQuery); } return code; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 86e564adab..30c85e28fe 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -209,153 +209,155 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1448) +#define YY_ACTTAB_COUNT (1461) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1250, 441, 441, 265, 441, 1246, 1413, 72, 304, 306, - /* 10 */ 72, 111, 30, 28, 348, 409, 252, 354, 20, 1263, - /* 20 */ 261, 1153, 990, 1261, 1261, 269, 1261, 1429, 31, 29, - /* 30 */ 27, 26, 25, 1399, 412, 441, 233, 98, 988, 117, - /* 40 */ 409, 305, 1399, 1014, 427, 1395, 1401, 11, 1386, 1429, - /* 50 */ 1301, 30, 28, 1094, 1395, 1401, 425, 1261, 799, 261, + /* 0 */ 1250, 441, 441, 266, 441, 1246, 1413, 72, 305, 307, + /* 10 */ 72, 111, 31, 29, 349, 409, 253, 355, 21, 1263, + /* 20 */ 262, 1153, 990, 1261, 1261, 270, 1261, 1429, 32, 30, + /* 30 */ 28, 27, 26, 1399, 412, 441, 234, 98, 988, 117, + /* 40 */ 409, 306, 1399, 1014, 427, 1395, 1401, 11, 1386, 1429, + /* 50 */ 1301, 31, 29, 1094, 1395, 1401, 425, 1261, 799, 262, /* 60 */ 798, 990, 98, 409, 69, 1414, 1415, 1418, 1462, 1, - /* 70 */ 96, 276, 254, 1458, 120, 1413, 1308, 988, 800, 411, - /* 80 */ 121, 1469, 1470, 1308, 1474, 98, 11, 1340, 399, 251, - /* 90 */ 30, 28, 507, 1490, 1306, 96, 1429, 1252, 261, 325, + /* 70 */ 96, 277, 255, 1458, 120, 1413, 1308, 988, 800, 411, + /* 80 */ 121, 1469, 1470, 1308, 1474, 98, 11, 1340, 402, 252, + /* 90 */ 31, 29, 507, 1490, 1306, 96, 1429, 1252, 262, 326, /* 100 */ 990, 440, 1523, 425, 989, 122, 1469, 1470, 1, 1474, - /* 110 */ 22, 24, 191, 427, 1399, 125, 988, 1386, 96, 1521, - /* 120 */ 31, 29, 27, 26, 25, 11, 1395, 1402, 123, 1469, + /* 110 */ 23, 25, 191, 427, 1399, 125, 988, 1386, 96, 1521, + /* 120 */ 32, 30, 28, 27, 26, 11, 1395, 1402, 123, 1469, /* 130 */ 1470, 507, 1474, 70, 1414, 1415, 1418, 1462, 991, 1386, - /* 140 */ 440, 1461, 1458, 989, 135, 134, 1413, 1, 31, 29, - /* 150 */ 27, 26, 25, 186, 65, 994, 995, 1038, 1039, 1040, - /* 160 */ 1041, 1042, 1043, 1044, 1045, 99, 232, 1429, 1010, 1406, - /* 170 */ 507, 299, 1253, 298, 425, 318, 126, 991, 330, 1010, - /* 180 */ 428, 1404, 989, 264, 427, 1429, 1348, 331, 1386, 12, - /* 190 */ 1413, 398, 425, 131, 994, 995, 1038, 1039, 1040, 1041, + /* 140 */ 440, 1461, 1458, 989, 135, 134, 1413, 1, 32, 30, + /* 150 */ 28, 27, 26, 186, 65, 994, 995, 1038, 1039, 1040, + /* 160 */ 1041, 1042, 1043, 1044, 1045, 99, 233, 1429, 1010, 1406, + /* 170 */ 507, 300, 1253, 299, 425, 319, 126, 991, 331, 1010, + /* 180 */ 428, 1404, 989, 265, 427, 273, 1348, 332, 1386, 12, + /* 190 */ 1413, 126, 1339, 1341, 994, 995, 1038, 1039, 1040, 1041, /* 200 */ 1042, 1043, 1044, 1045, 69, 1414, 1415, 1418, 1462, 441, - /* 210 */ 126, 1429, 254, 1458, 1535, 311, 991, 47, 425, 1308, - /* 220 */ 46, 441, 1336, 1496, 402, 266, 440, 312, 427, 133, + /* 210 */ 126, 1429, 255, 1458, 1535, 312, 991, 1239, 425, 1308, + /* 220 */ 159, 441, 1336, 1496, 352, 267, 440, 313, 427, 133, /* 230 */ 1306, 1261, 1386, 994, 995, 1038, 1039, 1040, 1041, 1042, - /* 240 */ 1043, 1044, 1045, 1261, 404, 400, 30, 28, 70, 1414, - /* 250 */ 1415, 1418, 1462, 64, 261, 329, 990, 1459, 324, 323, - /* 260 */ 322, 321, 320, 60, 317, 316, 315, 314, 310, 309, - /* 270 */ 308, 307, 988, 84, 1118, 12, 83, 82, 81, 80, - /* 280 */ 79, 78, 77, 76, 75, 30, 28, 426, 1156, 361, - /* 290 */ 111, 356, 300, 261, 360, 990, 126, 160, 1264, 357, - /* 300 */ 355, 126, 358, 7, 395, 1116, 1117, 1119, 1120, 84, - /* 310 */ 6, 988, 83, 82, 81, 80, 79, 78, 77, 76, - /* 320 */ 75, 441, 1308, 1523, 30, 28, 507, 338, 273, 503, - /* 330 */ 502, 441, 261, 1306, 990, 126, 125, 1258, 989, 114, - /* 340 */ 1521, 1225, 7, 1261, 272, 31, 29, 27, 26, 25, - /* 350 */ 988, 1339, 1341, 1261, 874, 464, 463, 462, 878, 461, + /* 240 */ 1043, 1044, 1045, 1261, 1237, 354, 31, 29, 70, 1414, + /* 250 */ 1415, 1418, 1462, 340, 262, 330, 990, 1459, 325, 324, + /* 260 */ 323, 322, 321, 340, 318, 317, 316, 315, 311, 310, + /* 270 */ 309, 308, 988, 84, 1118, 12, 83, 82, 81, 80, + /* 280 */ 79, 78, 77, 76, 75, 31, 29, 426, 1156, 362, + /* 290 */ 476, 357, 301, 262, 361, 990, 126, 160, 213, 358, + /* 300 */ 356, 1291, 359, 7, 396, 1116, 1117, 1119, 1120, 84, + /* 310 */ 1178, 988, 83, 82, 81, 80, 79, 78, 77, 76, + /* 320 */ 75, 441, 1308, 1523, 31, 29, 507, 339, 274, 399, + /* 330 */ 476, 441, 262, 1306, 990, 126, 125, 1258, 989, 114, + /* 340 */ 1521, 1225, 7, 1261, 1012, 32, 30, 28, 27, 26, + /* 350 */ 988, 131, 1386, 1261, 874, 464, 463, 462, 878, 461, /* 360 */ 880, 881, 460, 883, 457, 507, 889, 454, 891, 892, - /* 370 */ 451, 448, 991, 27, 26, 25, 113, 989, 1167, 213, - /* 380 */ 1413, 7, 1291, 1375, 31, 29, 27, 26, 25, 994, + /* 370 */ 451, 448, 991, 1177, 113, 48, 1167, 989, 47, 403, + /* 380 */ 1413, 7, 404, 400, 32, 30, 28, 27, 26, 994, /* 390 */ 995, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 441, - /* 400 */ 1523, 1429, 1201, 1026, 507, 438, 339, 384, 425, 1239, - /* 410 */ 126, 991, 1178, 1522, 428, 1012, 989, 1521, 427, 283, - /* 420 */ 1349, 1261, 1386, 31, 29, 27, 26, 25, 994, 995, + /* 400 */ 1523, 1429, 1201, 1108, 507, 438, 111, 385, 425, 1013, + /* 410 */ 126, 991, 1176, 1522, 1264, 1386, 989, 1521, 427, 503, + /* 420 */ 502, 1261, 1386, 32, 30, 28, 27, 26, 994, 995, /* 430 */ 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 115, 1414, - /* 440 */ 1415, 1418, 1089, 31, 29, 27, 26, 25, 385, 271, - /* 450 */ 991, 361, 1015, 356, 1386, 339, 360, 111, 235, 160, - /* 460 */ 245, 357, 355, 1177, 358, 1263, 1413, 994, 995, 1038, - /* 470 */ 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1537, 235, 1523, - /* 480 */ 30, 28, 159, 9, 8, 403, 351, 1429, 261, 1057, - /* 490 */ 990, 1155, 125, 49, 425, 291, 1521, 246, 375, 244, - /* 500 */ 243, 1026, 350, 1148, 427, 1386, 988, 353, 1386, 1057, - /* 510 */ 293, 1257, 1176, 413, 1175, 93, 92, 91, 90, 89, + /* 440 */ 1415, 1418, 6, 32, 30, 28, 27, 26, 386, 272, + /* 450 */ 991, 362, 1015, 357, 1386, 1248, 361, 111, 236, 160, + /* 460 */ 246, 358, 356, 1175, 359, 1263, 1413, 994, 995, 1038, + /* 470 */ 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1537, 236, 1523, + /* 480 */ 31, 29, 159, 9, 8, 1011, 352, 1429, 262, 1057, + /* 490 */ 990, 1155, 125, 1059, 425, 292, 1521, 247, 376, 245, + /* 500 */ 244, 1026, 351, 1148, 427, 1386, 988, 354, 1386, 1057, + /* 510 */ 294, 1063, 1174, 413, 1173, 93, 92, 91, 90, 89, /* 520 */ 88, 87, 86, 85, 68, 1414, 1415, 1418, 1462, 1523, - /* 530 */ 49, 274, 234, 1458, 159, 1058, 416, 1, 351, 111, - /* 540 */ 95, 1093, 125, 1174, 1523, 441, 1521, 1263, 1256, 1013, - /* 550 */ 1059, 439, 1070, 1062, 1386, 1058, 1386, 125, 476, 353, - /* 560 */ 507, 1521, 172, 1308, 441, 441, 1173, 1261, 1063, 1147, - /* 570 */ 205, 275, 989, 1062, 1307, 1172, 23, 259, 1052, 1053, - /* 580 */ 1054, 1055, 1056, 1060, 1061, 1386, 1261, 1261, 1011, 1476, - /* 590 */ 1194, 21, 1171, 1170, 1169, 1166, 23, 259, 1052, 1053, - /* 600 */ 1054, 1055, 1056, 1060, 1061, 1198, 991, 1473, 1386, 1101, - /* 610 */ 1165, 147, 362, 1248, 119, 1012, 112, 1386, 1481, 1089, - /* 620 */ 344, 219, 146, 994, 995, 1038, 1039, 1040, 1041, 1042, - /* 630 */ 1043, 1044, 1045, 217, 1386, 1386, 1386, 1386, 1164, 1163, - /* 640 */ 1162, 1161, 1160, 136, 1159, 1158, 50, 9, 8, 144, - /* 650 */ 1237, 1413, 1386, 420, 495, 494, 493, 492, 491, 490, + /* 530 */ 50, 275, 235, 1458, 22, 1058, 416, 1, 1308, 111, + /* 540 */ 95, 50, 125, 1172, 1523, 441, 1521, 1263, 1256, 1307, + /* 550 */ 798, 439, 1070, 1062, 1386, 1058, 1386, 125, 467, 1257, + /* 560 */ 507, 1521, 172, 417, 441, 441, 347, 1261, 1171, 1147, + /* 570 */ 205, 276, 989, 1062, 1089, 1170, 24, 260, 1052, 1053, + /* 580 */ 1054, 1055, 1056, 1060, 1061, 1386, 1261, 1261, 1244, 32, + /* 590 */ 30, 28, 27, 26, 1169, 1166, 24, 260, 1052, 1053, + /* 600 */ 1054, 1055, 1056, 1060, 1061, 1198, 991, 28, 27, 26, + /* 610 */ 1386, 147, 428, 1101, 119, 1093, 112, 1386, 1349, 1012, + /* 620 */ 345, 219, 146, 994, 995, 1038, 1039, 1040, 1041, 1042, + /* 630 */ 1043, 1044, 1045, 217, 1165, 1476, 1386, 1386, 1164, 1163, + /* 640 */ 1162, 1161, 1160, 136, 1159, 1158, 51, 1026, 420, 144, + /* 650 */ 998, 1413, 1375, 1473, 495, 494, 493, 492, 491, 490, /* 660 */ 489, 488, 207, 485, 484, 483, 482, 481, 480, 479, - /* 670 */ 478, 477, 1429, 1476, 1476, 486, 58, 467, 294, 425, - /* 680 */ 1386, 1386, 1386, 1386, 1386, 798, 1386, 1386, 417, 427, - /* 690 */ 424, 1472, 1471, 1386, 1254, 152, 476, 154, 150, 1413, - /* 700 */ 153, 346, 373, 143, 67, 138, 998, 140, 383, 69, - /* 710 */ 1414, 1415, 1418, 1462, 1092, 371, 1244, 254, 1458, 1535, - /* 720 */ 1429, 1150, 1151, 165, 137, 415, 466, 412, 1519, 45, - /* 730 */ 44, 303, 156, 130, 1189, 155, 104, 427, 297, 387, - /* 740 */ 158, 1386, 1413, 157, 1168, 1226, 241, 1187, 289, 396, - /* 750 */ 285, 281, 127, 188, 1302, 345, 364, 69, 1414, 1415, - /* 760 */ 1418, 1462, 181, 1429, 1492, 254, 1458, 120, 43, 367, - /* 770 */ 425, 1115, 1001, 175, 126, 997, 177, 1430, 1413, 187, - /* 780 */ 427, 410, 421, 190, 1386, 391, 1489, 32, 32, 32, - /* 790 */ 1064, 1023, 957, 194, 2, 1010, 196, 278, 1049, 1429, - /* 800 */ 69, 1414, 1415, 1418, 1462, 1413, 425, 282, 254, 1458, - /* 810 */ 1535, 101, 240, 242, 433, 418, 427, 841, 102, 1480, - /* 820 */ 1386, 202, 966, 211, 313, 413, 1429, 409, 510, 1338, - /* 830 */ 132, 104, 319, 425, 867, 327, 224, 1414, 1415, 1418, - /* 840 */ 326, 1000, 210, 427, 328, 94, 332, 1386, 1019, 98, - /* 850 */ 333, 499, 43, 209, 446, 862, 1523, 895, 102, 103, - /* 860 */ 1413, 899, 905, 70, 1414, 1415, 1418, 1462, 413, 125, - /* 870 */ 334, 423, 1458, 1521, 1018, 335, 139, 66, 1017, 336, - /* 880 */ 203, 1429, 96, 104, 102, 337, 904, 105, 425, 142, - /* 890 */ 48, 145, 184, 1469, 408, 340, 407, 1016, 427, 1523, - /* 900 */ 347, 349, 1386, 74, 1413, 1251, 376, 149, 1247, 1413, - /* 910 */ 437, 377, 125, 151, 106, 107, 1521, 1249, 115, 1414, - /* 920 */ 1415, 1418, 1245, 108, 109, 1429, 352, 359, 378, 250, - /* 930 */ 1429, 1015, 425, 382, 167, 390, 388, 425, 168, 379, - /* 940 */ 170, 397, 427, 1503, 431, 386, 1386, 427, 1493, 260, - /* 950 */ 389, 1386, 995, 974, 392, 164, 414, 1536, 1502, 5, - /* 960 */ 1483, 173, 228, 1414, 1415, 1418, 1413, 228, 1414, 1415, - /* 970 */ 1418, 1413, 406, 394, 393, 176, 253, 4, 401, 1089, - /* 980 */ 97, 1014, 33, 183, 180, 422, 255, 1429, 118, 1477, - /* 990 */ 419, 429, 1429, 182, 425, 17, 1347, 1346, 430, 425, - /* 1000 */ 263, 434, 1444, 435, 427, 198, 1538, 1413, 1386, 427, - /* 1010 */ 200, 57, 436, 1386, 1520, 1413, 258, 59, 1262, 212, - /* 1020 */ 473, 189, 444, 214, 227, 1414, 1415, 1418, 1429, 228, - /* 1030 */ 1414, 1415, 1418, 487, 208, 425, 1429, 506, 216, 220, - /* 1040 */ 221, 1413, 39, 425, 218, 427, 1380, 1379, 277, 1386, - /* 1050 */ 1413, 279, 262, 427, 1376, 280, 405, 1386, 984, 985, - /* 1060 */ 128, 284, 1429, 1374, 286, 228, 1414, 1415, 1418, 425, - /* 1070 */ 287, 1429, 288, 226, 1414, 1415, 1418, 1413, 425, 427, - /* 1080 */ 1373, 290, 1372, 1386, 1363, 1413, 292, 129, 427, 295, - /* 1090 */ 296, 969, 1386, 968, 1357, 1356, 302, 1355, 1429, 229, - /* 1100 */ 1414, 1415, 1418, 301, 1354, 425, 1429, 1331, 222, 1414, - /* 1110 */ 1415, 1418, 940, 425, 1330, 427, 1329, 1328, 1327, 1386, - /* 1120 */ 1326, 1413, 1325, 427, 1324, 1323, 1413, 1386, 1322, 1321, - /* 1130 */ 1320, 1319, 100, 1318, 1317, 230, 1414, 1415, 1418, 1316, - /* 1140 */ 1315, 1314, 1429, 223, 1414, 1415, 1418, 1429, 1313, 425, - /* 1150 */ 1312, 1311, 1310, 1309, 425, 1200, 1371, 942, 1365, 427, - /* 1160 */ 1353, 1344, 1413, 1386, 427, 1240, 141, 1413, 1386, 811, - /* 1170 */ 1199, 1197, 1186, 1413, 341, 343, 1185, 1182, 342, 231, - /* 1180 */ 1414, 1415, 1418, 1429, 1426, 1414, 1415, 1418, 1429, 1242, - /* 1190 */ 425, 73, 912, 486, 1429, 425, 910, 1241, 1195, 1190, - /* 1200 */ 427, 425, 1238, 148, 1386, 427, 840, 839, 247, 1386, - /* 1210 */ 248, 427, 365, 838, 1413, 1386, 837, 835, 834, 1188, - /* 1220 */ 1425, 1414, 1415, 1418, 1236, 1424, 1414, 1415, 1418, 1413, - /* 1230 */ 249, 238, 1414, 1415, 1418, 1429, 1181, 370, 268, 267, - /* 1240 */ 1413, 368, 425, 1180, 372, 71, 1370, 163, 1003, 42, - /* 1250 */ 1429, 976, 427, 1364, 990, 1413, 1386, 425, 110, 206, - /* 1260 */ 380, 1429, 1352, 472, 996, 166, 1351, 427, 425, 1343, - /* 1270 */ 988, 1386, 237, 1414, 1415, 1418, 1429, 381, 427, 36, - /* 1280 */ 366, 206, 1386, 425, 474, 472, 51, 239, 1414, 1415, - /* 1290 */ 1418, 169, 171, 427, 3, 374, 32, 1386, 236, 1414, - /* 1300 */ 1415, 1418, 37, 471, 470, 469, 474, 468, 116, 162, - /* 1310 */ 174, 1114, 369, 225, 1414, 1415, 1418, 363, 442, 178, - /* 1320 */ 161, 1108, 1107, 14, 507, 471, 470, 469, 52, 468, - /* 1330 */ 999, 179, 53, 34, 19, 1404, 989, 15, 1086, 185, - /* 1340 */ 1085, 35, 124, 1141, 41, 16, 10, 40, 54, 1136, - /* 1350 */ 1135, 256, 1140, 1139, 257, 8, 1050, 1024, 192, 13, - /* 1360 */ 18, 432, 193, 1112, 1004, 195, 197, 55, 1342, 199, - /* 1370 */ 991, 56, 201, 60, 1005, 38, 1403, 896, 445, 270, - /* 1380 */ 204, 1007, 995, 443, 449, 452, 447, 994, 995, 893, - /* 1390 */ 450, 873, 890, 453, 455, 884, 456, 458, 882, 459, - /* 1400 */ 61, 907, 888, 887, 886, 465, 62, 63, 906, 903, - /* 1410 */ 901, 475, 885, 809, 831, 830, 829, 828, 827, 826, - /* 1420 */ 825, 824, 823, 842, 821, 820, 1196, 819, 818, 817, - /* 1430 */ 1184, 816, 815, 814, 496, 497, 500, 1183, 501, 1179, - /* 1440 */ 498, 504, 505, 1154, 992, 215, 508, 509, + /* 670 */ 478, 477, 1429, 1476, 1476, 424, 1386, 165, 58, 425, + /* 680 */ 1386, 1386, 1386, 1386, 1386, 466, 1386, 1386, 284, 427, + /* 690 */ 418, 1472, 1471, 1386, 1481, 1089, 1254, 486, 152, 1413, + /* 700 */ 295, 150, 374, 143, 67, 138, 384, 140, 1168, 69, + /* 710 */ 1414, 1415, 1418, 1462, 1226, 372, 1001, 255, 1458, 1535, + /* 720 */ 1429, 1194, 1189, 154, 137, 415, 153, 412, 1519, 46, + /* 730 */ 45, 304, 156, 130, 1187, 155, 158, 427, 298, 157, + /* 740 */ 1302, 1386, 1413, 363, 365, 64, 242, 188, 290, 397, + /* 750 */ 286, 282, 127, 9, 8, 60, 368, 69, 1414, 1415, + /* 760 */ 1418, 1462, 181, 1429, 346, 255, 1458, 120, 104, 44, + /* 770 */ 425, 388, 1115, 175, 126, 997, 177, 421, 1413, 187, + /* 780 */ 427, 1150, 1151, 1049, 1386, 392, 1489, 33, 1092, 410, + /* 790 */ 1064, 33, 33, 194, 1023, 957, 196, 1492, 1430, 1429, + /* 800 */ 69, 1414, 1415, 1418, 1462, 1413, 425, 190, 255, 1458, + /* 810 */ 1535, 101, 102, 2, 433, 202, 427, 1010, 104, 1480, + /* 820 */ 1386, 867, 283, 279, 241, 413, 1429, 409, 510, 243, + /* 830 */ 966, 44, 841, 425, 862, 211, 225, 1414, 1415, 1418, + /* 840 */ 320, 1000, 210, 427, 314, 94, 1338, 1386, 132, 98, + /* 850 */ 333, 499, 446, 209, 102, 895, 1523, 899, 103, 104, + /* 860 */ 1413, 905, 904, 70, 1414, 1415, 1418, 1462, 413, 125, + /* 870 */ 328, 423, 1458, 1521, 1019, 327, 329, 66, 334, 1018, + /* 880 */ 203, 1429, 96, 102, 335, 336, 105, 139, 425, 337, + /* 890 */ 1017, 338, 184, 1469, 408, 142, 407, 49, 427, 1523, + /* 900 */ 341, 145, 1386, 1016, 1413, 348, 74, 350, 379, 1413, + /* 910 */ 437, 1251, 125, 149, 378, 380, 1521, 1247, 115, 1414, + /* 920 */ 1415, 1418, 151, 106, 353, 1429, 360, 377, 107, 1249, + /* 930 */ 1429, 1015, 425, 1245, 108, 391, 109, 425, 168, 167, + /* 940 */ 170, 251, 427, 387, 398, 389, 1386, 427, 1493, 261, + /* 950 */ 390, 1386, 431, 974, 393, 164, 414, 1536, 383, 1503, + /* 960 */ 995, 173, 229, 1414, 1415, 1418, 1413, 229, 1414, 1415, + /* 970 */ 1418, 1413, 1502, 395, 5, 176, 1089, 394, 254, 406, + /* 980 */ 1483, 97, 401, 4, 1014, 17, 34, 1429, 256, 1477, + /* 990 */ 422, 183, 1429, 419, 425, 1347, 429, 435, 198, 425, + /* 1000 */ 430, 264, 182, 180, 427, 118, 1346, 1413, 1386, 427, + /* 1010 */ 1444, 434, 1262, 1386, 436, 1413, 259, 200, 59, 57, + /* 1020 */ 212, 1520, 214, 208, 228, 1414, 1415, 1418, 1429, 229, + /* 1030 */ 1414, 1415, 1418, 473, 444, 425, 1429, 189, 487, 506, + /* 1040 */ 220, 1413, 1538, 425, 221, 427, 40, 216, 1380, 1386, + /* 1050 */ 1413, 218, 263, 427, 1379, 278, 405, 1386, 1376, 280, + /* 1060 */ 281, 984, 1429, 285, 985, 229, 1414, 1415, 1418, 425, + /* 1070 */ 128, 1429, 1374, 222, 1414, 1415, 1418, 1413, 425, 427, + /* 1080 */ 287, 288, 289, 1386, 1373, 1413, 1372, 291, 427, 1363, + /* 1090 */ 293, 129, 1386, 296, 297, 969, 1357, 968, 1429, 227, + /* 1100 */ 1414, 1415, 1418, 1356, 303, 425, 1429, 302, 230, 1414, + /* 1110 */ 1415, 1418, 1355, 425, 1354, 427, 940, 1331, 1330, 1386, + /* 1120 */ 1329, 1413, 1328, 427, 1327, 1326, 1413, 1386, 1325, 1324, + /* 1130 */ 1323, 1322, 1321, 1320, 1319, 223, 1414, 1415, 1418, 100, + /* 1140 */ 1318, 1317, 1429, 231, 1414, 1415, 1418, 1429, 1316, 425, + /* 1150 */ 1315, 1314, 1313, 1312, 425, 1311, 1310, 942, 1309, 427, + /* 1160 */ 1200, 1371, 1413, 1386, 427, 1365, 1353, 1413, 1386, 1344, + /* 1170 */ 141, 1240, 1199, 1413, 811, 1197, 342, 1186, 343, 224, + /* 1180 */ 1414, 1415, 1418, 1429, 232, 1414, 1415, 1418, 1429, 344, + /* 1190 */ 425, 1185, 1182, 1242, 1429, 425, 910, 73, 1241, 148, + /* 1200 */ 427, 425, 912, 486, 1386, 427, 840, 1195, 839, 1386, + /* 1210 */ 248, 427, 1190, 1188, 1413, 1386, 838, 837, 835, 834, + /* 1220 */ 1426, 1414, 1415, 1418, 249, 1425, 1414, 1415, 1418, 1413, + /* 1230 */ 250, 1424, 1414, 1415, 1418, 1429, 366, 369, 269, 268, + /* 1240 */ 1413, 371, 425, 1181, 1180, 373, 1370, 71, 1003, 1238, + /* 1250 */ 1429, 43, 427, 163, 976, 1413, 1386, 425, 1364, 110, + /* 1260 */ 381, 1429, 1352, 990, 996, 1236, 1351, 427, 425, 382, + /* 1270 */ 166, 1386, 239, 1414, 1415, 1418, 1429, 37, 427, 988, + /* 1280 */ 1343, 1413, 1386, 425, 52, 169, 116, 238, 1414, 1415, + /* 1290 */ 1418, 3, 367, 427, 178, 33, 38, 1386, 240, 1414, + /* 1300 */ 1415, 1418, 1429, 174, 1114, 19, 206, 375, 171, 425, + /* 1310 */ 472, 1404, 179, 237, 1414, 1415, 1418, 14, 442, 427, + /* 1320 */ 1107, 162, 206, 1386, 370, 53, 472, 20, 185, 364, + /* 1330 */ 999, 474, 161, 507, 15, 124, 35, 1086, 1085, 226, + /* 1340 */ 1414, 1415, 1418, 36, 1141, 989, 16, 474, 54, 1136, + /* 1350 */ 471, 470, 469, 10, 468, 1135, 42, 257, 1140, 41, + /* 1360 */ 1139, 258, 8, 192, 1004, 1050, 471, 470, 469, 1024, + /* 1370 */ 468, 13, 18, 1342, 193, 1112, 195, 197, 55, 991, + /* 1380 */ 432, 1007, 995, 199, 201, 56, 1005, 1403, 896, 60, + /* 1390 */ 39, 445, 271, 204, 443, 447, 994, 995, 893, 449, + /* 1400 */ 450, 452, 890, 453, 455, 884, 456, 458, 882, 888, + /* 1410 */ 459, 873, 465, 907, 61, 62, 63, 906, 887, 886, + /* 1420 */ 903, 885, 901, 831, 809, 475, 830, 823, 829, 828, + /* 1430 */ 827, 826, 825, 824, 842, 821, 820, 819, 1196, 1184, + /* 1440 */ 818, 817, 816, 1183, 815, 814, 496, 497, 500, 501, + /* 1450 */ 1179, 504, 498, 505, 1154, 992, 215, 508, 1154, 1154, + /* 1460 */ 509, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 226, 210, 210, 217, 210, 226, 204, 216, 216, 210, @@ -376,135 +378,135 @@ static const YYCODETYPE yy_lookahead[] = { /* 150 */ 14, 15, 16, 130, 209, 155, 156, 157, 158, 159, /* 160 */ 160, 161, 162, 163, 164, 220, 18, 225, 20, 69, /* 170 */ 92, 137, 227, 139, 232, 27, 176, 138, 30, 20, - /* 180 */ 242, 81, 104, 245, 242, 225, 248, 39, 246, 69, - /* 190 */ 204, 128, 232, 44, 155, 156, 157, 158, 159, 160, + /* 180 */ 242, 81, 104, 245, 242, 234, 248, 39, 246, 69, + /* 190 */ 204, 176, 241, 242, 155, 156, 157, 158, 159, 160, /* 200 */ 161, 162, 163, 164, 262, 263, 264, 265, 266, 210, - /* 210 */ 176, 225, 270, 271, 272, 216, 138, 68, 232, 225, - /* 220 */ 71, 210, 232, 281, 264, 231, 20, 216, 242, 239, + /* 210 */ 176, 225, 270, 271, 272, 216, 138, 0, 232, 225, + /* 220 */ 57, 210, 232, 281, 61, 231, 20, 216, 242, 239, /* 230 */ 236, 232, 246, 155, 156, 157, 158, 159, 160, 161, - /* 240 */ 162, 163, 164, 232, 181, 182, 12, 13, 262, 263, - /* 250 */ 264, 265, 266, 69, 20, 107, 22, 271, 110, 111, - /* 260 */ 112, 113, 114, 79, 116, 117, 118, 119, 120, 121, + /* 240 */ 162, 163, 164, 232, 0, 82, 12, 13, 262, 263, + /* 250 */ 264, 265, 266, 46, 20, 107, 22, 271, 110, 111, + /* 260 */ 112, 113, 114, 46, 116, 117, 118, 119, 120, 121, /* 270 */ 122, 123, 38, 21, 155, 69, 24, 25, 26, 27, /* 280 */ 28, 29, 30, 31, 32, 12, 13, 14, 0, 49, - /* 290 */ 225, 51, 251, 20, 54, 22, 176, 57, 233, 59, - /* 300 */ 60, 176, 62, 69, 185, 186, 187, 188, 189, 21, - /* 310 */ 43, 38, 24, 25, 26, 27, 28, 29, 30, 31, - /* 320 */ 32, 210, 225, 282, 12, 13, 92, 216, 231, 207, - /* 330 */ 208, 210, 20, 236, 22, 176, 295, 216, 104, 213, - /* 340 */ 299, 215, 69, 232, 234, 12, 13, 14, 15, 16, - /* 350 */ 38, 241, 242, 232, 83, 84, 85, 86, 87, 88, + /* 290 */ 46, 51, 251, 20, 54, 22, 176, 57, 218, 59, + /* 300 */ 60, 221, 62, 69, 185, 186, 187, 188, 189, 21, + /* 310 */ 204, 38, 24, 25, 26, 27, 28, 29, 30, 31, + /* 320 */ 32, 210, 225, 282, 12, 13, 92, 216, 231, 128, + /* 330 */ 46, 210, 20, 236, 22, 176, 295, 216, 104, 213, + /* 340 */ 299, 215, 69, 232, 20, 12, 13, 14, 15, 16, + /* 350 */ 38, 44, 246, 232, 83, 84, 85, 86, 87, 88, /* 360 */ 89, 90, 91, 92, 93, 92, 95, 96, 97, 98, - /* 370 */ 99, 100, 138, 14, 15, 16, 203, 104, 205, 218, - /* 380 */ 204, 69, 221, 0, 12, 13, 14, 15, 16, 155, + /* 370 */ 99, 100, 138, 204, 203, 68, 205, 104, 71, 20, + /* 380 */ 204, 69, 181, 182, 12, 13, 14, 15, 16, 155, /* 390 */ 156, 157, 158, 159, 160, 161, 162, 163, 164, 210, - /* 400 */ 282, 225, 0, 70, 92, 216, 46, 210, 232, 0, - /* 410 */ 176, 138, 204, 295, 242, 20, 104, 299, 242, 36, - /* 420 */ 248, 232, 246, 12, 13, 14, 15, 16, 155, 156, + /* 400 */ 282, 225, 0, 70, 92, 216, 225, 210, 232, 20, + /* 410 */ 176, 138, 204, 295, 233, 246, 104, 299, 242, 207, + /* 420 */ 208, 232, 246, 12, 13, 14, 15, 16, 155, 156, /* 430 */ 157, 158, 159, 160, 161, 162, 163, 164, 262, 263, - /* 440 */ 264, 265, 175, 12, 13, 14, 15, 16, 251, 217, - /* 450 */ 138, 49, 20, 51, 246, 46, 54, 225, 47, 57, + /* 440 */ 264, 265, 43, 12, 13, 14, 15, 16, 251, 217, + /* 450 */ 138, 49, 20, 51, 246, 226, 54, 225, 47, 57, /* 460 */ 35, 59, 60, 204, 62, 233, 204, 155, 156, 157, /* 470 */ 158, 159, 160, 161, 162, 163, 164, 301, 47, 282, /* 480 */ 12, 13, 57, 1, 2, 20, 61, 225, 20, 78, - /* 490 */ 22, 0, 295, 212, 232, 134, 299, 72, 251, 74, + /* 490 */ 22, 0, 295, 124, 232, 134, 299, 72, 251, 74, /* 500 */ 75, 70, 77, 131, 242, 246, 38, 82, 246, 78, - /* 510 */ 149, 230, 204, 251, 204, 24, 25, 26, 27, 28, + /* 510 */ 149, 142, 204, 251, 204, 24, 25, 26, 27, 28, /* 520 */ 29, 30, 31, 32, 262, 263, 264, 265, 266, 282, - /* 530 */ 212, 217, 270, 271, 57, 124, 3, 69, 61, 225, - /* 540 */ 222, 4, 295, 204, 282, 210, 299, 233, 230, 20, - /* 550 */ 124, 216, 70, 142, 246, 124, 246, 295, 46, 82, - /* 560 */ 92, 299, 130, 225, 210, 210, 204, 232, 142, 197, - /* 570 */ 216, 216, 104, 142, 236, 204, 165, 166, 167, 168, - /* 580 */ 169, 170, 171, 172, 173, 246, 232, 232, 20, 260, - /* 590 */ 0, 165, 204, 204, 204, 204, 165, 166, 167, 168, - /* 600 */ 169, 170, 171, 172, 173, 0, 138, 278, 246, 14, - /* 610 */ 204, 33, 22, 226, 36, 20, 18, 246, 174, 175, + /* 530 */ 212, 217, 270, 271, 165, 124, 3, 69, 225, 225, + /* 540 */ 222, 212, 295, 204, 282, 210, 299, 233, 230, 236, + /* 550 */ 22, 216, 70, 142, 246, 124, 246, 295, 80, 230, + /* 560 */ 92, 299, 130, 67, 210, 210, 38, 232, 204, 197, + /* 570 */ 216, 216, 104, 142, 175, 204, 165, 166, 167, 168, + /* 580 */ 169, 170, 171, 172, 173, 246, 232, 232, 226, 12, + /* 590 */ 13, 14, 15, 16, 204, 204, 165, 166, 167, 168, + /* 600 */ 169, 170, 171, 172, 173, 0, 138, 14, 15, 16, + /* 610 */ 246, 33, 242, 14, 36, 4, 18, 246, 248, 20, /* 620 */ 42, 23, 44, 155, 156, 157, 158, 159, 160, 161, - /* 630 */ 162, 163, 164, 35, 246, 246, 246, 246, 204, 204, - /* 640 */ 204, 204, 204, 45, 204, 204, 68, 1, 2, 71, - /* 650 */ 0, 204, 246, 67, 49, 50, 51, 52, 53, 54, + /* 630 */ 162, 163, 164, 35, 204, 260, 246, 246, 204, 204, + /* 640 */ 204, 204, 204, 45, 204, 204, 68, 70, 67, 71, + /* 650 */ 38, 204, 0, 278, 49, 50, 51, 52, 53, 54, /* 660 */ 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, - /* 670 */ 65, 66, 225, 260, 260, 67, 209, 80, 70, 232, - /* 680 */ 246, 246, 246, 246, 246, 22, 246, 246, 67, 242, - /* 690 */ 47, 278, 278, 246, 227, 73, 46, 73, 76, 204, - /* 700 */ 76, 38, 21, 125, 106, 127, 38, 129, 254, 262, - /* 710 */ 263, 264, 265, 266, 177, 34, 226, 270, 271, 272, - /* 720 */ 225, 199, 200, 226, 146, 192, 226, 232, 281, 131, - /* 730 */ 132, 133, 73, 135, 0, 76, 67, 242, 140, 70, - /* 740 */ 73, 246, 204, 76, 205, 215, 148, 0, 150, 293, - /* 750 */ 152, 153, 154, 302, 235, 207, 22, 262, 263, 264, - /* 760 */ 265, 266, 287, 225, 261, 270, 271, 272, 67, 22, - /* 770 */ 232, 70, 104, 67, 176, 38, 70, 225, 204, 284, - /* 780 */ 242, 280, 196, 296, 246, 290, 291, 67, 67, 67, - /* 790 */ 70, 70, 70, 67, 283, 20, 70, 210, 155, 225, - /* 800 */ 262, 263, 264, 265, 266, 204, 232, 36, 270, 271, - /* 810 */ 272, 67, 257, 214, 70, 194, 242, 38, 67, 281, - /* 820 */ 246, 70, 136, 252, 210, 251, 225, 210, 19, 210, - /* 830 */ 115, 67, 240, 232, 70, 124, 262, 263, 264, 265, - /* 840 */ 238, 104, 33, 242, 238, 36, 210, 246, 20, 232, - /* 850 */ 256, 42, 67, 44, 67, 70, 282, 70, 67, 67, + /* 670 */ 65, 66, 225, 260, 260, 47, 246, 226, 209, 232, + /* 680 */ 246, 246, 246, 246, 246, 226, 246, 246, 36, 242, + /* 690 */ 194, 278, 278, 246, 174, 175, 227, 67, 73, 204, + /* 700 */ 70, 76, 21, 125, 106, 127, 254, 129, 205, 262, + /* 710 */ 263, 264, 265, 266, 215, 34, 104, 270, 271, 272, + /* 720 */ 225, 0, 0, 73, 146, 192, 76, 232, 281, 131, + /* 730 */ 132, 133, 73, 135, 0, 76, 73, 242, 140, 76, + /* 740 */ 235, 246, 204, 22, 22, 69, 148, 302, 150, 293, + /* 750 */ 152, 153, 154, 1, 2, 79, 22, 262, 263, 264, + /* 760 */ 265, 266, 287, 225, 207, 270, 271, 272, 67, 67, + /* 770 */ 232, 70, 70, 67, 176, 38, 70, 196, 204, 284, + /* 780 */ 242, 199, 200, 155, 246, 290, 291, 67, 177, 280, + /* 790 */ 70, 67, 67, 67, 70, 70, 70, 261, 225, 225, + /* 800 */ 262, 263, 264, 265, 266, 204, 232, 296, 270, 271, + /* 810 */ 272, 67, 67, 283, 70, 70, 242, 20, 67, 281, + /* 820 */ 246, 70, 36, 210, 257, 251, 225, 210, 19, 214, + /* 830 */ 136, 67, 38, 232, 70, 252, 262, 263, 264, 265, + /* 840 */ 240, 104, 33, 242, 210, 36, 210, 246, 115, 232, + /* 850 */ 210, 42, 67, 44, 67, 70, 282, 70, 67, 67, /* 860 */ 204, 70, 70, 262, 263, 264, 265, 266, 251, 295, - /* 870 */ 242, 270, 271, 299, 20, 250, 212, 68, 20, 232, - /* 880 */ 71, 225, 265, 67, 67, 243, 70, 70, 232, 212, - /* 890 */ 212, 212, 275, 276, 277, 210, 279, 20, 242, 282, - /* 900 */ 206, 225, 246, 210, 204, 225, 232, 225, 225, 204, - /* 910 */ 101, 256, 295, 225, 225, 225, 299, 225, 262, 263, - /* 920 */ 264, 265, 225, 225, 225, 225, 214, 214, 145, 206, - /* 930 */ 225, 20, 232, 242, 209, 126, 232, 232, 129, 255, - /* 940 */ 209, 184, 242, 292, 183, 250, 246, 242, 261, 249, - /* 950 */ 243, 246, 156, 144, 249, 146, 300, 301, 292, 191, - /* 960 */ 289, 247, 262, 263, 264, 265, 204, 262, 263, 264, - /* 970 */ 265, 204, 190, 246, 179, 247, 246, 178, 246, 175, - /* 980 */ 232, 20, 115, 273, 288, 195, 198, 225, 286, 260, - /* 990 */ 193, 246, 225, 285, 232, 69, 247, 247, 246, 232, - /* 1000 */ 246, 127, 269, 244, 242, 232, 303, 204, 246, 242, - /* 1010 */ 209, 209, 243, 246, 298, 204, 249, 69, 232, 221, - /* 1020 */ 214, 297, 228, 210, 262, 263, 264, 265, 225, 262, - /* 1030 */ 263, 264, 265, 214, 209, 232, 225, 206, 211, 219, - /* 1040 */ 219, 204, 253, 232, 202, 242, 0, 0, 60, 246, - /* 1050 */ 204, 38, 249, 242, 0, 151, 294, 246, 38, 38, - /* 1060 */ 38, 151, 225, 0, 38, 262, 263, 264, 265, 232, - /* 1070 */ 38, 225, 151, 262, 263, 264, 265, 204, 232, 242, - /* 1080 */ 0, 38, 0, 246, 0, 204, 38, 69, 242, 142, - /* 1090 */ 141, 104, 246, 138, 0, 0, 134, 0, 225, 262, - /* 1100 */ 263, 264, 265, 50, 0, 232, 225, 0, 262, 263, - /* 1110 */ 264, 265, 81, 232, 0, 242, 0, 0, 0, 246, + /* 870 */ 124, 270, 271, 299, 20, 238, 238, 68, 256, 20, + /* 880 */ 71, 225, 265, 67, 242, 250, 70, 212, 232, 232, + /* 890 */ 20, 243, 275, 276, 277, 212, 279, 212, 242, 282, + /* 900 */ 210, 212, 246, 20, 204, 206, 210, 225, 145, 204, + /* 910 */ 101, 225, 295, 225, 256, 255, 299, 225, 262, 263, + /* 920 */ 264, 265, 225, 225, 214, 225, 214, 232, 225, 225, + /* 930 */ 225, 20, 232, 225, 225, 126, 225, 232, 129, 209, + /* 940 */ 209, 206, 242, 250, 184, 232, 246, 242, 261, 249, + /* 950 */ 243, 246, 183, 144, 249, 146, 300, 301, 242, 292, + /* 960 */ 156, 247, 262, 263, 264, 265, 204, 262, 263, 264, + /* 970 */ 265, 204, 292, 246, 191, 247, 175, 179, 246, 190, + /* 980 */ 289, 232, 246, 178, 20, 69, 115, 225, 198, 260, + /* 990 */ 195, 273, 225, 193, 232, 247, 246, 244, 232, 232, + /* 1000 */ 246, 246, 285, 288, 242, 286, 247, 204, 246, 242, + /* 1010 */ 269, 127, 232, 246, 243, 204, 249, 209, 69, 209, + /* 1020 */ 221, 298, 210, 209, 262, 263, 264, 265, 225, 262, + /* 1030 */ 263, 264, 265, 214, 228, 232, 225, 297, 214, 206, + /* 1040 */ 219, 204, 303, 232, 219, 242, 253, 211, 0, 246, + /* 1050 */ 204, 202, 249, 242, 0, 60, 294, 246, 0, 38, + /* 1060 */ 151, 38, 225, 151, 38, 262, 263, 264, 265, 232, + /* 1070 */ 38, 225, 0, 262, 263, 264, 265, 204, 232, 242, + /* 1080 */ 38, 38, 151, 246, 0, 204, 0, 38, 242, 0, + /* 1090 */ 38, 69, 246, 142, 141, 104, 0, 138, 225, 262, + /* 1100 */ 263, 264, 265, 0, 134, 232, 225, 50, 262, 263, + /* 1110 */ 264, 265, 0, 232, 0, 242, 81, 0, 0, 246, /* 1120 */ 0, 204, 0, 242, 0, 0, 204, 246, 0, 0, - /* 1130 */ 0, 0, 115, 0, 0, 262, 263, 264, 265, 0, + /* 1130 */ 0, 0, 0, 0, 0, 262, 263, 264, 265, 115, /* 1140 */ 0, 0, 225, 262, 263, 264, 265, 225, 0, 232, /* 1150 */ 0, 0, 0, 0, 232, 0, 0, 22, 0, 242, - /* 1160 */ 0, 0, 204, 246, 242, 0, 43, 204, 246, 48, - /* 1170 */ 0, 0, 0, 204, 38, 43, 0, 0, 36, 262, - /* 1180 */ 263, 264, 265, 225, 262, 263, 264, 265, 225, 0, - /* 1190 */ 232, 78, 38, 67, 225, 232, 22, 0, 0, 0, - /* 1200 */ 242, 232, 0, 76, 246, 242, 38, 38, 22, 246, - /* 1210 */ 22, 242, 39, 38, 204, 246, 38, 38, 38, 0, - /* 1220 */ 262, 263, 264, 265, 0, 262, 263, 264, 265, 204, - /* 1230 */ 22, 262, 263, 264, 265, 225, 0, 22, 12, 13, - /* 1240 */ 204, 38, 232, 0, 22, 20, 0, 147, 22, 130, - /* 1250 */ 225, 38, 242, 0, 22, 204, 246, 232, 143, 57, - /* 1260 */ 22, 225, 0, 61, 38, 127, 0, 242, 232, 0, - /* 1270 */ 38, 246, 262, 263, 264, 265, 225, 130, 242, 130, - /* 1280 */ 4, 57, 246, 232, 82, 61, 69, 262, 263, 264, - /* 1290 */ 265, 43, 125, 242, 67, 19, 67, 246, 262, 263, - /* 1300 */ 264, 265, 67, 101, 102, 103, 82, 105, 69, 33, - /* 1310 */ 70, 70, 36, 262, 263, 264, 265, 41, 92, 69, - /* 1320 */ 44, 70, 70, 180, 92, 101, 102, 103, 69, 105, - /* 1330 */ 104, 67, 69, 174, 67, 81, 104, 180, 70, 81, - /* 1340 */ 70, 67, 81, 70, 68, 67, 180, 71, 4, 38, - /* 1350 */ 38, 38, 38, 38, 38, 2, 155, 70, 81, 69, - /* 1360 */ 69, 128, 70, 70, 138, 69, 69, 69, 0, 43, - /* 1370 */ 138, 69, 125, 79, 22, 69, 81, 70, 38, 38, - /* 1380 */ 81, 155, 156, 80, 38, 38, 69, 155, 156, 70, - /* 1390 */ 69, 22, 70, 69, 38, 70, 69, 38, 70, 69, - /* 1400 */ 69, 38, 94, 94, 94, 82, 69, 69, 104, 38, - /* 1410 */ 22, 47, 94, 48, 22, 38, 38, 38, 38, 38, - /* 1420 */ 38, 38, 22, 38, 38, 38, 0, 38, 38, 38, - /* 1430 */ 0, 38, 38, 38, 38, 36, 38, 0, 37, 0, - /* 1440 */ 43, 22, 21, 304, 22, 22, 21, 20, 304, 304, - /* 1450 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1460 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1160 */ 0, 0, 204, 246, 242, 0, 0, 204, 246, 0, + /* 1170 */ 43, 0, 0, 204, 48, 0, 38, 0, 36, 262, + /* 1180 */ 263, 264, 265, 225, 262, 263, 264, 265, 225, 43, + /* 1190 */ 232, 0, 0, 0, 225, 232, 22, 78, 0, 76, + /* 1200 */ 242, 232, 38, 67, 246, 242, 38, 0, 38, 246, + /* 1210 */ 22, 242, 0, 0, 204, 246, 38, 38, 38, 38, + /* 1220 */ 262, 263, 264, 265, 22, 262, 263, 264, 265, 204, + /* 1230 */ 22, 262, 263, 264, 265, 225, 39, 38, 12, 13, + /* 1240 */ 204, 22, 232, 0, 0, 22, 0, 20, 22, 0, + /* 1250 */ 225, 130, 242, 147, 38, 204, 246, 232, 0, 143, + /* 1260 */ 22, 225, 0, 22, 38, 0, 0, 242, 232, 130, + /* 1270 */ 127, 246, 262, 263, 264, 265, 225, 130, 242, 38, + /* 1280 */ 0, 204, 246, 232, 69, 43, 69, 262, 263, 264, + /* 1290 */ 265, 67, 4, 242, 69, 67, 67, 246, 262, 263, + /* 1300 */ 264, 265, 225, 70, 70, 69, 57, 19, 125, 232, + /* 1310 */ 61, 81, 67, 262, 263, 264, 265, 180, 92, 242, + /* 1320 */ 70, 33, 57, 246, 36, 69, 61, 67, 81, 41, + /* 1330 */ 104, 82, 44, 92, 180, 81, 174, 70, 70, 262, + /* 1340 */ 263, 264, 265, 67, 70, 104, 67, 82, 4, 38, + /* 1350 */ 101, 102, 103, 180, 105, 38, 68, 38, 38, 71, + /* 1360 */ 38, 38, 2, 81, 138, 155, 101, 102, 103, 70, + /* 1370 */ 105, 69, 69, 0, 70, 70, 69, 69, 69, 138, + /* 1380 */ 128, 155, 156, 43, 125, 69, 22, 81, 70, 79, + /* 1390 */ 69, 38, 38, 81, 80, 69, 155, 156, 70, 38, + /* 1400 */ 69, 38, 70, 69, 38, 70, 69, 38, 70, 94, + /* 1410 */ 69, 22, 82, 38, 69, 69, 69, 104, 94, 94, + /* 1420 */ 38, 94, 22, 22, 48, 47, 38, 22, 38, 38, + /* 1430 */ 38, 38, 38, 38, 38, 38, 38, 38, 0, 0, + /* 1440 */ 38, 38, 38, 0, 38, 38, 38, 36, 38, 37, + /* 1450 */ 0, 22, 43, 21, 304, 22, 22, 21, 304, 304, + /* 1460 */ 20, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1470 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1480 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1490 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, @@ -522,91 +524,93 @@ static const YYCODETYPE yy_lookahead[] = { /* 1610 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1620 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1630 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1640 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1640 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1650 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1660 */ 304, 304, }; #define YY_SHIFT_COUNT (510) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1439) +#define YY_SHIFT_MAX (1450) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 598, 0, 39, 78, 78, 78, 78, 234, 78, 78, /* 10 */ 312, 468, 120, 273, 312, 312, 312, 312, 312, 312, /* 20 */ 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, - /* 30 */ 312, 312, 312, 206, 206, 206, 159, 1226, 1226, 34, - /* 40 */ 81, 81, 125, 1226, 81, 81, 81, 81, 81, 81, - /* 50 */ 360, 395, 465, 465, 125, 529, 395, 81, 81, 395, - /* 60 */ 81, 395, 529, 395, 395, 81, 512, 148, 431, 411, - /* 70 */ 411, 252, 425, 1232, 240, 1232, 1232, 1232, 1232, 1232, - /* 80 */ 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, - /* 90 */ 1232, 1232, 1232, 1232, 38, 409, 23, 23, 23, 650, - /* 100 */ 568, 529, 395, 395, 395, 597, 271, 271, 271, 271, - /* 110 */ 271, 271, 809, 288, 402, 372, 119, 477, 63, 663, - /* 120 */ 432, 444, 267, 444, 595, 533, 537, 775, 771, 779, - /* 130 */ 686, 775, 775, 715, 711, 711, 775, 828, 529, 854, - /* 140 */ 360, 568, 858, 360, 360, 775, 360, 877, 395, 395, - /* 150 */ 395, 395, 395, 395, 395, 395, 395, 395, 395, 779, - /* 160 */ 779, 775, 877, 568, 828, 783, 529, 854, 512, 568, - /* 170 */ 858, 512, 911, 757, 761, 796, 757, 761, 796, 796, - /* 180 */ 768, 782, 795, 799, 804, 568, 961, 867, 788, 790, - /* 190 */ 797, 926, 395, 761, 796, 796, 761, 796, 874, 568, - /* 200 */ 858, 512, 597, 512, 568, 948, 779, 779, 775, 512, - /* 210 */ 877, 1448, 1448, 1448, 1448, 1448, 605, 578, 491, 1276, - /* 220 */ 1202, 1224, 16, 108, 333, 136, 136, 136, 136, 136, - /* 230 */ 136, 136, 149, 36, 482, 426, 359, 359, 359, 359, - /* 240 */ 383, 361, 608, 622, 624, 659, 667, 590, 734, 747, - /* 250 */ 681, 669, 701, 706, 646, 522, 621, 586, 720, 643, - /* 260 */ 721, 100, 722, 726, 744, 751, 764, 668, 737, 785, - /* 270 */ 787, 791, 792, 816, 817, 184, 1046, 1047, 988, 1054, - /* 280 */ 1013, 904, 1020, 1021, 1022, 910, 1063, 1026, 1032, 921, - /* 290 */ 1080, 1043, 1082, 1048, 1084, 1018, 947, 949, 987, 955, - /* 300 */ 1094, 1095, 1053, 962, 1097, 1104, 1031, 1107, 1114, 1116, - /* 310 */ 1117, 1118, 1120, 1122, 1124, 1125, 1128, 1129, 1130, 1131, - /* 320 */ 1017, 1133, 1134, 1139, 1140, 1141, 1148, 1135, 1150, 1151, - /* 330 */ 1152, 1153, 1155, 1156, 1158, 1160, 1161, 1123, 1165, 1121, - /* 340 */ 1170, 1171, 1136, 1142, 1132, 1172, 1176, 1177, 1189, 1113, - /* 350 */ 1127, 1154, 1126, 1174, 1197, 1168, 1169, 1175, 1178, 1126, - /* 360 */ 1179, 1180, 1198, 1186, 1199, 1188, 1173, 1219, 1208, 1203, - /* 370 */ 1236, 1215, 1243, 1222, 1225, 1246, 1119, 1100, 1213, 1253, - /* 380 */ 1115, 1238, 1147, 1138, 1262, 1266, 1149, 1269, 1217, 1248, - /* 390 */ 1167, 1227, 1229, 1143, 1240, 1235, 1241, 1239, 1250, 1251, - /* 400 */ 1259, 1252, 1264, 1254, 1263, 1267, 1157, 1268, 1270, 1258, - /* 410 */ 1159, 1274, 1261, 1273, 1278, 1166, 1344, 1311, 1312, 1313, - /* 420 */ 1314, 1315, 1316, 1353, 1201, 1277, 1287, 1290, 1291, 1292, - /* 430 */ 1293, 1296, 1297, 1233, 1298, 1368, 1326, 1247, 1302, 1294, - /* 440 */ 1295, 1299, 1352, 1306, 1303, 1307, 1340, 1341, 1317, 1319, - /* 450 */ 1346, 1321, 1322, 1347, 1324, 1325, 1356, 1327, 1328, 1359, - /* 460 */ 1330, 1308, 1309, 1310, 1318, 1369, 1323, 1331, 1363, 1304, - /* 470 */ 1337, 1338, 1371, 1126, 1388, 1365, 1364, 1392, 1377, 1378, - /* 480 */ 1379, 1380, 1381, 1382, 1383, 1400, 1385, 1126, 1386, 1387, - /* 490 */ 1389, 1390, 1391, 1393, 1394, 1395, 1426, 1396, 1399, 1397, - /* 500 */ 1430, 1398, 1401, 1437, 1439, 1419, 1421, 1422, 1423, 1425, - /* 510 */ 1427, + /* 30 */ 312, 312, 312, 312, 206, 206, 206, 159, 1226, 1226, + /* 40 */ 34, 81, 81, 15, 1226, 81, 81, 81, 81, 81, + /* 50 */ 81, 207, 324, 359, 15, 389, 324, 81, 81, 324, + /* 60 */ 81, 324, 389, 324, 324, 81, 284, 148, 431, 411, + /* 70 */ 411, 252, 425, 1241, 240, 1241, 1241, 1241, 1241, 1241, + /* 80 */ 1241, 1241, 1241, 1241, 1241, 1241, 1241, 1241, 1241, 1241, + /* 90 */ 1241, 1241, 1241, 1241, 38, 217, 23, 23, 23, 244, + /* 100 */ 465, 389, 324, 324, 324, 478, 271, 271, 271, 271, + /* 110 */ 271, 271, 809, 288, 402, 372, 119, 163, 201, 528, + /* 120 */ 432, 520, 399, 520, 599, 533, 611, 797, 786, 794, + /* 130 */ 694, 797, 797, 733, 746, 746, 797, 854, 389, 859, + /* 140 */ 207, 465, 870, 207, 207, 797, 207, 883, 324, 324, + /* 150 */ 324, 324, 324, 324, 324, 324, 324, 324, 324, 794, + /* 160 */ 794, 797, 883, 465, 854, 763, 389, 859, 284, 465, + /* 170 */ 870, 284, 911, 760, 769, 804, 760, 769, 804, 804, + /* 180 */ 783, 789, 798, 805, 801, 465, 964, 871, 790, 795, + /* 190 */ 800, 916, 324, 769, 804, 804, 769, 804, 884, 465, + /* 200 */ 870, 284, 478, 284, 465, 949, 794, 794, 797, 284, + /* 210 */ 883, 1461, 1461, 1461, 1461, 1461, 605, 578, 491, 1288, + /* 220 */ 1249, 1265, 333, 16, 108, 577, 136, 136, 136, 136, + /* 230 */ 136, 136, 136, 307, 36, 482, 369, 593, 593, 593, + /* 240 */ 593, 652, 361, 630, 625, 650, 659, 663, 721, 722, + /* 250 */ 734, 681, 701, 702, 706, 752, 582, 496, 581, 720, + /* 260 */ 628, 724, 100, 725, 726, 744, 745, 751, 612, 737, + /* 270 */ 764, 785, 787, 791, 792, 816, 676, 1048, 1054, 995, + /* 280 */ 1058, 1021, 909, 1023, 1026, 1032, 912, 1072, 1042, 1043, + /* 290 */ 931, 1084, 1049, 1086, 1052, 1089, 1022, 951, 953, 991, + /* 300 */ 959, 1096, 1103, 1057, 970, 1112, 1114, 1035, 1117, 1118, + /* 310 */ 1120, 1122, 1124, 1125, 1128, 1129, 1130, 1131, 1132, 1133, + /* 320 */ 1134, 1024, 1140, 1141, 1148, 1150, 1151, 1152, 1135, 1153, + /* 330 */ 1155, 1156, 1158, 1160, 1161, 1165, 1166, 1169, 1127, 1171, + /* 340 */ 1126, 1172, 1175, 1138, 1142, 1146, 1177, 1191, 1192, 1193, + /* 350 */ 1119, 1123, 1164, 1136, 1174, 1198, 1168, 1170, 1178, 1179, + /* 360 */ 1136, 1180, 1181, 1207, 1188, 1212, 1202, 1197, 1213, 1208, + /* 370 */ 1199, 1243, 1219, 1244, 1223, 1227, 1246, 1121, 1106, 1216, + /* 380 */ 1258, 1116, 1238, 1139, 1143, 1262, 1266, 1147, 1280, 1215, + /* 390 */ 1242, 1183, 1224, 1228, 1137, 1233, 1229, 1234, 1217, 1225, + /* 400 */ 1236, 1250, 1245, 1230, 1256, 1260, 1154, 1267, 1268, 1247, + /* 410 */ 1162, 1276, 1254, 1274, 1279, 1173, 1344, 1311, 1317, 1319, + /* 420 */ 1320, 1322, 1323, 1360, 1210, 1282, 1299, 1302, 1303, 1304, + /* 430 */ 1305, 1307, 1308, 1252, 1309, 1373, 1340, 1259, 1316, 1310, + /* 440 */ 1306, 1312, 1364, 1321, 1314, 1318, 1353, 1354, 1326, 1328, + /* 450 */ 1361, 1331, 1332, 1363, 1334, 1335, 1366, 1337, 1338, 1369, + /* 460 */ 1341, 1315, 1324, 1325, 1327, 1389, 1330, 1345, 1375, 1313, + /* 470 */ 1346, 1347, 1382, 1136, 1400, 1376, 1378, 1401, 1388, 1390, + /* 480 */ 1391, 1392, 1393, 1394, 1395, 1405, 1396, 1136, 1397, 1398, + /* 490 */ 1399, 1402, 1403, 1404, 1406, 1407, 1438, 1408, 1411, 1409, + /* 500 */ 1439, 1410, 1412, 1443, 1450, 1429, 1432, 1433, 1434, 1436, + /* 510 */ 1440, }; #define YY_REDUCE_COUNT (215) #define YY_REDUCE_MIN (-226) -#define YY_REDUCE_MAX (1051) +#define YY_REDUCE_MAX (1077) static const short yy_reduce_ofst[] = { /* 0 */ -180, 262, 495, -198, -58, 447, 538, 574, -129, 601, /* 10 */ 656, -14, 617, 700, 705, 762, 176, 767, 803, 811, /* 20 */ 837, 846, 873, 881, 917, 922, 958, 963, 969, 1010, - /* 30 */ 1025, 1036, 1051, -195, -170, -147, 197, -213, -204, 41, - /* 40 */ -209, -206, 247, -132, -208, -175, -1, 11, 111, 121, - /* 50 */ 318, -142, -176, -40, 118, -62, -214, 189, 335, -6, - /* 60 */ 354, 232, 110, 97, 314, 355, -55, -201, -156, -156, - /* 70 */ -156, 173, -185, -107, 126, 208, 259, 308, 310, 339, - /* 80 */ 362, 371, 388, 389, 390, 391, 406, 434, 435, 436, - /* 90 */ 437, 438, 440, 441, 122, 281, 329, 413, 414, 467, - /* 100 */ -10, 172, 65, -149, 338, 161, -226, -221, 387, 490, - /* 110 */ 497, 500, 454, 539, 530, 451, 456, 519, 475, 548, - /* 120 */ 503, 501, 501, 501, 552, 487, 511, 587, 555, 599, - /* 130 */ 571, 614, 619, 592, 602, 606, 636, 594, 628, 625, - /* 140 */ 664, 647, 642, 677, 678, 685, 679, 694, 676, 680, - /* 150 */ 682, 683, 688, 689, 690, 692, 697, 698, 699, 712, - /* 160 */ 713, 693, 723, 674, 655, 684, 691, 695, 725, 704, - /* 170 */ 707, 731, 687, 651, 714, 727, 666, 728, 730, 732, - /* 180 */ 671, 696, 702, 708, 501, 748, 729, 710, 703, 716, - /* 190 */ 724, 733, 552, 749, 745, 752, 750, 754, 759, 773, - /* 200 */ 769, 801, 798, 802, 786, 794, 806, 819, 813, 825, - /* 210 */ 831, 789, 820, 821, 827, 842, + /* 30 */ 1025, 1036, 1051, 1077, -195, -170, -147, 197, -213, -204, + /* 40 */ 41, -209, -206, 247, -132, -208, -175, -1, 11, 111, + /* 50 */ 121, 318, -142, -176, 118, -62, -214, 189, 335, -6, + /* 60 */ 354, 232, -49, 97, 314, 355, -55, -201, -156, -156, + /* 70 */ -156, 171, -185, -107, 126, 106, 169, 208, 259, 308, + /* 80 */ 310, 339, 364, 371, 390, 391, 430, 434, 435, 436, + /* 90 */ 437, 438, 440, 441, 212, 329, 375, 413, 414, 469, + /* 100 */ -10, 370, 181, -149, 313, 80, -226, -221, 229, 362, + /* 110 */ 451, 459, 452, 503, 499, 445, 456, 505, 475, 557, + /* 120 */ 536, 509, 509, 509, 573, 511, 530, 613, 567, 615, + /* 130 */ 583, 634, 636, 600, 637, 638, 640, 622, 642, 635, + /* 140 */ 675, 657, 648, 683, 685, 690, 689, 699, 682, 686, + /* 150 */ 688, 692, 697, 698, 703, 704, 708, 709, 711, 710, + /* 160 */ 712, 696, 735, 695, 658, 660, 716, 693, 730, 713, + /* 170 */ 707, 731, 687, 667, 714, 727, 680, 728, 732, 736, + /* 180 */ 691, 715, 719, 717, 509, 749, 729, 718, 739, 723, + /* 190 */ 740, 741, 573, 748, 750, 754, 759, 755, 753, 766, + /* 200 */ 771, 808, 799, 810, 780, 806, 819, 824, 812, 814, + /* 210 */ 833, 793, 821, 825, 836, 849, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, @@ -614,7 +618,7 @@ static const YYACTIONTYPE yy_default[] = { /* 20 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 30 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 40 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, - /* 50 */ 1205, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 50 */ 1152, 1205, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 60 */ 1152, 1152, 1152, 1152, 1152, 1152, 1203, 1332, 1152, 1464, /* 70 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 80 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, @@ -631,11 +635,11 @@ static const YYACTIONTYPE yy_default[] = { /* 190 */ 1526, 1152, 1152, 1504, 1152, 1152, 1504, 1152, 1345, 1152, /* 200 */ 1152, 1203, 1152, 1203, 1152, 1259, 1152, 1152, 1152, 1203, /* 210 */ 1152, 1360, 1293, 1293, 1206, 1157, 1152, 1152, 1152, 1152, - /* 220 */ 1152, 1152, 1152, 1152, 1152, 1428, 1517, 1516, 1427, 1441, - /* 230 */ 1440, 1439, 1152, 1152, 1152, 1152, 1422, 1423, 1421, 1420, - /* 240 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, - /* 250 */ 1152, 1152, 1152, 1152, 1465, 1152, 1527, 1531, 1152, 1152, - /* 260 */ 1152, 1405, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 220 */ 1152, 1152, 1152, 1152, 1152, 1152, 1428, 1517, 1516, 1427, + /* 230 */ 1441, 1440, 1439, 1152, 1152, 1152, 1152, 1422, 1423, 1421, + /* 240 */ 1420, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 250 */ 1152, 1152, 1152, 1152, 1152, 1465, 1152, 1527, 1531, 1152, + /* 260 */ 1152, 1152, 1405, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 270 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 280 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 290 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, @@ -644,11 +648,11 @@ static const YYACTIONTYPE yy_default[] = { /* 320 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 330 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 340 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, - /* 350 */ 1152, 1152, 1304, 1152, 1152, 1152, 1152, 1152, 1152, 1229, - /* 360 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 350 */ 1152, 1152, 1152, 1304, 1152, 1152, 1152, 1152, 1152, 1152, + /* 360 */ 1229, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 370 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, /* 380 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, - /* 390 */ 1152, 1488, 1498, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 390 */ 1152, 1152, 1488, 1498, 1152, 1152, 1152, 1152, 1152, 1152, /* 400 */ 1152, 1152, 1152, 1405, 1152, 1515, 1152, 1474, 1470, 1152, /* 410 */ 1152, 1466, 1152, 1152, 1525, 1152, 1152, 1152, 1152, 1152, /* 420 */ 1152, 1152, 1152, 1460, 1152, 1152, 1152, 1152, 1152, 1152, @@ -1424,7 +1428,7 @@ static const char *const yyRuleName[] = { /* 343 */ "partition_by_clause_opt ::= PARTITION BY expression_list", /* 344 */ "twindow_clause_opt ::=", /* 345 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 346 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 346 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", /* 347 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", /* 348 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", /* 349 */ "sliding_opt ::=", @@ -2385,7 +2389,7 @@ static const struct { { 286, -3 }, /* (343) partition_by_clause_opt ::= PARTITION BY expression_list */ { 287, 0 }, /* (344) twindow_clause_opt ::= */ { 287, -6 }, /* (345) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 287, -4 }, /* (346) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 287, -4 }, /* (346) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ { 287, -6 }, /* (347) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { 287, -8 }, /* (348) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ { 247, 0 }, /* (349) sliding_opt ::= */ @@ -3590,7 +3594,7 @@ static YYACTIONTYPE yy_reduce( case 345: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ { yymsp[-5].minor.yy104 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } break; - case 346: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + case 346: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ { yymsp[-3].minor.yy104 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } break; case 347: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 6dc3b53ae3..72670bb79a 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -54,7 +54,7 @@ private: static const int max_err_len = 1024; bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = doParse(&cxt_, &query_); + int32_t code = parse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == parseCode); @@ -63,7 +63,7 @@ private: return false; } parsedAstStr_ = toString(query_->pRoot); - code = doTranslate(&cxt_, query_); + code = translate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == translateCode); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f57f476d51..819eb6d699 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - nodesWalkList(pExprs, doNameExpr, NULL); + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx return TSDB_CODE_OUT_OF_MEMORY; } - nodesWalkList(pExprs, doCreateColumn, &cxt); + nodesWalkExprs(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); return cxt.errCode; @@ -477,6 +477,18 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm return code; } +static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindowNode* pState, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_STATE; + pWindow->pStateExpr = nodesCloneNode(pState->pExpr); + + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); if (NULL == pWindow) { @@ -525,6 +537,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele } switch (nodeType(pSelect->pWindow)) { + case QUERY_NODE_STATE_WINDOW: + return createWindowLogicNodeByState(pCxt, (SStateWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_SESSION_WINDOW: return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: @@ -642,6 +656,29 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } + SNodeList* pCols = NULL; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, &pCols); + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { + pPartition->node.pTargets = nodesCloneList(pCols); + if (NULL == pPartition->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + pPartition->pPartitionKeys = nodesCloneList(pSelect->pPartitionByList); + if (NULL == pPartition->pPartitionKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pPartition; + } else { + nodesDestroyNode(pPartition); + } + + return code; return TSDB_CODE_SUCCESS; } @@ -650,7 +687,35 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + // set grouyp keys, agg funcs and having conditions + pAgg->pGroupKeys = nodesCloneList(pSelect->pProjectionList); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_SELECT); + } + + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; } static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..965c5c9579 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -176,6 +176,10 @@ static int16_t getUnsetSlotId(const SArray* pSlotIdsInfo) { } static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) { + if (NULL == pList) { + return TSDB_CODE_SUCCESS; + } + int32_t code = TSDB_CODE_SUCCESS; SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; @@ -219,6 +223,23 @@ static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDat return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false); } +static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlockDescNode* pDataBlockDesc) { + if (NULL == pNode || NULL == *pNode) { + return TSDB_CODE_SUCCESS; + } + + SNodeList* pList = NULL; + int32_t code = nodesListMakeAppend(&pList, *pNode); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pList, pDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + *pNode = nodesListGetNode(pList, 0); + } + nodesClearList(pList); + return code; +} + static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true); } @@ -244,6 +265,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { } // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { + pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR; return DEAL_RES_ERROR; } ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; @@ -264,7 +286,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkNode(pRes, doSetSlotId, &cxt); + nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return cxt.errCode; @@ -285,7 +307,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkList(pRes, doSetSlotId, &cxt); + nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return cxt.errCode; @@ -419,7 +441,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + SQueryNodeAddr addr = { .nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet }; taosArrayPush(pCxt->pExecNodeList, &addr); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; @@ -606,7 +628,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; - nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { nodesDestroyList(cxt.pPrecalcExprs); *pPrecalcExprs = NULL; @@ -614,6 +636,25 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN return cxt.errCode; } +static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeList** pPrecalcExprs, SNode** pRewritten) { + if (NULL == pNode) { + return TSDB_CODE_SUCCESS; + } + + SNodeList* pList = NULL; + int32_t code = nodesListMakeAppend(&pList, pNode); + SNodeList* pRewrittenList = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = rewritePrecalcExprs(pCxt, pList, pPrecalcExprs, &pRewrittenList); + } + if (TSDB_CODE_SUCCESS == code) { + *pRewritten = nodesListGetNode(pRewrittenList, 0); + } + nodesClearList(pList); + nodesClearList(pRewrittenList); + return code; +} + static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); if (NULL == pAgg) { @@ -818,6 +859,40 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* return createWindowPhysiNodeFinalize(pCxt, pChildren, &pSession->window, pWindowLogicNode, pPhyNode); } +static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW); + if (NULL == pState) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pPrecalcExprs = NULL; + SNode* pStateKey = NULL; + int32_t code = rewritePrecalcExpr(pCxt, pWindowLogicNode->pStateExpr, &pPrecalcExprs, &pStateKey); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pState->window.pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pState->window.pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pStateKey, &pState->pStateKey); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlot(pCxt, &pState->pStateKey, pState->window.node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pState); + return code; + } + + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pState->window, pWindowLogicNode, pPhyNode); +} + static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { case WINDOW_TYPE_INTERVAL: @@ -825,7 +900,7 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr case WINDOW_TYPE_SESSION: return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: - break; + return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); default: break; } @@ -867,6 +942,41 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren return code; } +static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { + SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); + if (NULL == pPart) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pPartitionKeys = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pPartLogicNode->pPartitionKeys, &pPrecalcExprs, &pPartitionKeys); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pPart->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pPart->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPartitionKeys, &pPart->pPartitionKeys); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pPart->pPartitionKeys, pPart->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pPart; + } else { + nodesDestroyNode(pPart); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -883,6 +993,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_SORT: return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index ad76c8f879..7f79fc6a5a 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, selectConstant) { + setDatabase("root", "test"); + + bind("SELECT 2-1 FROM t1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, stSimple) { setDatabase("root", "test"); @@ -210,6 +217,32 @@ TEST_F(PlannerTest, sessionWindow) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, stateWindow) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM t1 state_window(c1)"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) FROM t1 state_window(c1 + 10)"); + ASSERT_TRUE(run()); +} + +TEST_F(PlannerTest, partitionBy) { + setDatabase("root", "test"); + + bind("SELECT * FROM t1 partition by c1"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) FROM t1 partition by c1"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) FROM t1 partition by c1 group by c2"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) FROM st1 partition by tag1, tag2 interval(10s)"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, orderBy) { setDatabase("root", "test"); @@ -223,6 +256,19 @@ TEST_F(PlannerTest, orderBy) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, distinct) { + setDatabase("root", "test"); + + bind("SELECT distinct c1 FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT distinct c1, c2 + 10 FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT distinct c1 + 10 a FROM t1 order by a"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, limit) { setDatabase("root", "test"); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 573eaed2e6..cfd4a3ec7b 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -107,15 +107,17 @@ typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; int8_t taskType; + int8_t explain; - bool emptyRes; bool queryFetched; bool queryEnd; bool queryContinue; bool queryInQueue; int32_t rspCode; - SQWConnInfo connInfo; + SQWConnInfo ctrlConnInfo; + SQWConnInfo dataConnInfo; + int8_t events[QW_EVENT_MAX]; qTaskInfo_t taskHandle; diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 313b9459e2..c0f4d6d157 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -23,7 +23,7 @@ extern "C" { #include "qworkerInt.h" #include "dataSinkMgt.h" -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType); +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); @@ -38,6 +38,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComple int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn); int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code); int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 42d10a2361..a83fcc26f2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -432,8 +432,10 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { - tmsgReleaseHandle(ctx->connInfo.handle, TAOS_CONN_SERVER); - ctx->connInfo.handle = NULL; + tmsgReleaseHandle(ctx->ctrlConnInfo.handle, TAOS_CONN_SERVER); + ctx->ctrlConnInfo.handle = NULL; + + // NO need to release dataConnInfo qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); @@ -537,6 +539,29 @@ int32_t qwDropTask(QW_FPARAMS_DEF) { return TSDB_CODE_SUCCESS; } + +int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + qTaskInfo_t *taskHandle = &ctx->taskHandle; + + if (TASK_TYPE_TEMP == ctx->taskType) { + if (ctx->explain) { + SExplainExecInfo *execInfo = NULL; + int32_t resNum = 0; + QW_ERR_RET(qGetExplainExecInfo(ctx->taskHandle, &resNum, &execInfo)); + + SQWConnInfo connInfo = {0}; + connInfo.handle = ctx->ctrlConnInfo.handle; + + QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum)); + } + + qwFreeTaskHandle(QW_FPARAMS(), taskHandle); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; @@ -562,10 +587,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%"PRIu64, useconds); dsEndPut(sinkHandle, useconds); - - if (TASK_TYPE_TEMP == ctx->taskType) { - qwFreeTaskHandle(QW_FPARAMS(), taskHandle); - } + + QW_ERR_RET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); if (queryEnd) { *queryEnd = true; @@ -658,19 +681,6 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void bool queryEnd = false; int32_t code = 0; - if (ctx->emptyRes) { - QW_TASK_DLOG_E("query end with empty result"); - - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED); - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - - *rspMsg = rsp; - *dataLen = 0; - pOutput->queryEnd = true; - - return TSDB_CODE_SUCCESS; - } - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { @@ -760,12 +770,12 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - dropConnection = &ctx->connInfo; + dropConnection = &ctx->ctrlConnInfo; QW_ERR_JRET(qwDropTask(QW_FPARAMS())); dropConnection = NULL; - qwBuildAndSendDropRsp(&ctx->connInfo, code); - QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); + qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); break; @@ -798,12 +808,12 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - dropConnection = &ctx->connInfo; + dropConnection = &ctx->ctrlConnInfo; QW_ERR_JRET(qwDropTask(QW_FPARAMS())); dropConnection = NULL; - qwBuildAndSendDropRsp(&ctx->connInfo, code); - QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); + qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } @@ -863,17 +873,13 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } if (QW_PHASE_POST_QUERY == phase) { - if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { - ctx->emptyRes = true; - } - #if 0 if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { readyConnection = &ctx->connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); } #else - connInfo.handle = ctx->connInfo.handle; + connInfo.handle = ctx->ctrlConnInfo.handle; readyConnection = &connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); @@ -886,8 +892,8 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - qwBuildAndSendDropRsp(&ctx->connInfo, code); - QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); + qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(qwDropTask(QW_FPARAMS())); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); @@ -931,7 +937,7 @@ _return: QW_RET(code); } -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { int32_t code = 0; bool queryRsped = false; struct SSubplan *plan = NULL; @@ -947,9 +953,10 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); atomic_store_8(&ctx->taskType, taskType); + atomic_store_8(&ctx->explain, explain); - atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); - atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + atomic_store_ptr(&ctx->ctrlConnInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->ctrlConnInfo.ahandle, qwMsg->connInfo.ahandle); QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -1011,8 +1018,8 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (ctx->phase == QW_PHASE_PRE_QUERY) { - ctx->connInfo.handle == qwMsg->connInfo.handle; - ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; + ctx->ctrlConnInfo.handle == qwMsg->connInfo.handle; + ctx->ctrlConnInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); needRsp = false; QW_TASK_DLOG_E("ready msg will not rsp now"); @@ -1089,10 +1096,13 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (rsp) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); - atomic_store_8((int8_t*)&ctx->queryEnd, qComplete); + if (qComplete) { + atomic_store_8((int8_t*)&ctx->queryEnd, true); + } - qwMsg->connInfo = ctx->connInfo; + qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); @@ -1113,7 +1123,7 @@ _return: qwFreeFetchRsp(rsp); rsp = NULL; - qwMsg->connInfo = ctx->connInfo; + qwMsg->connInfo = ctx->dataConnInfo; qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } @@ -1151,14 +1161,17 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { - atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); - atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + atomic_store_ptr(&ctx->dataConnInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->dataConnInfo.ahandle, qwMsg->connInfo.ahandle); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); - atomic_store_8((int8_t*)&ctx->queryEnd, qComplete); + if (qComplete) { + atomic_store_8((int8_t*)&ctx->queryEnd, true); + } } if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { @@ -1236,8 +1249,8 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (!rsped) { - ctx->connInfo.handle = qwMsg->connInfo.handle; - ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; + ctx->ctrlConnInfo.handle = qwMsg->connInfo.handle; + ctx->ctrlConnInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 80ae013653..bf8f97aa4c 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -85,6 +85,27 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) { return TSDB_CODE_SUCCESS; } +int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num) { + SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo}; + + int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + tSerializeSExplainRsp(pRsp, contLen, &rsp); + + SRpcMsg rpcRsp = { + .msgType = TDMT_VND_EXPLAIN_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .pCont = pRsp, + .contLen = contLen, + .code = 0, + }; + + tmsgSendRsp(&rpcRsp); + + return TSDB_CODE_SUCCESS; +} + int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); void *pRsp = rpcMallocCont(contLen); @@ -327,7 +348,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql); taosMemoryFreeClear(sql); - QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); + QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType, msg->explain)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 3f41ad875c..e51115e9c0 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -75,6 +75,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) p = getVectorDoubleValue_FLOAT; } else if (srcType == TSDB_DATA_TYPE_DOUBLE) { p = getVectorDoubleValue_DOUBLE; + } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { + p = getVectorDoubleValue_BIGINT; } else { assert(0); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 05f9a36551..393f143e5f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { resGroup = taosArrayInit(4, sizeof(SFilterGroup)); SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); @@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (LOGIC_COND_TYPE_OR == node->condType) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); cell = cell->pNext; @@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { filterInitUnitsFields(info); SFltBuildGroupCtx tctx = {.info = info, .group = group}; - nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); @@ -3314,7 +3314,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t -int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { +int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { + SFilterInfo *info = NULL; SFilterRange ra = {0}; SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); @@ -3566,7 +3567,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat); FLT_RET(pStat->code); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 3a6f28141a..f1f4e91a3c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -10,7 +10,8 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type - || OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type) { + || OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type + || OP_TYPE_MINUS == type) { return 1; } @@ -669,7 +670,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); + nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; @@ -693,7 +694,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); if (pDst) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 6971377f2e..8b738d58c0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -12,7 +12,7 @@ typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); typedef int16_t (*_len_fn)(char *, int32_t); /** Math functions **/ -double tlog(double v, double base) { +static double tlog(double v, double base) { return log(v) / log(base); } @@ -113,7 +113,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu return TSDB_CODE_SUCCESS; } -int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { +static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { return TSDB_CODE_FAILED; @@ -138,7 +138,7 @@ int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarPa return TSDB_CODE_SUCCESS; } -int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn_2 valFn) { +static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn_2 valFn) { if (inputNum != 2 || !IS_NUMERIC_TYPE(GET_PARAM_TYPE(&pInput[0])) || !IS_NUMERIC_TYPE(GET_PARAM_TYPE(&pInput[1]))) { return TSDB_CODE_FAILED; } @@ -167,7 +167,7 @@ int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } -int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _float_fn f1, _double_fn d1) { +static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _float_fn f1, _double_fn d1) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { return TSDB_CODE_FAILED; @@ -215,11 +215,11 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p } /** String functions **/ -int16_t tlength(char *input, int32_t type) { +static int16_t tlength(char *input, int32_t type) { return varDataLen(input); } -int16_t tcharlength(char *input, int32_t type) { +static int16_t tcharlength(char *input, int32_t type) { if (type == TSDB_DATA_TYPE_VARCHAR) { return varDataLen(input); } else { //NCHAR @@ -227,7 +227,7 @@ int16_t tcharlength(char *input, int32_t type) { } } -void tltrim(char *input, char *output, int32_t type, int32_t charLen) { +static void tltrim(char *input, char *output, int32_t type, int32_t charLen) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = 0; i < charLen; ++i) { @@ -257,7 +257,7 @@ void tltrim(char *input, char *output, int32_t type, int32_t charLen) { varDataSetLen(output, resLen); } -void trtrim(char *input, char *output, int32_t type, int32_t charLen) { +static void trtrim(char *input, char *output, int32_t type, int32_t charLen) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = charLen - 1; i >= 0; --i) { @@ -286,7 +286,7 @@ void trtrim(char *input, char *output, int32_t type, int32_t charLen) { varDataSetLen(output, resLen); } -int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) { +static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; @@ -312,12 +312,22 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p return TSDB_CODE_SUCCESS; } -static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) { - pOutputData->pData = taosMemoryCalloc(len, sizeof(char)); - pOutputData->info.type = type; - pOutputData->info.bytes = len; - pOutputData->varmeta.length = len; - pOutputData->varmeta.allocLen = len; +static int32_t concatCopyHelper(const char *input, char *output, bool hasNcharCol, int32_t type, int16_t *dataLen) { + if (hasNcharCol && type == TSDB_DATA_TYPE_VARCHAR) { + TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); + bool ret = taosMbsToUcs4(varDataVal(input), varDataLen(input), newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, NULL); + if (!ret) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + memcpy(varDataVal(output) + *dataLen, newBuf, varDataLen(input) * TSDB_NCHAR_SIZE); + *dataLen += varDataLen(input) * TSDB_NCHAR_SIZE; + taosMemoryFree(newBuf); + } else { + memcpy(varDataVal(output) + *dataLen, varDataVal(input), varDataLen(input)); + *dataLen += varDataLen(input); + } + return TSDB_CODE_SUCCESS; } int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { @@ -332,11 +342,15 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t inputLen = 0; int32_t numOfRows = 0; + bool hasNcharCol = false; for (int32_t i = 0; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + int32_t type = GET_PARAM_TYPE(&pInput[i]); + if (!IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; } + if (type == TSDB_DATA_TYPE_NCHAR) { + hasNcharCol = true; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } @@ -344,8 +358,12 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; input[i] = pInputData[i]->pData; + int32_t factor = 1; + if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { + factor = TSDB_NCHAR_SIZE; + } if (pInput[i].numOfRows == 1) { - inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * factor * numOfRows; } else { inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; } @@ -371,8 +389,10 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); - dataLen += varDataLen(input[i]); + int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } if (pInput[i].numOfRows != 1) { input[i] += varDataTLen(input[i]); } @@ -390,6 +410,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu return TSDB_CODE_SUCCESS; } + int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator return TSDB_CODE_FAILED; @@ -402,27 +423,34 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t inputLen = 0; int32_t numOfRows = 0; + bool hasNcharCol = false; for (int32_t i = 1; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) { + int32_t type = GET_PARAM_TYPE(&pInput[i]); + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i]))) { return TSDB_CODE_FAILED; } + if (type == TSDB_DATA_TYPE_NCHAR) { + hasNcharCol = true; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } } for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; + input[i] = pInputData[i]->pData; + int32_t factor = 1; + if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { + factor = TSDB_NCHAR_SIZE; + } if (i == 0) { // calculate required separator space - int32_t factor = (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1; inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor; } else if (pInput[i].numOfRows == 1) { - inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * factor; } else { inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; } - input[i] = pInputData[i]->pData; } int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; @@ -441,8 +469,11 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p continue; } - memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); - dataLen += varDataLen(input[i]); + int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + if (pInput[i].numOfRows != 1) { input[i] += varDataTLen(input[i]); } @@ -450,8 +481,10 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p if (i < inputNum - 1) { //insert the separator char *sep = pInputData[0]->pData; - memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); - dataLen += varDataLen(sep); + int32_t ret = concatCopyHelper(sep, output, hasNcharCol, GET_PARAM_TYPE(&pInput[0]), &dataLen); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } } } varDataSetLen(output, dataLen); @@ -467,7 +500,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p return TSDB_CODE_SUCCESS; } -int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { +static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; @@ -512,7 +545,7 @@ int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam } -int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { +static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; @@ -576,7 +609,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu char *input = pInputData->pData; char *output = NULL; - int32_t outputLen = pInputData->varmeta.length; + int32_t outputLen = pInputData->varmeta.length * pInput->numOfRows; char *outputBuf = taosMemoryCalloc(outputLen, 1); output = outputBuf; @@ -597,12 +630,12 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu startPosBytes = MAX(startPosBytes, 0); } - subLen = MIN(subLen, len - startPosBytes); - if (subLen > 0) { - memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen); + int32_t resLen = MIN(subLen, len - startPosBytes); + if (resLen > 0) { + memcpy(varDataVal(output), varDataVal(input) + startPosBytes, resLen); } - varDataSetLen(output, subLen); + varDataSetLen(output, resLen); colDataAppend(pOutputData, i , output, false); input += varDataTLen(input); output += varDataTLen(output); @@ -687,6 +720,7 @@ int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return doLengthFunction(pInput, inputNum, pOutput, tcharlength); } +#if 0 static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: @@ -751,6 +785,7 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf default: assert(0); } } +#endif bool getTimePseudoFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(int64_t); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index eb242b3e17..8ba8717854 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -772,6 +772,32 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam doReleaseVec(pRightCol, rightConvert); } +void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + pOut->numOfRows = pLeft->numOfRows; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : (pLeft->numOfRows - 1); + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + int32_t leftConvert = 0; + SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); + + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); + + double *output = (double *)pOutputCol->pData; + for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { + *output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i); + } + + pOutputCol->hasNull = pLeftCol->hasNull; + if (pOutputCol->hasNull) { + memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); + } + + doReleaseVec(pLeftCol, leftConvert); +} + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { #if 0 int32_t len = pLeft->bytes + pRight->bytes; @@ -1101,6 +1127,8 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMathDivide; case OP_TYPE_MOD: return vectorMathRemainder; + case OP_TYPE_MINUS: + return vectorMathMinus; case OP_TYPE_GREATER_THAN: return vectorGreater; case OP_TYPE_GREATER_EQUAL: @@ -1139,4 +1167,4 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { assert(0); return NULL; } -} \ No newline at end of file +} diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 1a82c8f955..1e6978d889 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -234,15 +234,16 @@ TEST(timerangeTest, greater) { flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); - SFilterInfo *filter = NULL; - int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); - ASSERT_EQ(code, 0); + //SFilterInfo *filter = NULL; + //int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + //ASSERT_EQ(code, 0); STimeWindow win = {0}; - code = filterGetTimeRange(filter, &win); + bool isStrict = false; + int32_t code = filterGetTimeRange(opNode1, &win, &isStrict); ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, INT64_MAX); - filterFreeInfo(filter); + //filterFreeInfo(filter); nodesDestroyNode(opNode1); } @@ -263,15 +264,16 @@ TEST(timerangeTest, greater_and_lower) { flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); - SFilterInfo *filter = NULL; - int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); - ASSERT_EQ(code, 0); + //SFilterInfo *filter = NULL; + //int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + //ASSERT_EQ(code, 0); STimeWindow win = {0}; - code = filterGetTimeRange(filter, &win); + bool isStrict = false; + int32_t code = filterGetTimeRange(logicNode, &win, &isStrict); ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, tbig); - filterFreeInfo(filter); + //filterFreeInfo(filter); nodesDestroyNode(logicNode); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 22bd039219..62a96b6438 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -26,6 +26,7 @@ extern "C" { #include "scheduler.h" #include "thash.h" #include "trpc.h" +#include "command.h" #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 @@ -165,6 +166,7 @@ typedef struct SSchJob { SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* + SExplainCtx *explainCtx; int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; @@ -211,6 +213,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_NEED_FETCH(_job) SCH_IS_QUERY_JOB(_job) #define SCH_IS_WAIT_ALL_JOB(_job) (!SCH_IS_QUERY_JOB(_job)) #define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job)) +#define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) @@ -251,6 +254,8 @@ int32_t schFetchFromRemote(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId); int32_t schCloneSMsgSendInfo(void *src, void **dst); +int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); +void schFreeJobImpl(void *job); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c4186bc511..51ee1fedae 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -67,6 +67,81 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * return TSDB_CODE_SUCCESS; } +int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, + int64_t startTs, bool syncSchedule) { + int32_t code = 0; + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); + if (NULL == pJob) { + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->attr.explainMode = pDag->explainInfo.mode; + pJob->attr.syncSchedule = syncSchedule; + pJob->transport = transport; + pJob->sql = sql; + + if (pNodeList != NULL) { + pJob->nodeList = taosArrayDup(pNodeList); + } + + SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + SCH_ERR_JRET(qExecExplainBegin(pDag, &pJob->explainCtx, startTs)); + } + + pJob->execTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == pJob->execTasks) { + SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->succTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == pJob->succTasks) { + SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->failTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == pJob->failTasks) { + SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + tsem_init(&pJob->rspSem, 0, 0); + + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); + } + + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + + pJob->status = JOB_TASK_STATUS_NOT_START; + + *pSchJob = pJob; + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeJobImpl(pJob); + SCH_RET(code); +} + + void schFreeRpcCtx(SRpcCtx *pCtx) { if (NULL == pCtx) { return; @@ -124,6 +199,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m int32_t reqMsgType = msgType - 1; switch (msgType) { case TDMT_SCH_LINK_BROKEN: + case TDMT_VND_EXPLAIN_RSP: return TSDB_CODE_SUCCESS; case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { @@ -969,6 +1045,19 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code)); } +int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { + SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + + atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); + atomic_store_ptr(&pJob->resData, pRsp); + + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + + schProcessOnDataFetched(pJob); + + return TSDB_CODE_SUCCESS; +} + // Note: no more task error processing, handled in function internal int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { @@ -987,7 +1076,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch case TDMT_VND_CREATE_TABLE_RSP: { SVCreateTbBatchRsp batchRsp = {0}; if (msg) { - tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp); + SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp)); if (batchRsp.rspList) { int32_t num = taosArrayGetSize(batchRsp.rspList); for (int32_t i = 0; i < num; ++i) { @@ -1025,7 +1114,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch case TDMT_VND_QUERY_RSP: { SQueryTableRsp rsp = {0}; if (msg) { - tDeserializeSQueryTableRsp(msg, msgSize, &rsp); + SCH_ERR_JRET(tDeserializeSQueryTableRsp(msg, msgSize, &rsp)); SCH_ERR_JRET(rsp.code); } @@ -1051,6 +1140,36 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } + case TDMT_VND_EXPLAIN_RSP: { + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (!SCH_IS_EXPLAIN_JOB(pJob)) { + SCH_TASK_ELOG("invalid msg received for none explain query, msg type:%s", TMSG_INFO(msgType)); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (pJob->resData) { + SCH_TASK_ELOG("explain result is already generated, res:%p", pJob->resData); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SExplainRsp rsp = {0}; + if (tDeserializeSExplainRsp(msg, msgSize, &rsp)) { + taosMemoryFree(rsp.subplanInfo); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qExplainUpdateExecInfo(pJob->explainCtx, &rsp, pTask->plan->id.groupId, &pRsp)); + + if (pRsp) { + SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } + break; + } case TDMT_VND_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -1059,6 +1178,24 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + if (SCH_IS_EXPLAIN_JOB(pJob)) { + if (rsp->completed) { + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp)); + if (pRsp) { + SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } + + return TSDB_CODE_SUCCESS; + } + + atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + + SCH_ERR_JRET(schFetchFromRemote(pJob)); + + return TSDB_CODE_SUCCESS; + } + if (pJob->resData) { SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); taosMemoryFreeClear(rsp); @@ -1159,6 +1296,10 @@ int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); } +int32_t schHandleExplainCallback(void *param, const SDataBuf *pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_EXPLAIN_RSP, code); +} + int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); @@ -1248,6 +1389,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_RES_READY: *fp = schHandleReadyCallback; break; + case TDMT_VND_EXPLAIN: + *fp = schHandleExplainCallback; + break; case TDMT_VND_FETCH: *fp = schHandleFetchCallback; break; @@ -1268,6 +1412,43 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } +int32_t schGenerateTaskCallBackAHandle(SSchJob *pJob, SSchTask *pTask, int32_t msgType, SMsgSendInfo **pMsgSendInfo) { + int32_t code = 0; + SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == msgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = SCH_TASK_ID(pTask); + param->transport = pJob->transport; + + msgSendInfo->param = param; + msgSendInfo->fp = fp; + + *pMsgSendInfo = msgSendInfo; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFree(param); + taosMemoryFree(msgSendInfo); + + SCH_RET(code); +} + void schFreeRpcCtxVal(const void *arg) { if (NULL == arg) { return; @@ -1354,8 +1535,8 @@ _return: int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { int32_t code = 0; - SSchTaskCallbackParam *param = NULL; - SMsgSendInfo *pMsgSendInfo = NULL; + SMsgSendInfo *pReadyMsgSendInfo = NULL; + SMsgSendInfo *pExplainMsgSendInfo = NULL; pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { @@ -1363,31 +1544,18 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(schGenerateTaskCallBackAHandle(pJob, pTask, TDMT_VND_RES_READY, &pReadyMsgSendInfo)); + SCH_ERR_JRET(schGenerateTaskCallBackAHandle(pJob, pTask, TDMT_VND_EXPLAIN, &pExplainMsgSendInfo)); + + int32_t msgType = TDMT_VND_RES_READY_RSP; + SRpcCtxVal ctxVal = {.val = pReadyMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { + SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); - if (NULL == param) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - int32_t msgType = TDMT_VND_RES_READY_RSP; - __async_send_cb_fn_t fp = NULL; - SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_RES_READY, &fp)); - - param->queryId = pJob->queryId; - param->refId = pJob->refId; - param->taskId = SCH_TASK_ID(pTask); - param->transport = pJob->transport; - - pMsgSendInfo->param = param; - pMsgSendInfo->fp = fp; - - SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + msgType = TDMT_VND_EXPLAIN_RSP; + ctxVal.val = pExplainMsgSendInfo; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1400,8 +1568,16 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { _return: taosHashCleanup(pCtx->args); - taosMemoryFreeClear(param); - taosMemoryFreeClear(pMsgSendInfo); + + if (pReadyMsgSendInfo) { + taosMemoryFreeClear(pReadyMsgSendInfo->param); + taosMemoryFreeClear(pReadyMsgSendInfo); + } + + if (pExplainMsgSendInfo) { + taosMemoryFreeClear(pExplainMsgSendInfo->param); + taosMemoryFreeClear(pExplainMsgSendInfo); + } SCH_RET(code); } @@ -1584,32 +1760,13 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + SMsgSendInfo *pMsgSendInfo = NULL; + SCH_ERR_JRET(schGenerateTaskCallBackAHandle(pJob, pTask, msgType, &pMsgSendInfo)); - SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); - if (NULL == param) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - __async_send_cb_fn_t fp = NULL; - SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - - param->queryId = pJob->queryId; - param->refId = pJob->refId; - param->taskId = SCH_TASK_ID(pTask); - param->transport = trans->transInst; - - pMsgSendInfo->param = param; pMsgSendInfo->msgInfo.pData = msg; pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgInfo.handle = trans->transHandle; pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = fp; qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, pJob->refId, @@ -1626,8 +1783,11 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet _return: - taosMemoryFreeClear(param); - taosMemoryFreeClear(pMsgSendInfo); + if (pMsgSendInfo) { + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); + } + SCH_RET(code); } @@ -1769,6 +1929,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->taskId = htobe64(pTask->taskId); pMsg->refId = htobe64(pJob->refId); pMsg->taskType = TASK_TYPE_TEMP; + pMsg->explain = SCH_IS_EXPLAIN_JOB(pJob); pMsg->phyLen = htonl(pTask->msgLen); pMsg->sqlLen = htonl(len); @@ -2085,6 +2246,8 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); + qExplainFreeCtx(pJob->explainCtx); + taosMemoryFreeClear(pJob->resData); taosMemoryFreeClear(pJob); @@ -2092,70 +2255,17 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - bool syncSchedule) { + int64_t startTs, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); - if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { + if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pDag->queryId); } int32_t code = 0; - SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); - if (NULL == pJob) { - qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + SSchJob *pJob = NULL; + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); - pJob->attr.explainMode = pDag->explainInfo.mode; - pJob->attr.syncSchedule = syncSchedule; - pJob->transport = transport; - pJob->sql = sql; - - if (pNodeList != NULL) { - pJob->nodeList = taosArrayDup(pNodeList); - } - - SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); - - pJob->execTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == pJob->execTasks) { - SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pJob->succTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == pJob->succTasks) { - SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pJob->failTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == pJob->failTasks) { - SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - tsem_init(&pJob->rspSem, 0, 0); - - int64_t refId = taosAddRef(schMgmt.jobRef, pJob); - if (refId < 0) { - SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); - SCH_ERR_JRET(terrno); - } - - if (NULL == schAcquireJob(refId)) { - SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); - SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); - } - - pJob->refId = refId; - - SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); - - pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); *job = pJob->refId; @@ -2268,7 +2378,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - SQueryResult *pRes) { + int64_t startTs, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2276,7 +2386,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); } SSchJob *job = schAcquireJob(*pJob); @@ -2294,7 +2404,11 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); + if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { + SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); + } else { + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index cf04b06579..5d9322a575 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -951,7 +951,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index dfb5eb35d6..79c72b3a35 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -193,6 +193,7 @@ static void uvHandleReq(SSrvConn* pConn) { transMsg.ahandle = (void*)pHead->ahandle; transMsg.handle = NULL; + // transDestroyBuffer(&pConn->readBuf); transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; if (pConn->status == ConnNormal) { @@ -249,6 +250,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { conn->broken = true; if (conn->status == ConnAcquire) { if (conn->regArg.init) { + tTrace("server conn %p broken, notify server app", conn); STrans* pTransInst = conn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); @@ -270,7 +272,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnSendCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - transClearBuffer(&conn->readBuf); + // transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 7d3c3aa012..cdf0908167 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -23,20 +23,20 @@ #include "tutil.h" typedef struct { - int index; - SEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; + int index; + SEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t * pOverSem; TdThread thread; - void * pRpc; + void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); + // tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + // pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -51,9 +51,9 @@ static void *sendRequest(void *param) { SInfo * pInfo = (SInfo *)param; SRpcMsg rpcMsg = {0}; - tDebug("thread:%d, start to send request", pInfo->index); + tError("thread:%d, start to send request", pInfo->index); - tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); int u100 = 0; int u500 = 0; int u1000 = 0; @@ -68,7 +68,7 @@ static void *sendRequest(void *param) { // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); int64_t end = taosGetTimestampUs() - start; @@ -88,7 +88,7 @@ static void *sendRequest(void *param) { } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); - tDebug("thread:%d, it is over", pInfo->index); + tError("thread:%d, it is over", pInfo->index); tcount++; return NULL; @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - TdThreadAttr thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -124,7 +124,7 @@ int main(int argc, char *argv[]) { rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.connType = TAOS_CONN_CLIENT; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -170,6 +170,10 @@ int main(int argc, char *argv[]) { } } + const char *path = "/tmp/transport/client"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); @@ -178,8 +182,8 @@ int main(int argc, char *argv[]) { return -1; } - tInfo("client is initialized"); - tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); + tError("client is initialized"); + tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; @@ -208,8 +212,9 @@ int main(int argc, char *argv[]) { endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; float usedTime = (endTime - startTime) / 1000.0f; // mseconds - tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, + msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 3a086371b0..8ed3bbc960 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -125,7 +125,7 @@ int main(int argc, char *argv[]) { rpcInit.idleTime = 2 * 1500; rpcInit.afp = retrieveAuthInfo; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -160,6 +160,11 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; + + const char *path = "/tmp/transport/server"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b2332aded7..041aec4054 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -28,11 +28,11 @@ * * BOOLEAN Compression Algorithm: * We provide two methods for compress boolean types. Because boolean types in C - * code are char bytes with 0 and 1 values only, only one bit can used to discrimenate + * code are char bytes with 0 and 1 values only, only one bit can used to discriminate * the values. * 1. The first method is using only 1 bit to represent the boolean value with 1 for * true and 0 for false. Then the compression rate is 1/8. - * 2. The second method is using run length encoding (RLE) methods. This methos works + * 2. The second method is using run length encoding (RLE) methods. This method works * better when there are a lot of consecutive true values or false values. * * STRING Compression Algorithm: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 90fa624a8d..38cd5fad4b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -432,6 +432,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status erro TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") +//planner +TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") + #ifdef TAOS_ERROR_C }; #endif diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 88e5c12770..809d008aa7 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -311,10 +311,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - return -1; - } // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { @@ -355,6 +351,11 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + return -1; + } + pushfrontNodeInEntryList(pe, pNewNode); assert(pe->next != NULL); @@ -368,9 +369,12 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } else { // not support the update operation, return error if (pHashObj->enableUpdate) { + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + return -1; + } + doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); - } else { - FREE_HASH_NODE(pNewNode); } taosHashEntryWUnlock(pHashObj, pe); diff --git a/source/util/src/tlockfree.c b/source/util/src/tlockfree.c index 8edcca0c16..a755a67cc8 100644 --- a/source/util/src/tlockfree.c +++ b/source/util/src/tlockfree.c @@ -53,6 +53,21 @@ void taosWLockLatch(SRWLatch *pLatch) { } } +int32_t taosWTryLockLatch(SRWLatch *pLatch) { + SRWLatch oLatch, nLatch; + oLatch = atomic_load_32(pLatch); + if (oLatch) { + return -1; + } + + nLatch = oLatch | TD_RWLATCH_WRITE_FLAG; + if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) { + return 0; + } + + return -1; +} + void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); } void taosRLockLatch(SRWLatch *pLatch) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 563acfec78..46e2d567ce 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -747,4 +747,4 @@ void taosSetAllDebugFlag(int32_t flag) { fsDebugFlag = flag; uInfo("all debug flag are set to %d", flag); -} \ No newline at end of file +} diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 600413068c..139c35de45 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -15,7 +15,9 @@ #define _DEFAULT_SOURCE #include "tprocess.h" +#include "taos.h" #include "taoserror.h" +#include "thash.h" #include "tlog.h" #include "tqueue.h" @@ -48,8 +50,9 @@ typedef struct SProcObj { ProcFreeFp parentFreeHeadFp; ProcMallocFp parentMallocBodyFp; ProcFreeFp parentFreeBodyFp; - void *pParent; + void *parent; const char *name; + SHashObj *hash; int32_t pid; bool isChild; bool stopFlag; @@ -151,8 +154,8 @@ static void taosProcCleanupQueue(SProcQueue *pQueue) { #endif } -static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody, - int32_t rawBodyLen, ProcFuncType ftype) { +static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, + const char *pBody, int32_t rawBodyLen, int64_t handle, ProcFuncType ftype) { const int32_t headLen = CEIL8(rawHeadLen); const int32_t bodyLen = CEIL8(rawBodyLen); const int32_t fullLen = headLen + bodyLen + 8; @@ -164,6 +167,14 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t return -1; } + if (handle != 0 && ftype == PROC_REQ) { + if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handle, sizeof(int64_t)) != 0) { + taosThreadMutexUnlock(&pQueue->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + const int32_t pos = pQueue->tail; if (pQueue->tail < pQueue->total) { *(int16_t *)(pQueue->pBuffer + pQueue->tail) = headLen; @@ -317,6 +328,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->name = pCfg->name; pProc->pChildQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + cstart, csize); pProc->pParentQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + pstart, psize); + pProc->hash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { taosProcCleanupQueue(pProc->pChildQueue); taosMemoryFree(pProc); @@ -324,7 +336,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { } pProc->name = pCfg->name; - pProc->pParent = pCfg->pParent; + pProc->parent = pCfg->parent; pProc->childMallocHeadFp = pCfg->childMallocHeadFp; pProc->childFreeHeadFp = pCfg->childFreeHeadFp; pProc->childMallocBodyFp = pCfg->childMallocBodyFp; @@ -384,7 +396,7 @@ static void taosProcThreadLoop(SProcObj *pProc) { taosMsleep(1); continue; } else { - (*consumeFp)(pProc->pParent, pHead, headLen, pBody, bodyLen, ftype); + (*consumeFp)(pProc->parent, pHead, headLen, pBody, bodyLen, ftype); } } } @@ -424,19 +436,41 @@ void taosProcCleanup(SProcObj *pProc) { taosProcStop(pProc); taosProcCleanupQueue(pProc->pChildQueue); taosProcCleanupQueue(pProc->pParentQueue); + if (pProc->hash != NULL) { + taosHashCleanup(pProc->hash); + pProc->hash = NULL; + } + uDebug("proc:%s, is cleaned up", pProc->name); taosMemoryFree(pProc); } } int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - ProcFuncType ftype) { - return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen, ftype); + void *handle, ProcFuncType ftype) { + return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ftype); +} + +void taosProcRemoveHandle(SProcObj *pProc, void *handle) { + int64_t h = (int64_t)handle; + taosThreadMutexLock(&pProc->pChildQueue->mutex); + taosHashRemove(pProc->hash, &h, sizeof(int64_t)); + taosThreadMutexUnlock(&pProc->pChildQueue->mutex); +} + +void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) { + taosThreadMutexLock(&pProc->pChildQueue->mutex); + void *h = taosHashIterate(pProc->hash, NULL); + while (h != NULL) { + void *handle = *((void **)h); + (*HandleFp)(handle); + } + taosThreadMutexUnlock(&pProc->pChildQueue->mutex); } void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, ProcFuncType ftype) { - while (taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, ftype) != 0) { + while (taosProcQueuePush(pProc, pProc->pParentQueue, pHead, headLen, pBody, bodyLen, 0, ftype) != 0) { taosMsleep(1); } } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b756976394..5a3ee003f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -41,7 +41,7 @@ # ---- tmq ./test.sh -f tsim/tmq/basic.sim ./test.sh -f tsim/tmq/basic1.sim -#./test.sh -f tsim/tmq/oneTopic.sim +./test.sh -f tsim/tmq/oneTopic.sim #./test.sh -f tsim/tmq/multiTopic.sim # --- stable diff --git a/tests/script/tsim/query/diff.sim b/tests/script/tsim/query/diff.sim new file mode 100644 index 0000000000..ebbd470944 --- /dev/null +++ b/tests/script/tsim/query/diff.sim @@ -0,0 +1,129 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$dbPrefix = db +$tbPrefix = ctb +$mtPrefix = stb +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sleep 100 + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +print ===> select diff(tbcol) from $tb +sql select diff(tbcol) from $tb +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $data11 != 1 then + return -1 +endi + +print =============== step3 +$cc = 4 * 60000 +$ms = 1601481600000 + $cc +print ===> select diff(tbcol) from $tb where ts > $ms +sql select diff(tbcol) from $tb where ts > $ms +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $data11 != 1 then + return -1 +endi + +$cc = 4 * 60000 +$ms = 1601481600000 + $cc +print ===> select diff(tbcol) from $tb where ts <= $ms +sql select diff(tbcol) from $tb where ts <= $ms +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $data11 != 1 then + return -1 +endi + +print =============== step4 +print ===> select diff(tbcol) as b from $tb +sql select diff(tbcol) as b from $tb +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $data11 != 1 then + return -1 +endi + +print =============== step5 +print ===> select diff(tbcol) as b from $tb interval(1m) +sql select diff(tbcol) as b from $tb interval(1m) -x step5 + return -1 +step5: + +print =============== step6 +$cc = 4 * 60000 +$ms = 1601481600000 + $cc +print ===> select diff(tbcol) as b from $tb where ts <= $ms interval(1m) +sql select diff(tbcol) as b from $tb where ts <= $ms interval(1m) -x step6 + return -1 +step6: + +print =============== clear +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index 98f63f42e3..c1d3437e4a 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -79,7 +79,7 @@ loop_test_pos: # session(ts,5a) print ====> select count(*) from dev_001 session(ts,5a) -sql select count(*) from dev_001 session(ts,5a) +sql select _wstartts, count(*) from dev_001 session(ts,5a) print ====> rows: $rows print ====> $data00 $data01 $data02 $data03 $data04 $data05 print ====> $data10 $data11 $data12 $data13 $data14 $data15 @@ -100,7 +100,7 @@ endi print ====> select count(*) from (select * from dev_001) session(ts,5a) -sql select count(*) from (select * from dev_001) session(ts,5a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,5a) if $rows != 15 then return -1 endi @@ -109,7 +109,7 @@ if $data01 != 2 then endi print ====> select count(*) from dev_001 session(ts,1s) -sql select count(*) from dev_001 session(ts,1s) +sql select _wstartts, count(*) from dev_001 session(ts,1s) if $rows != 12 then return -1 endi @@ -118,7 +118,7 @@ if $data01 != 5 then endi print ====> select count(*) from (select * from dev_001) session(ts,1s) -sql select count(*) from (select * from dev_001) session(ts,1s) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1s) if $rows != 12 then return -1 endi @@ -127,7 +127,7 @@ if $data01 != 5 then endi print ====> select count(*) from dev_001 session(ts,1000a) -sql select count(*) from dev_001 session(ts,1000a) +sql select _wstartts, count(*) from dev_001 session(ts,1000a) if $rows != 12 then return -1 endi @@ -136,7 +136,7 @@ if $data01 != 5 then endi print ====> select count(*) from (select * from dev_001) session(ts,1000a) -sql select count(*) from (select * from dev_001) session(ts,1000a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1000a) if $rows != 12 then return -1 endi @@ -145,7 +145,7 @@ if $data01 != 5 then endi print ====> select count(*) from dev_001 session(ts,1m) -sql select count(*) from dev_001 session(ts,1m) +sql select _wstartts, count(*) from dev_001 session(ts,1m) if $rows != 9 then return -1 endi @@ -154,7 +154,7 @@ if $data01 != 8 then endi print ====> select count(*) from (select * from dev_001) session(ts,1m) -sql select count(*) from (select * from dev_001) session(ts,1m) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1m) if $rows != 9 then return -1 endi @@ -163,7 +163,7 @@ if $data01 != 8 then endi print ====> select count(*) from dev_001 session(ts,1h) -sql select count(*) from dev_001 session(ts,1h) +sql select _wstartts, count(*) from dev_001 session(ts,1h) if $rows != 6 then return -1 endi @@ -172,7 +172,7 @@ if $data01 != 11 then endi print ====> select count(*) from (select * from dev_001) session(ts,1h) -sql select count(*) from (select * from dev_001) session(ts,1h) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1h) if $rows != 6 then return -1 endi @@ -181,7 +181,7 @@ if $data01 != 11 then endi print ====> select count(*) from dev_001 session(ts,1d) -sql select count(*) from dev_001 session(ts,1d) +sql select _wstartts, count(*) from dev_001 session(ts,1d) if $rows != 4 then return -1 endi @@ -190,7 +190,7 @@ if $data01 != 13 then endi print ====> select count(*) from (select * from dev_001) session(ts,1d) -sql select count(*) from (select * from dev_001) session(ts,1d) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1d) if $rows != 4 then return -1 endi @@ -199,7 +199,7 @@ if $data01 != 13 then endi print ====> select count(*) from dev_001 session(ts,1w) -sql select count(*) from dev_001 session(ts,1w) +sql select _wstartts, count(*) from dev_001 session(ts,1w) if $rows != 2 then return -1 endi @@ -208,7 +208,7 @@ if $data01 != 15 then endi print ====> select count(*) from (select * from dev_001) session(ts,1w) -sql select count(*) from (select * from dev_001) session(ts,1w) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1w) if $rows != 2 then return -1 endi @@ -216,80 +216,83 @@ if $data01 != 15 then return -1 endi + +print ====> leastsquares not supported yet. print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) -sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) -if $rows != 2 then - return -1 -endi -if $data01 != 13 then - return -1 -endi -if $data02 != 1 then - return -1 -endi -if $data03 != 13 then - return -1 -endi -if $data04 != 7 then - return -1 -endi -if $data05 != 91 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data07 != 13 then - return -1 -endi -if $data08 != @{slop:1.000000, intercept:0.000000}@ then - return -1 -endi -if $data09 != 12 then - return -1 -endi +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi +#if $data09 != 12 then +# return -1 +#endi # $data0-10 != 3.741657387 # $data0-11 != 1 # $data1-11 != 14 print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) -sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) -if $rows != 2 then - return -1 -endi -if $data01 != 13 then - return -1 -endi -if $data02 != 1 then - return -1 -endi -if $data03 != 13 then - return -1 -endi -if $data04 != 7 then - return -1 -endi -if $data05 != 91 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data07 != 13 then - return -1 -endi -if $data08 != @{slop:1.000000, intercept:0.000000}@ then - return -1 -endi +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi -sql_error select * from dev_001 session(ts,1w) -sql_error select count(*) from st session(ts,1w) -sql_error select count(*) from dev_001 group by tagtype session(ts,1w) -sql_error select count(*) from dev_001 session(ts,1n) -sql_error select count(*) from dev_001 session(ts,1y) -sql_error select count(*) from dev_001 session(ts,0s) -sql_error select count(*) from dev_001 session(i,1y) -sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' +print ================> syntax error check not active ================> reactive +#sql_error select * from dev_001 session(ts,1w) +#sql_error select count(*) from st session(ts,1w) +#sql_error select count(*) from dev_001 group by tagtype session(ts,1w) +#sql_error select count(*) from dev_001 session(ts,1n) +#sql_error select count(*) from dev_001 session(ts,1y) +#sql_error select count(*) from dev_001 session(ts,0s) +#sql_error select count(*) from dev_001 session(i,1y) +#sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' print ====> create database d1 precision 'us' sql create database d1 precision 'us' @@ -297,17 +300,19 @@ sql use d1 sql create table dev_001 (ts timestamp ,i timestamp ,j int) sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) print ====> select count(*) from dev_001 session(ts,1u) -sql select count(*) from dev_001 session(ts,1u) +sql select _wstartts, count(*) from dev_001 session(ts,1u) if $rows != 2 then + print expect 2, actual: $rows return -1 endi + if $data01 != 3 then return -1 endi -sql_error select count(*) from dev_001 session(i,1s) -sql create table secondts(ts timestamp,t2 timestamp,i int) -sql_error select count(*) from secondts session(t2,2s) +#sql_error select count(*) from dev_001 session(i,1s) +#sql create table secondts(ts timestamp,t2 timestamp,i int) +#sql_error select count(*) from secondts session(t2,2s) if $loop_test == 0 then print =============== stop and restart taosd diff --git a/tests/script/tsim/query/stddev.sim b/tests/script/tsim/query/stddev.sim new file mode 100644 index 0000000000..01cde31966 --- /dev/null +++ b/tests/script/tsim/query/stddev.sim @@ -0,0 +1,124 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$dbPrefix = db +$tbPrefix = ctb +$mtPrefix = stb +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sleep 100 + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select stddev(tbcol) from $tb +print ===> $data00 +if $data00 != 5.766281297 then + return -1 +endi + +print =============== step3 +$cc = 4 * 60000 +$ms = 1601481600000 + $cc + +print ===> select stddev(tbcol) from $tb where ts <= $ms +sql select stddev(tbcol) from $tb where ts <= $ms +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1.414213562 then + return -1 +endi + +print =============== step4 +sql select stddev(tbcol) as b from $tb +print ===> $data00 +if $data00 != 5.766281297 then + return -1 +endi + +print =============== step5 +sql select stddev(tbcol) as b from $tb interval(1m) +print ===> $data01 +if $data01 != 0.000000000 then + return -1 +endi + +sql select stddev(tbcol) as b from $tb interval(1d) +print ===> $data01 +if $data01 != 5.766281297 then + return -1 +endi + +print =============== step6 +$cc = 4 * 60000 +$ms = 1601481600000 + $cc + +sql select stddev(tbcol) as b from $tb where ts <= $ms interval(1m) +print ===> $data01 +if $data01 != 0.000000000 then + return -1 +endi +if $rows != 5 then + return -1 +endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file