diff --git a/cmake/cmake.define b/cmake/cmake.define index fb6ba8cc2e..1655154506 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -44,7 +44,6 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) SET(COMMON_FLAGS "/W3 /D_WIN32") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) @@ -68,7 +67,13 @@ ELSE () IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ADD_DEFINITIONS("-D_TD_ARM_") ELSE () - ADD_DEFINITIONS("-msse4.2 -mfma") + ADD_DEFINITIONS("-msse4.2") + IF("${FMA_SUPPORT}" MATCHES "true") + MESSAGE(STATUS "turn fma function support on") + ADD_DEFINITIONS("-mfma") + ELSE () + MESSAGE(STATUS "turn fma function support off") + ENDIF() ENDIF () ENDIF () diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 85d11be817..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -28,15 +28,16 @@ typedef int64_t tb_uid_t; #define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_SYSTEM_TABLE = 6, - TSDB_TABLE_MAX = 7 + TSDB_TABLE_MAX = 7 } ETableType; typedef enum { @@ -78,15 +79,15 @@ typedef enum { } ETsdbSmaType; typedef enum { - TSDB_RSMA_RETENTION_0 = 0, - TSDB_RSMA_RETENTION_1 = 1, - TSDB_RSMA_RETENTION_2 = 2, - TSDB_RSMA_RETENTION_MAX = 3 -} ERSmaRetention; + TSDB_RETENTION_L0 = 0, + TSDB_RETENTION_L1 = 1, + TSDB_RETENTION_L2 = 2, + TSDB_RETENTION_MAX = 3 +} ERetentionLevel; extern char *qtypeStr[]; -#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_HTTP 11 #undef TD_DEBUG_PRINT_ROW diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c0eb53d8ca..16653ebfee 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -70,7 +70,7 @@ typedef struct SDataBlockInfo { uint64_t groupId; // no need to serialize int16_t numOfCols; int16_t hasVarCol; - int16_t capacity; + int32_t capacity; } SDataBlockInfo; typedef struct SSDataBlock { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a597123cd7..4d3ad591ce 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -182,7 +182,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u } int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); @@ -224,6 +224,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid); + SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a4c693c2e2..11d0f34c45 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -63,20 +63,20 @@ extern "C" { typedef struct { col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) int8_t type; // column type - int8_t sma; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... + int8_t flags; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON int32_t bytes; // column bytes (0~16M) int32_t offset; // point offset in STpRow after the header part. } STColumn; #pragma pack(pop) #define colType(col) ((col)->type) -#define colSma(col) ((col)->sma) +#define colFlags(col) ((col)->flags) #define colColId(col) ((col)->colId) #define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) #define colSetType(col, t) (colType(col) = (t)) -#define colSetSma(col, s) (colSma(col) = (s)) +#define colSetFlags(col, f) (colFlags(col) = (f)) #define colSetColId(col, id) (colColId(col) = (id)) #define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) @@ -146,7 +146,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -387,6 +387,8 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; + + #define isAllRowsNull(pCol) ((pCol)->len == 0) #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } @@ -480,8 +482,7 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, - TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fc3d575317..f253d31963 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,10 @@ extern char tsCompressor[]; extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; +// internal +extern int32_t tsTransPullupMs; +extern int32_t tsMaRebalanceMs; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b85b8ceb7a..f6160cbbf5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -181,8 +181,8 @@ typedef struct SField { } SField; typedef struct SRetention { - int32_t freq; - int32_t keep; + int64_t freq; + int64_t keep; int8_t freqUnit; int8_t keepUnit; } SRetention; @@ -243,20 +243,11 @@ typedef struct { int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); // for debug int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); -// TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter -int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); -STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter); - typedef struct { int32_t index; // index of failed block in submit blocks int32_t vnode; // vnode index of failed block @@ -283,8 +274,10 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == SCHEMA_SMA_ON) + #define SSCHMEA_TYPE(s) ((s)->type) -#define SSCHMEA_SMA(s) ((s)->sma) +#define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 02d8b76b9b..4002db06ea 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -42,6 +42,7 @@ typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueTy typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq); typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp); +typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type); typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc); @@ -52,6 +53,7 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; + SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -62,6 +64,7 @@ int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); void tmsgSendRsp(const SRpcMsg* pRsp); +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b0439c722c..36bef5e85a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) // sync integration TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f6304401fc..1fa9db9927 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -113,6 +113,9 @@ typedef enum EFunctionType { FUNCTION_TYPE_WENDTS, FUNCTION_TYPE_WDURATION, + // internal function + FUNCTION_TYPE_SELECT_VALUE, + // user defined funcion FUNCTION_TYPE_UDF = 10000 } EFunctionType; @@ -141,6 +144,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsNonstandardSQLFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 453b49e4c6..c94d75338a 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -17,6 +17,7 @@ #define _TD_INDEX_H_ #include "os.h" +#include "taoserror.h" #include "tarray.h" #ifdef __cplusplus @@ -41,11 +42,22 @@ typedef enum { UPDATE_VALUE, // update index column value ADD_INDEX, // add index on specify column DROP_INDEX, // drop existed index - DROP_SATBLE // drop stable + DROP_SATBLE, // drop stable + DEFAULT // query } SIndexOperOnColumn; -typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; +typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType; +typedef enum { + QUERY_TERM = 0, + QUERY_PREFIX, + QUERY_SUFFIX, + QUERY_REGEX, + QUERY_LESS_THAN, + QUERY_LESS_EQUAL, + QUERY_GREATER_THAN, + QUERY_GREATER_EQUAL, + QUERY_RANGE +} EIndexQueryType; /* * create multi query @@ -166,8 +178,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6513a44e88..115bb4349d 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -185,6 +185,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_WINDOW, + QUERY_NODE_LOGIC_PLAN_FILL, QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_PLAN_PARTITION, QUERY_NODE_LOGIC_SUBPLAN, @@ -202,6 +203,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_PARTITION, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a34d4e3c80..a756217c7a 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -102,7 +102,6 @@ typedef struct SWindowLogicNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; int64_t sessionGap; SNode* pTspk; SNode* pStateExpr; @@ -110,6 +109,14 @@ typedef struct SWindowLogicNode { int64_t watermark; } SWindowLogicNode; +typedef struct SFillLogicNode { + SLogicNode node; + EFillMode mode; + SNode* pWStartTs; + SNode* pValues; // SNodeListNode + STimeWindow timeRange; +} SFillLogicNode; + typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; @@ -223,10 +230,12 @@ typedef struct SProjectPhysiNode { typedef struct SJoinPhysiNode { SPhysiNode node; EJoinType joinType; - SNode* pOnConditions; // in or out tuple ? + SNode* pOnConditions; SNodeList* pTargets; } SJoinPhysiNode; +typedef SJoinPhysiNode SSortMergeJoinPhysiNode; + typedef struct SAggPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function @@ -263,9 +272,17 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SFillPhysiNode { + SPhysiNode node; + EFillMode mode; + SNode* pWStartTs; // SColumnNode + SNode* pValues; // SNodeListNode + SNodeList* pTargets; + STimeWindow timeRange; +} SFillPhysiNode; + typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; SNodeList* pPartitionKeys; @@ -340,7 +357,7 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SArray* pPlaceholderValues; + SArray* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5f1ee157e1..91c80f6cf5 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -89,7 +89,7 @@ typedef struct SValueNode { char* p; } datum; int64_t typeData; - char unit; + char unit; } SValueNode; typedef struct SOperatorNode { @@ -209,9 +209,11 @@ typedef enum EFillMode { } EFillMode; typedef struct SFillNode { - ENodeType type; // QUERY_NODE_FILL - EFillMode mode; - SNode* pValues; // SNodeListNode + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode + SNode* pWStartTs; // _wstartts pseudo column + STimeWindow timeRange; } SFillNode; typedef struct SSelectStmt { @@ -300,7 +302,7 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs); int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes); @@ -314,11 +316,11 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -void* nodesGetValueFromNode(SValueNode* pNode); -int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value); -char* nodesGetStrValueFromNode(SValueNode* pNode); -char* getFillModeString(EFillMode mode); -void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); +void* nodesGetValueFromNode(SValueNode* pNode); +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value); +char* nodesGetStrValueFromNode(SValueNode* pNode); +char* getFillModeString(EFillMode mode); +void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index de71192a9b..84fa166881 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" typedef struct SStmtCallback { TAOS_STMT* pStmt; @@ -34,24 +34,26 @@ typedef struct SStmtCallback { typedef struct SParseContext { uint64_t requestId; int32_t acctId; - const char *db; + const char* db; bool topicQuery; - void *pTransporter; + void* pTransporter; SEpSet mgmtEpSet; - const char *pSql; // sql string - size_t sqlLen; // length of the sql string - char *pMsg; // extended error message if exists to help identifying the problem in sql statement. - int32_t msgLen; // max length of the msg - struct SCatalog *pCatalog; - SStmtCallback *pStmtCb; + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + char* pMsg; // extended error message if exists to help identifying the problem in sql statement. + int32_t msgLen; // max length of the msg + struct SCatalog* pCatalog; + SStmtCallback* pStmtCb; + const char* pUser; + bool isSuperUser; } SParseContext; typedef struct SCmdMsgInfo { int16_t msgType; - SEpSet epSet; - void* pMsg; + SEpSet epSet; + void* pMsg; int32_t msgLen; - void* pExtension; // todo remove it soon + void* pExtension; // todo remove it soon } SCmdMsgInfo; typedef enum EQueryExecMode { @@ -63,21 +65,21 @@ typedef enum EQueryExecMode { typedef struct SQuery { EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pDbList; - SArray* pTableList; - bool showRewrite; - int32_t placeholderNum; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; + int32_t placeholderNum; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); -bool isInsertSql(const char* pStr, size_t length); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); @@ -89,14 +91,16 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); -int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); -int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); - +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum); +int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, + int32_t msgBufLen); #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ffe435a8b2..ca10465ed4 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -89,7 +89,7 @@ typedef struct SSyncFSM { struct SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 // abstract definition of log store in raft @@ -149,6 +149,8 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); +void syncGetEpSet(int64_t rid, SEpSet* pEpSet); +int32_t syncGetVgId(int64_t rid); typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, diff --git a/include/os/osDef.h b/include/os/osDef.h index 8da0c2c00c..6f6199de7a 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -62,7 +62,7 @@ extern "C" { #define strncasecmp _strnicmp #define wcsncasecmp _wcsnicmp #define strtok_r strtok_s - #define snprintf _snprintf + // #define snprintf _snprintf #define in_addr_t unsigned long // #define socklen_t int diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 00708f0933..abc752955d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -264,7 +264,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CANT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) +#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) @@ -619,9 +620,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) #define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) +#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) +#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL TAOS_DEF_ERROR_CODE(0, 0x2701) +#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN TAOS_DEF_ERROR_CODE(0, 0x2702) //function #define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h new file mode 100644 index 0000000000..83e6ad0868 --- /dev/null +++ b/include/util/tskiplist2.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_UTIL_SKIPLIST2_H_ +#define _TD_UTIL_SKIPLIST2_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SL_MAX_LEVEL 15 + +typedef struct SSkipList2 SSkipList2; +typedef struct SSLCursor SSLCursor; +typedef struct SSLCfg SSLCfg; +typedef struct SSLNode SSLNode; + +typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); + +// SSkipList2 +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); +int32_t slClose(SSkipList2 *pSl); +int32_t slClear(SSkipList2 *pSl); + +// SSLCursor +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); +int32_t slcClose(SSLCursor *pSlc); +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); +int32_t slcMoveToNext(SSLCursor *pSlc); +int32_t slcMoveToPrev(SSLCursor *pSlc); +int32_t slcMoveToFirst(SSLCursor *pSlc); +int32_t slcMoveToLast(SSLCursor *pSlc); +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); +int32_t slcDrop(SSLCursor *pSlc); + +// struct +struct SSLCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; + tslCmprFn cmprFn; + void *pPool; + void *(*xMalloc)(void *, int32_t size); + void (*xFree)(void *, void *); +}; + +struct SSLCursor { + SSkipList2 *pSl; + SSLNode **forwards[SL_MAX_LEVEL]; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5ddffa0cbd..f7ef1f7e81 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -649,6 +649,19 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -658,8 +671,11 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; + STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - + *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 84db728ce4..875ae15071 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -168,13 +168,6 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint32_t total = numOfRow1 + numOfRow2; - if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); - uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); - memset(tmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = tmp; - } - uint32_t remindBits = BitPos(numOfRow1); uint32_t shiftBits = 8 - remindBits; @@ -209,10 +202,9 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, - uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRow2 == 0) { return numOfRow1; } @@ -221,14 +213,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co pColumnInfoData->hasNull = pSource->hasNull; } + uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + if (finalNumOfRows > *capacity) { + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + *capacity = finalNumOfRows; + pColumnInfoData->varmeta.offset = (int32_t*)p; } - pColumnInfoData->varmeta.offset = (int32_t*)p; for (int32_t i = 0; i < numOfRow2; ++i) { if (pSource->varmeta.offset[i] == -1) { pColumnInfoData->varmeta.offset[i + numOfRow1] = -1; @@ -253,15 +250,27 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + if (finalNumOfRows > *capacity) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } - int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + pColumnInfoData->pData = tmp; + + if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { + char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); + memset(btmp + BitmapLen(numOfRow1), 0, extend); + + pColumnInfoData->nullbitmap = btmp; + } + + *capacity = finalNumOfRows; } - pColumnInfoData->pData = tmp; + doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + int32_t offset = pColumnInfoData->info.bytes * numOfRow1; memcpy(pColumnInfoData->pData + offset, pSource->pData, pSource->info.bytes * numOfRow2); } @@ -350,29 +359,22 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { // if pIndexMap = NULL, merger one column by on column int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) { assert(pSrc != NULL && pDest != NULL); + int32_t capacity = pDest->info.capacity; - int32_t numOfCols = pDest->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; if (pIndexMap) { mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); } + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); - uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); - uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); - - int32_t newSize = oldLen + newLen; - char* tmp = taosMemoryRealloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } + capacity = pDest->info.capacity; + colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); } + pDest->info.capacity = capacity; pDest->info.rows += pSrc->info.rows; return TSDB_CODE_SUCCESS; } @@ -1462,6 +1464,132 @@ void blockDebugShowData(const SArray* dataBlocks) { } } +/** + * @brief TODO: Assume that the final generated result it less than 3M + * + * @param pReq + * @param pDataBlocks + * @param vgId + * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in + * SDataBlock->info.uid + * @param suid // TODO: check with Liao whether suid response is reasonable + * + * TODO: colId should be set + */ +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, + tb_uid_t suid) { + int32_t sz = taosArrayGetSize(pDataBlocks); + int32_t bufSize = sizeof(SSubmitReq); + for (int32_t i = 0; i < sz; ++i) { + SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + bufSize += sizeof(SSubmitBlk); + } + + ASSERT(bufSize < 3 * 1024 * 1024); + + *pReq = taosMemoryCalloc(1, bufSize); + if(!(*pReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + void* pDataBuf = *pReq; + + int32_t msgLen = sizeof(SSubmitReq); + int32_t numOfBlks = 0; + SRowBuilder rb = {0}; + tdSRowInit(&rb, 0); // TODO: use the latest version + + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + + if(rb.nCols != colNum) { + tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); + } + + SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); + pSubmitBlk->suid = suid; + pSubmitBlk->uid = uid; + pSubmitBlk->numOfRows = rows; + + ++numOfBlks; + + msgLen += sizeof(SSubmitBlk); + int32_t dataLen = 0; + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + printf("|"); + bool isStartKey = false; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + } else { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); + break; + } + break; + case TSDB_DATA_TYPE_NCHAR: { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + TASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k); + } else { + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + } + break; + } + } + dataLen += TD_ROW_LEN(rb.pBuf); + } + pSubmitBlk->dataLen = dataLen; + msgLen += pSubmitBlk->dataLen; + } + + (*pReq)->length = msgLen; + + (*pReq)->header.vgId = htonl(vgId); + (*pReq)->header.contLen = htonl(msgLen); + (*pReq)->length = (*pReq)->header.contLen; + (*pReq)->numOfBlocks = htonl(numOfBlks); + SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); + while (numOfBlks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } + + return TSDB_CODE_SUCCESS; +} + SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { SSubmitReq* ret = NULL; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f5fa1c6491..e659b9ec04 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -88,7 +88,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); tlen += taosEncodeFixedI8(buf, colType(pCol)); - tlen += taosEncodeFixedI8(buf, colSma(pCol)); + tlen += taosEncodeFixedI8(buf, colFlags(pCol)); tlen += taosEncodeFixedI16(buf, colColId(pCol)); tlen += taosEncodeFixedI16(buf, colBytes(pCol)); } @@ -111,14 +111,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; - int8_t sma = 0; + int8_t flags = 0; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); - buf = taosDecodeFixedI8(buf, &sma); + buf = taosDecodeFixedI8(buf, &flags); buf = taosDecodeFixedI16(buf, &colId); buf = taosDecodeFixedI32(buf, &bytes); - if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) { + if (tdAddColToSchema(&schemaBuilder, type, flags, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; } @@ -198,7 +198,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->version = version; } -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) { +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { @@ -211,7 +211,7 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); colSetType(pCol, type); colSetColId(pCol, colId); - colSetSma(pCol, sma); + colSetFlags(pCol, flags); if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bbf14b2fdc..87a9c521af 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +// internal +int32_t tsTransPullupMs = 6000; +int32_t tsMaRebalanceMs = 2000; + void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); tsDiskCfg[index].level = level; @@ -220,7 +224,8 @@ struct SConfig *taosGetCfg() { return tsCfg; } -static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { +static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, + char *apolloUrl) { char cfgDir[PATH_MAX] = {0}; char cfgFile[PATH_MAX + 100] = {0}; @@ -296,15 +301,10 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) return -1; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; @@ -474,15 +474,18 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); SEp firstEp = {0}; - taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); SEp secondEp = {0}; - taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); @@ -579,8 +582,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, - char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { osDefaultInit(); SConfig *pCfg = cfgInit(); @@ -632,7 +635,24 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { +static int32_t taosCheckGlobalCfg() { + uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); + if (ipv4 == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); + return -1; + } + + if (tsServerPort <= 0) { + uError("invalid server port:%u, dnode can not be initialized", tsServerPort); + return -1; + } + + return 0; +} + +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, + bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -670,6 +690,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); cfgDumpCfg(tsCfg, tsc, false); + + if (taosCheckGlobalCfg() != 0) { + return -1; + } + return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index fc993511ab..a91f5f8223 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -34,77 +34,6 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return -1; } - pIter->totalLen = pMsg->length; - ASSERT(pIter->totalLen > 0); - pIter->len = 0; - pIter->pMsg = pMsg; - if (pMsg->length <= sizeof(SSubmitReq)) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - return 0; -} - -int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - ASSERT(pIter->len >= 0); - - if (pIter->len == 0) { - pIter->len += sizeof(SSubmitReq); - } else { - if (pIter->len >= pIter->totalLen) { - ASSERT(0); - } - - SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); - ASSERT(pIter->len > 0); - } - - if (pIter->len > pIter->totalLen) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - *pPBlock = NULL; - return -1; - } - - *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - - return 0; -} - -int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->dataLen <= 0) return -1; - pIter->totalLen = pBlock->dataLen; - pIter->len = 0; - pIter->row = (STSRow *)(pBlock->data + pBlock->schemaLen); - return 0; -} - -STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { - STSRow *row = pIter->row; - - if (pIter->len >= pIter->totalLen) { - return NULL; - } else { - pIter->len += TD_ROW_LEN(row); - if (pIter->len < pIter->totalLen) { - pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row)); - } - return row; - } -} - -// TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter - -int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - pIter->totalLen = htonl(pMsg->length); ASSERT(pIter->totalLen > 0); pIter->len = 0; @@ -117,7 +46,7 @@ int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return 0; } -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { +int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { ASSERT(pIter->len >= 0); if (pIter->len == 0) { @@ -152,7 +81,7 @@ int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { return 0; } -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { +int32_t tInitSubmitBlkIter(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pMsgIter->dataLen <= 0) return -1; pIter->totalLen = pMsgIter->dataLen; pIter->len = 0; @@ -160,7 +89,7 @@ int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubm return 0; } -STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { +STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { STSRow *row = pIter->row; if (pIter->len >= pIter->totalLen) { @@ -176,17 +105,17 @@ STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) { SSubmitMsgIter msgIter = {0}; - if (tInitSubmitMsgIterEx(pReq, &msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; while (true) { SSubmitBlk *pBlock = NULL; - if (tGetSubmitMsgNextEx(&msgIter, &pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; SSubmitBlkIter blkIter = {0}; - tInitSubmitBlkIterEx(&msgIter, pBlock, &blkIter); + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); STSRowIter rowIter = {0}; tdSTSRowIterInit(&rowIter, pTschema); STSRow *row; - while ((row = tGetSubmitBlkNextEx(&blkIter)) != NULL) { + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { tdSRowPrint(row, pTschema, "stream"); } } @@ -1701,8 +1630,8 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -1747,8 +1676,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { @@ -2177,8 +2106,8 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2221,8 +2150,8 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { @@ -2839,8 +2768,8 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2896,8 +2825,8 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 5b4bb539e3..42612cecb9 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -15,33 +15,83 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" +#include "taoserror.h" static SMsgCb tsDefaultMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { - return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); + PutToQueueFp fp = pMsgCb->queueFps[qtype]; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype); + GetQueueSizeFp fp = pMsgCb->qsizeFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, vgId, qtype); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { - return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); + SendReqFp fp = pMsgCb->sendReqFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, epSet, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } -void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } +void tmsgSendRsp(const SRpcMsg* pRsp) { + SendRspFp fp = tsDefaultMsgCb.sendRspFp; + if (fp != NULL) { + return (*fp)(tsDefaultMsgCb.pWrapper, pRsp); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} + +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { + SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { - (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg); + RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp; + if (fp != NULL) { + (*fp)(pMsgCb->pWrapper, pMsg); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReleaseHandle(void* handle, int8_t type) { - (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type); + ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, handle, type); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReportStartup(const char* name, const char* desc) { - (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc); + ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, name, desc); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 10ab53eeee..446894556e 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -130,10 +130,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -320,6 +320,37 @@ static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); + ASSERT(pRsp->pCont == NULL); + if (pWrapper->procType != DND_PROC_CHILD) { + SRpcMsg resp = {0}; + SMEpSet msg = {.epSet = *pNewEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + resp.pCont = rpcMallocCont(len); + resp.contLen = len; + tSerializeSMEpSet(resp.pCont, len, &msg); + + resp.code = TSDB_CODE_RPC_REDIRECT; + resp.handle = pRsp->handle; + resp.refId = pRsp->refId; + rpcSendResponse(&resp); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} + +#if 0 +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); + if (pWrapper->procType != DND_PROC_CHILD) { + rpcSendRedirectRsp(pRsp->handle, pNewEpSet); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} +#endif + static inline void dmRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != DND_PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); @@ -406,6 +437,14 @@ SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper) { return cfg; } +bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + static int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -420,6 +459,7 @@ static int32_t dmInitClient(SDnode *pDnode) { rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; rpcInit.parent = pDnode; + rpcInit.rfp = rpcRfp; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); @@ -477,7 +517,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -551,6 +591,7 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { SMsgCb msgCb = { .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, + .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartupByWrapper, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b63a7f5b32..e3816396c9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -284,6 +284,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 818fd00430..f3d7253e71 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -149,8 +149,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - rsp.code = TSDB_CODE_SYN_NOT_LEADER; - tmsgSendRsp(&rsp); + // rsp.code = TSDB_CODE_SYN_NOT_LEADER; + // tmsgSendRsp(&rsp); + dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl))); + rsp.code = TSDB_CODE_RPC_REDIRECT; + SEpSet newEpSet; + syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); + newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + tmsgSendRedirectRsp(&rsp, &newEpSet); + } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; tmsgSendRsp(&rsp); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 456a370db6..4a0df80358 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -57,11 +57,11 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT = 3, - TRN_STAGE_COMMIT_LOG = 4, - TRN_STAGE_UNDO_ACTION = 5, - TRN_STAGE_UNDO_LOG = 6, - TRN_STAGE_ROLLBACK = 7, + TRN_STAGE_ROLLBACK = 3, + TRN_STAGE_UNDO_ACTION = 4, + TRN_STAGE_UNDO_LOG = 5, + TRN_STAGE_COMMIT = 6, + TRN_STAGE_COMMIT_LOG = 7, TRN_STAGE_FINISHED = 8 } ETrnStage; @@ -72,6 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -91,10 +92,12 @@ typedef enum { TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_CONSUMER_RECOVER = 1023, TRN_TYPE_BASIC_SCOPE_END, + TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_DROP_DNODE = 2002, TRN_TYPE_GLOBAL_SCOPE_END, + TRN_TYPE_DB_SCOPE = 3000, TRN_TYPE_CREATE_DB = 3001, TRN_TYPE_ALTER_DB = 3002, @@ -102,6 +105,7 @@ typedef enum { TRN_TYPE_SPLIT_VGROUP = 3004, TRN_TYPE_MERGE_VGROUP = 3015, TRN_TYPE_DB_SCOPE_END, + TRN_TYPE_STB_SCOPE = 4000, TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, @@ -131,7 +135,7 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; - ETrnType transType; + ETrnType type; int32_t code; int32_t failedTimes; void* rpcHandle; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 7644ec3c4c..7bd2bd70ee 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -44,6 +44,8 @@ typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); @@ -59,6 +61,7 @@ void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index e4ecdf3283..b3eb7f2f95 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,9 +24,12 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); +// for trans test +SSdbRaw *mndUserActionEncode(SUserObj *pUser); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 347e959d3c..dfad99bdfb 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -110,8 +110,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); - SDB_SET_INT32(pRaw, dataPos, pRetension->freq, _OVER) - SDB_SET_INT32(pRaw, dataPos, pRetension->keep, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } @@ -183,8 +183,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { if (pDb->cfg.pRetensions == NULL) goto _OVER; for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { SRetention retension = {0}; - SDB_GET_INT32(pRaw, dataPos, &retension.freq, _OVER) - SDB_GET_INT32(pRaw, dataPos, &retension.keep, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.freq, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.keep, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, _OVER) if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bbd952b6d0..f3438bd0dc 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -22,9 +22,9 @@ #include "mndSync.h" #include "mndUser.h" -#define MND_TRANS_VER_NUMBER 1 -#define MND_TRANS_ARRAY_SIZE 8 -#define MND_TRANS_RESERVE_SIZE 64 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 +#define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -63,13 +63,15 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + SSdbTable table = { + .sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); @@ -84,7 +86,7 @@ void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t rawDataLen = sizeof(STrans) + MND_TRANS_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -116,78 +118,89 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, MND_TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; } int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->transType, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoActionNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoActionNum, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->id, _OVER) + + ETrnStage stage = pTrans->stage; + if (stage == TRN_STAGE_REDO_LOG || stage == TRN_STAGE_REDO_ACTION) { + stage = TRN_STAGE_PREPARE; + } else if (stage == TRN_STAGE_UNDO_ACTION || stage == TRN_STAGE_UNDO_LOG) { + stage = TRN_STAGE_ROLLBACK; + } else if (stage == TRN_STAGE_COMMIT_LOG || stage == TRN_STAGE_FINISHED) { + stage = TRN_STAGE_COMMIT; + } else { + } + + SDB_SET_INT16(pRaw, dataPos, stage, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->policy, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, commitLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < redoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) } for (int32_t i = 0; i < undoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, _OVER) } - SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, _OVER) if (pTrans->param != NULL) { - SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -TRANS_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr()); sdbFreeRaw(pRaw); @@ -214,38 +227,38 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; STransAction action = {0}; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TRANS_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != MND_TRANS_VER_NUMBER) { + if (sver != TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto TRANS_DECODE_OVER; + goto _OVER; } pRow = sdbAllocRow(sizeof(STrans)); - if (pRow == NULL) goto TRANS_DECODE_OVER; + if (pRow == NULL) goto _OVER; pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto TRANS_DECODE_OVER; + if (pTrans == NULL) goto _OVER; - SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) - int16_t type = 0; - int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &stage, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &type, TRANS_DECODE_OVER) - pTrans->policy = policy; + int16_t policy = 0; + int16_t type = 0; + SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->stage = stage; - pTrans->transType = type; - SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, TRANS_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, TRANS_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoActionNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoActionNum, TRANS_DECODE_OVER) + pTrans->policy = policy; + pTrans->type = type; + SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &commitLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) pTrans->redoLogs = taosArrayInit(redoLogNum, sizeof(void *)); pTrans->undoLogs = taosArrayInit(undoLogNum, sizeof(void *)); @@ -253,79 +266,79 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); - if (pTrans->redoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->commitLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->redoActions == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoActions == NULL) goto TRANS_DECODE_OVER; + if (pTrans->redoLogs == NULL) goto _OVER; + if (pTrans->undoLogs == NULL) goto _OVER; + if (pTrans->commitLogs == NULL) goto _OVER; + if (pTrans->redoActions == NULL) goto _OVER; + if (pTrans->undoActions == NULL) goto _OVER; for (int32_t i = 0; i < redoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < undoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < commitLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < redoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } for (int32_t i = 0; i < undoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } - SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); - SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } - SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; -TRANS_DECODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); @@ -400,6 +413,16 @@ static const char *mndTransType(ETrnType type) { return "subscribe"; case TRN_TYPE_REBALANCE: return "rebalance"; + case TRN_TYPE_COMMIT_OFFSET: + return "commit-offset"; + case TRN_TYPE_CREATE_STREAM: + return "create-stream"; + case TRN_TYPE_DROP_STREAM: + return "drop-stream"; + case TRN_TYPE_CONSUMER_LOST: + return "consumer-lost"; + case TRN_TYPE_CONSUMER_RECOVER: + return "consumer-recover"; case TRN_TYPE_CREATE_DNODE: return "create-qnode"; case TRN_TYPE_DROP_DNODE: @@ -453,7 +476,6 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); if (pTrans->startFunc > 0) { @@ -515,16 +537,15 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { return 0; } -static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { + STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; } return pTrans; } -static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTrans); } @@ -540,16 +561,16 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->transType = type; + pTrans->type = type; pTrans->createdTime = taosGetTimestampMs(); pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; pTrans->rpcRefId = pReq->refId; - pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->redoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -558,7 +579,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -585,14 +606,14 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -674,27 +695,27 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { } static bool mndIsBasicTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_BASIC_SCOPE && pTrans->stage < TRN_TYPE_BASIC_SCOPE_END; + return pTrans->type > TRN_TYPE_BASIC_SCOPE && pTrans->type < TRN_TYPE_BASIC_SCOPE_END; } static bool mndIsGlobalTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_GLOBAL_SCOPE && pTrans->stage < TRN_TYPE_GLOBAL_SCOPE_END; + return pTrans->type > TRN_TYPE_GLOBAL_SCOPE && pTrans->type < TRN_TYPE_GLOBAL_SCOPE_END; } static bool mndIsDbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_DB_SCOPE && pTrans->stage < TRN_TYPE_DB_SCOPE_END; + return pTrans->type > TRN_TYPE_DB_SCOPE && pTrans->type < TRN_TYPE_DB_SCOPE_END; } static bool mndIsStbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; + return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } -static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewTrans) { - if (mndIsBasicTrans(pNewTrans)) return 0; - +static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - int32_t code = 0; + bool conflict = false; + + if (mndIsBasicTrans(pNewTrans)) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -703,42 +724,35 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; + } else { } } - if (mndIsDbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; - break; - } - if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { + conflict = true; + } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; } + } else { } } - if (mndIsStbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; - break; - } - if (mndIsDbTrans(pTrans)) { + conflict = true; + } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; } + } else { } - if (mndIsStbTrans(pTrans)) continue; } sdbRelease(pMnode->pSdb, pTrans); @@ -746,12 +760,12 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return code; + return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndCheckTransCanBeStartedInParallel(pMnode, pTrans) != 0) { - terrno = TSDB_CODE_MND_TRANS_CANT_PARALLEL; + if (mndCheckTransConflict(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -805,7 +819,8 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { - bool sendRsp = false; + bool sendRsp = false; + int32_t code = pTrans->code; if (pTrans->stage == TRN_STAGE_FINISHED) { sendRsp = true; @@ -814,12 +829,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->policy == TRN_POLICY_ROLLBACK) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } - } - - if (pTrans->policy == TRN_POLICY_RETRY) { + } else { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } } @@ -831,14 +846,16 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, - .code = pTrans->code, - .ahandle = pTrans->rpcAHandle, - .refId = pTrans->rpcRefId, - .pCont = rpcCont, - .contLen = pTrans->rpcRspLen}; + SRpcMsg rspMsg = { + .handle = pTrans->rpcHandle, + .ahandle = pTrans->rpcAHandle, + .refId = pTrans->rpcRefId, + .code = code, + .pCont = rpcCont, + .contLen = pTrans->rpcRspLen, + }; tmsgSendRsp(&rspMsg); pTrans->rpcHandle = NULL; pTrans->rpcRsp = NULL; @@ -855,7 +872,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } SArray *pArray = NULL; @@ -865,18 +882,18 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pArray = pTrans->undoActions; } else { mError("trans:%d, invalid trans stage:%d while recv action rsp", pTrans->id, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } if (pArray == NULL) { mError("trans:%d, invalid trans stage:%d", transId, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } int32_t actionNum = taosArrayGetSize(pTrans->redoActions); if (action < 0 || action >= actionNum) { mError("trans:%d, invalid action:%d", transId, action); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } STransAction *pAction = taosArrayGet(pArray, action); @@ -892,7 +909,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pAction->acceptableCode); mndTransExecute(pMnode, pTrans); -HANDLE_ACTION_RSP_OVER: +_OVER: mndReleaseTrans(pMnode, pTrans); } @@ -902,27 +919,41 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { if (arraySize == 0) return 0; + int32_t code = 0; for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWriteWithoutFree(pSdb, pRaw); - if (code != 0) { - return code; + if (sdbWriteWithoutFree(pSdb, pRaw) != 0) { + code = ((terrno != 0) ? terrno : -1); } } - return 0; + terrno = code; + return code; } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->redoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("failed to execute redoLogs since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->undoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("failed to execute undoLogs since %s, return success", terrstr()); + } + + return 0; // return success in any case } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->commitLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("failed to execute commitLogs since %s", terrstr()); + } + return code; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -966,6 +997,12 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + pAction->msgSent = 0; + pAction->msgReceived = 0; + pAction->errCode = terrno; + if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { + rpcFreeCont(rpcMsg.pCont); + } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } @@ -1012,11 +1049,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + if (code != 0) { + mError("failed to execute redoActions since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + if (code != 0) { + mError("failed to execute undoActions since %s", terrstr()); + } + return code; } static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { @@ -1083,8 +1128,8 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_REDO_ACTION; - mError("trans:%d, stage from commit to redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), + pTrans->stage = TRN_STAGE_UNDO_ACTION; + mError("trans:%d, stage from commit to undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = true; } else { @@ -1109,7 +1154,7 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; pTrans->failedTimes++; - mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on commitLog since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1145,7 +1190,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { continueExec = false; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on undoAction since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1162,7 +1207,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { continueExec = true; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on rollback since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1235,7 +1280,7 @@ static int32_t mndProcessTransReq(SNodeMsg *pReq) { return 0; } -static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { SArray *pArray = NULL; if (pTrans->stage == TRN_STAGE_REDO_ACTION) { pArray = pTrans->redoActions; @@ -1253,14 +1298,14 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { if (pAction == NULL) continue; if (pAction->msgReceived == 0) { - mInfo("trans:%d, action:%d set processed", pTrans->id, i); + mInfo("trans:%d, action:%d set processed for kill msg received", pTrans->id, i); pAction->msgSent = 1; pAction->msgReceived = 1; pAction->errCode = 0; } if (pAction->errCode != 0) { - mInfo("trans:%d, action:%d set processed, errCode from %s to success", pTrans->id, i, + mInfo("trans:%d, action:%d set processed for kill msg received, errCode from %s to success", pTrans->id, i, tstrerror(pAction->errCode)); pAction->msgSent = 1; pAction->msgReceived = 1; @@ -1281,19 +1326,19 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto KILL_OVER; + goto _OVER; } mInfo("trans:%d, start to kill", killReq.transId); pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto KILL_OVER; + goto _OVER; } if (!pUser->superUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; - goto KILL_OVER; + goto _OVER; } pTrans = mndAcquireTrans(pMnode, killReq.transId); @@ -1305,7 +1350,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { code = mndKillTrans(pMnode, pTrans); -KILL_OVER: +_OVER: if (code != 0) { mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); return -1; @@ -1360,10 +1405,10 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char transType[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->transType), pShow->pMeta->pSchemas[cols].bytes); + char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->type), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)transType, false); + colDataAppend(pColInfo, numOfRows, (const char *)type, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2ced813003..5e15bdeb43 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -25,7 +25,6 @@ #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); -static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); @@ -90,7 +89,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return 0; } -static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { +SSdbRaw *mndUserActionEncode(SUserObj *pUser) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); @@ -238,7 +237,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { @@ -276,9 +275,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test code to be deleted later <====="); - mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b38fd6178c..e2814e95f0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,9 +43,6 @@ #include "mndUser.h" #include "mndVgroup.h" -#define MQ_TIMER_MS 2000 -#define TRNAS_TIMER_MS 6000 - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } - taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer); } static void mndPullupTelem(void *param, void *tmrId) { @@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mnode/impl/test/trans/CMakeLists.txt b/source/dnode/mnode/impl/test/trans/CMakeLists.txt index fa0ef9f263..3931433c19 100644 --- a/source/dnode/mnode/impl/test/trans/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/trans/CMakeLists.txt @@ -1,11 +1,37 @@ -aux_source_directory(. MNODE_TRANS_TEST_SRC) -add_executable(transTest ${MNODE_TRANS_TEST_SRC}) +add_executable(transTest1 "") +target_sources(transTest1 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans1.cpp" +) target_link_libraries( - transTest + transTest1 PUBLIC sut ) - -add_test( - NAME transTest - COMMAND transTest +target_include_directories( + transTest1 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest1 + COMMAND transTest1 +) + +add_executable(transTest2 "") +target_sources(transTest2 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans2.cpp" +) +target_link_libraries( + transTest2 + PUBLIC dnode mnode gtest_main +) +target_include_directories( + transTest2 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest2 + COMMAND transTest2 ) diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp similarity index 93% rename from source/dnode/mnode/impl/test/trans/trans.cpp rename to source/dnode/mnode/impl/test/trans/trans1.cpp index bcf6fe8536..07bf9a2bcf 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -11,10 +11,10 @@ #include "sut.h" -class MndTestTrans : public ::testing::Test { +class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans", 9013); + test.Init("/tmp/mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); @@ -26,7 +26,7 @@ class MndTestTrans : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; + char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); @@ -60,10 +60,10 @@ class MndTestTrans : public ::testing::Test { void TearDown() override {} }; -Testbase MndTestTrans::test; -TestServer MndTestTrans::server2; +Testbase MndTestTrans1::test; +TestServer MndTestTrans1::server2; -TEST_F(MndTestTrans, 00_Create_User_Crash) { +TEST_F(MndTestTrans1, 00_Create_User_Crash) { { test.SendShowReq(TSDB_MGMT_TABLE_TRANS, "trans", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -83,7 +83,7 @@ TEST_F(MndTestTrans, 00_Create_User_Crash) { } } -TEST_F(MndTestTrans, 01_Create_User_Crash) { +TEST_F(MndTestTrans1, 01_Create_User_Crash) { { SCreateUserReq createReq = {0}; strcpy(createReq.user, "u1"); @@ -107,7 +107,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { EXPECT_EQ(test.GetShowRows(), 2); } -TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { +TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; @@ -142,7 +142,7 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { } } -TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { +TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { { SCreateDnodeReq createReq = {0}; strcpy(createReq.fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp new file mode 100644 index 0000000000..b3cbcb6898 --- /dev/null +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -0,0 +1,513 @@ +/** + * @file trans.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module trans tests + * @version 1.0 + * @date 2022-05-02 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "mndTrans.h" +#include "mndUser.h" +#include "tcache.h" + +void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {} + +class MndTestTrans2 : public ::testing::Test { + protected: + static void InitLog() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 207; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + const char *logpath = "/tmp/td"; + taosRemoveDir(logpath); + taosMkDir(logpath); + tstrncpy(tsLogDir, logpath, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } + } + + static void InitMnode() { + static SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefaultMsgCb(&msgCb); + + SMnodeOpt opt = {0}; + opt.deploy = 1; + opt.replica = 1; + opt.replicas[0].id = 1; + opt.replicas[0].port = 9040; + strcpy(opt.replicas[0].fqdn, "localhost"); + opt.msgCb = msgCb; + + tsTransPullupMs = 1000; + + const char *mnodepath = "/tmp/mnode_test_trans"; + taosRemoveDir(mnodepath); + pMnode = mndOpen(mnodepath, &opt); + mndStart(pMnode); + } + + static void SetUpTestSuite() { + InitLog(); + walInit(); + InitMnode(); + } + + static void TearDownTestSuite() { + mndStop(pMnode); + mndClose(pMnode); + walCleanUp(); + taosCloseLog(); + taosStopCacheRefreshWorker(); + } + + static SMnode *pMnode; + + public: + void SetUp() override {} + void TearDown() override {} + + int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type, + SDbObj *pDb) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test action <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world redo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendRedoAction(pTrans, &action); + } + + if (hasUndoAction) { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world undo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendUndoAction(pTrans, &action); + } + + { + void *pRsp = taosMemoryCalloc(1, 256); + strcpy((char *)pRsp, "simple rsponse"); + mndTransSetRpcRsp(pTrans, pRsp, 256); + } + + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserGlobal(const char *acct, const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } +}; + +SMnode *MndTestTrans2::pMnode; + +TEST_F(MndTestTrans2, 01_Log) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "log1"; + const char *user2 = "log2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + + ASSERT_NE(pMnode, nullptr); + + EXPECT_EQ(CreateUserLog(acct, user1, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUserLog(acct_invalid, user2, TRN_TYPE_CREATE_USER, NULL), 0); + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + + mndTransPullup(pMnode); +} + +TEST_F(MndTestTrans2, 02_Action) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "action1"; + const char *user2 = "action2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } + + { + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } + + { + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = 0; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + } +} + +TEST_F(MndTestTrans2, 03_Kill) { + const char *acct = "root"; + const char *user1 = "kill1"; + const char *user2 = "kill2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 7; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } +} + +TEST_F(MndTestTrans2, 04_Conflict) { + const char *acct = "root"; + const char *user1 = "conflict1"; + const char *user2 = "conflict2"; + const char *user3 = "conflict3"; + const char *user4 = "conflict4"; + const char *user5 = "conflict5"; + const char *user6 = "conflict6"; + const char *user7 = "conflict7"; + const char *user8 = "conflict8"; + SUserObj *pUser = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t code = 0; + + ASSERT_NE(pMnode, nullptr); + + { + SDbObj dbObj1 = {0}; + dbObj1.uid = 9521; + strcpy(dbObj1.name, "db"); + SDbObj dbObj2 = {0}; + dbObj2.uid = 9522; + strcpy(dbObj2.name, "conflict db"); + + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &dbObj1), 0); + pUser = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + + transId = 8; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + + // stb scope + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DNODE, NULL), -1); + code = terrno; + EXPECT_EQ(code, TSDB_CODE_MND_TRANS_CONFLICT); + + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user3, TRN_TYPE_CREATE_STB, &dbObj1), 0); + + // db scope + pTrans->type = TRN_TYPE_CREATE_DB; + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DNODE, NULL), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj2), 0); + + // global scope + pTrans->type = TRN_TYPE_CREATE_DNODE; + EXPECT_EQ(CreateUserLog(acct, user6, TRN_TYPE_CREATE_DNODE, NULL), 0); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj2), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj2), -1); + + // global scope + pTrans->type = TRN_TYPE_CREATE_USER; + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), 0); + EXPECT_EQ(CreateUserLog(acct, user8, TRN_TYPE_CREATE_DB, &dbObj2), 0); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f4fccc2f49..62deb200c9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,7 +37,7 @@ #ifdef __cplusplus extern "C" { #endif -#define TSDB_VNODE_SMA_DEBUG // TODO: evaluate to remove the macro and the relative codes + // vnode typedef struct SVnode SVnode; typedef struct STsdbCfg STsdbCfg; // todo: remove @@ -145,7 +145,7 @@ struct STsdbCfg { int32_t keep2; // TODO: save to tsdb cfg file int8_t type; // ETsdbType - SRetention retentions[TSDB_RSMA_RETENTION_MAX]; + SRetention retentions[TSDB_RETENTION_MAX]; }; typedef enum { diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index ca7e823d6b..5970fdbe92 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -103,8 +103,6 @@ typedef struct { #if 1 -// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseSmaCursor(SMSmaCursor* pSmaCur); int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9ac8434949..078a493773 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,7 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows); +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); @@ -72,6 +72,7 @@ struct STsdb { char *path; SVnode *pVnode; bool repoLocked; + int8_t level; // retention level TdThreadMutex mutex; STsdbMemTable *mem; STsdbMemTable *imem; @@ -185,6 +186,7 @@ struct STsdbFS { #define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) +#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 162d733cc3..5215812ac5 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -40,7 +40,6 @@ static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5930dcaf96..5538a63b4b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -46,13 +46,13 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -77,6 +77,7 @@ int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -100,7 +101,7 @@ int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, - uint64_t taskId); + uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pMemRef); int32_t tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); @@ -123,7 +124,8 @@ int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, t int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); -int32_t tsdbTriggerRSma(STsdb* pTsdb, SMeta* pMeta, void* pMsg, int32_t inputType); +int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); +int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other @@ -181,14 +183,14 @@ struct SVnode { struct STbUidStore { tb_uid_t suid; + tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; #define TD_VID(PVNODE) (PVNODE)->config.vgId - -static FORCE_INLINE bool tsdbIsRollup(SVnode* pVnode) { +static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); return (pRetention->freq > 0 && pRetention->keep > 0); } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index bba707076c..cb556e8630 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeTbInfo(&pBuf, pTbCfg); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeSchemaEx(&pBuf, &schemaWrapper); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); if (ret < 0) { return -1; } @@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = nameLen + 1 + sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(ctbIdxKey); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { int32_t kLen = sizeof(pSmaCfg->indexUid); int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); - ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; @@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { val = NULL; vLen = 0; - ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cb0b3dc81c..397e074061 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -72,44 +72,61 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - SMetaReader mr = {0}; + TDBC *pNameIdxc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pCtbIdxc = NULL; + SCtbIdxKey *pCtbIdxKey; + const void *pKey = NULL; + int nKey; + const void *pData = NULL; + int nData; + int c, ret; - // validate req - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + // prepare uid idx cursor + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c != 0) { + terrno = TSDB_CODE_VND_TB_NOT_EXIST; + tdbDbcClose(pUidIdxc); goto _err; } - // do drop - // drop from pTbDb - // drop from pSkmDb - // drop from pUidIdx - // drop from pNameIdx - // { - // TDBC *pDbc1 = NULL; - // void *pKey = NULL; - // void *pVal = NULL; - // int kLen = 0; - // int vLen = 0; - // int ret = 0; + // prepare name idx cursor + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c != 0) { + ASSERT(0); + } - // // drop from pCtbIdx - // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); - // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); - // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); - // tdbDbcDrop(pDbc1); - // // drop from pTagIdx - // // drop from pTtlIdx - // } + tdbDbcDelete(pUidIdxc); + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pUidIdxc); + tdbDbcClose(pNameIdxc); - // clear and return - metaReaderClear(&mr); - metaError("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + // loop to drop each child table + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { + tdbDbcClose(pCtbIdxc); + goto _exit; + } + + for (;;) { + tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + pCtbIdxKey = (SCtbIdxKey *)pKey; + + if (pCtbIdxKey->suid > pReq->suid) break; + + // drop the child table (TODO) + + if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + } + +_exit: + metaDebug("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaReaderClear(&mr); metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; @@ -166,18 +183,122 @@ _err: return -1; } -int metaDropTable(SMeta *pMeta, tb_uid_t uid) { -#if 0 - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO: handle error +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + TDBC *pTbDbc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pNameIdxc = NULL; + const void *pData; + int nData; + tb_uid_t uid; + int64_t tver; + SMetaEntry me = {0}; + SCoder coder = {0}; + int8_t type; + int64_t ctime; + tb_uid_t suid; + int c, ret; + + // search & delete the name idx + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c) { + tdbDbcClose(pNameIdxc); + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO + ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); return -1; } -#endif + + uid = *(tb_uid_t *)pData; + + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pNameIdxc); + + // search & delete uid idx + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tver = *(int64_t *)pData; + tdbDbcDelete(pUidIdxc); + tdbDbcClose(pUidIdxc); + + // search and get meta entry + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // decode entry + void *pDataCopy = taosMemoryMalloc(nData); // remove the copy (todo) + memcpy(pDataCopy, pData, nData); + tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER); + ret = metaDecodeEntry(&coder, &me); + if (ret < 0) { + ASSERT(0); + return -1; + } + + type = me.type; + if (type == TSDB_CHILD_TABLE) { + ctime = me.ctbEntry.ctime; + suid = me.ctbEntry.suid; + } else if (type == TSDB_NORMAL_TABLE) { + ctime = me.ntbEntry.ctime; + suid = 0; + } else { + ASSERT(0); + } + + taosMemoryFree(pDataCopy); + tCoderClear(&coder); + tdbDbcClose(pTbDbc); + + if (type == TSDB_CHILD_TABLE) { + // remove the pCtbIdx + TDBC *pCtbIdxc = NULL; + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + tdbDbcDelete(pCtbIdxc); + tdbDbcClose(pCtbIdxc); + + // remove tags from pTagIdx (todo) + } else if (type == TSDB_NORMAL_TABLE) { + // remove from pSkmDb + } else { + ASSERT(0); + } + + // remove from ttl (todo) + if (ctime > 0) { + } return 0; } @@ -218,7 +339,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderClear(&coder); // write to table.db - if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -231,11 +352,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -258,12 +379,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -304,7 +425,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 89ec55cca1..511d57ed58 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -37,9 +37,9 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); // iterate and convert - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; if (pReadHandle->pBlock == NULL) break; // pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid); @@ -50,7 +50,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows); } - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; pReadHandle->ver = ver; memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; @@ -58,7 +58,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t bool tqNextDataBlock(STqReadHandle* pHandle) { while (1) { - if (tGetSubmitMsgNextEx(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } if (pHandle->pBlock == NULL) return false; @@ -169,8 +169,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterInit(&iter, pTschema); STSRow* row; int32_t curRow = 0; - tInitSubmitBlkIterEx(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - while ((row = tGetSubmitBlkNextEx(&pHandle->blkIter)) != NULL) { + tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6a8af73694..fc23534247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -935,7 +935,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlockCol->type = pDataCol->type; pAggrBlkCol->colId = pDataCol->colId; - if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) { + if (isSuper && IS_BSMA_ON(pColumn) && tDataTypes[pDataCol->type].statisFunc) { #if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4ff3227e08..6eda476b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,6 +15,8 @@ #include "tsdb.h" +extern const char *TSDB_LEVEL_DNAME[]; + typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) @@ -35,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); } -static void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); } // For backward compatibility @@ -588,8 +590,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - tsdbTxnFname[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), + TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -719,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -753,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -801,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b1af3e0461..7f024786de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,13 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +const char *TSDB_LEVEL_DNAME[] = { + "tsdb", + "rsma1", + "rsma2", +}; + +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); @@ -45,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } @@ -431,14 +437,15 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype return 0; } -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) { ASSERT(ftype != TSDB_FILE_MAX); if (ftype < TSDB_FILE_MAX) { if (ver == 0) { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid, + TSDB_FNAME_SUFFIX[ftype]); } else { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid, TSDB_FNAME_SUFFIX[ftype], ver); } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 48e672d9bc..ff4d99f510 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,35 +190,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) { +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { // STsdbMeta *pMeta = pRepo->tsdbMeta; // int32_t points = 0; // STable *pTable = NULL; @@ -232,15 +204,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows SSubmitBlk *pBlkCopy; // create container is nedd - tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid)); + tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid)); if (tptr == NULL) { - pTbData = tsdbNewTbData(pBlock->uid); + pTbData = tsdbNewTbData(pMsgIter->uid); if (pTbData == NULL) { return -1; } // Put into hash - taosHashPut(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid), &(pTbData), sizeof(pTbData)); + taosHashPut(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid), &(pTbData), sizeof(pTbData)); // Put into skiplist tSkipListPut(pMemTable->pSlIdx, pTbData); @@ -249,10 +221,10 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock)); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pMsgIter->dataLen + sizeof(*pBlock)); + memcpy(pBlkCopy, pBlock, pMsgIter->dataLen + sizeof(*pBlock)); - tInitSubmitBlkIter(pBlkCopy, &blkIter); + tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; keyMin = TD_ROW_KEY(blkIter.row); @@ -261,15 +233,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows // Set statistics keyMax = TD_ROW_KEY(blkIter.row); - pTbData->nrows += pBlock->numOfRows; + pTbData->nrows += pMsgIter->numOfRows; if (pTbData->keyMin > keyMin) pTbData->keyMin = keyMin; if (pTbData->keyMax < keyMax) pTbData->keyMax = keyMax; - pMemTable->nRow += pBlock->numOfRows; + pMemTable->nRow += pMsgIter->numOfRows; if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pBlock->numOfRows; + (*pAffectedRows) += pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 0827ba6eab..e18c01dc01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,21 +15,21 @@ #include "tsdb.h" -static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir); +static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); int tsdbOpen(SVnode *pVnode, int8_t type) { switch (type) { case TSDB_TYPE_TSDB: - return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_TSMA: ASSERT(0); break; case TSDB_TYPE_RSMA_L0: - return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_RSMA_L1: - return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR, TSDB_RETENTION_L1); case TSDB_TYPE_RSMA_L2: - return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR, TSDB_RETENTION_L2); default: ASSERT(0); break; @@ -37,7 +37,17 @@ int tsdbOpen(SVnode *pVnode, int8_t type) { return 0; } -int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { +/** + * @brief + * + * @param pVnode + * @param type + * @param ppTsdb + * @param dir + * @param level retention level + * @return int + */ +int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { STsdb *pTsdb = NULL; int slen = 0; @@ -55,6 +65,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); pTsdb->pVnode = pVnode; + pTsdb->level = level; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb)); @@ -67,7 +78,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { goto _err; } - tsdbDebug("vgId: %d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path); + tsdbDebug("vgId:%d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path); *ppTsdb = pTsdb; return 0; @@ -79,6 +90,7 @@ _err: int tsdbClose(STsdb *pTsdb) { if (pTsdb) { + // TODO: destroy mem/imem tsdbCloseFS(pTsdb); tsdbFreeFS(pTsdb->fs); taosMemoryFree(pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bf35241dc4..b6dcc52b5e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -98,46 +98,46 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SColumnDataAgg *pstatis; - SColumnDataAgg **plist; - SArray *defaultLoadColumn; // default load column - int32_t *slotIds; // colId to slotId + SColumnDataAgg* pstatis; + SColumnDataAgg** plist; + SArray* defaultLoadColumn; // default load column + int32_t* slotIds; // colId to slotId } SBlockLoadSuppInfo; typedef struct STsdbReadHandle { - STsdb* pTsdb; - SQueryFilePos cur; // current position - int16_t order; - STimeWindow window; // the primary query time window that applies to all queries -// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time -// SColumnDataAgg** pstatis;// the ptr array list to return to caller - int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list - bool locateStart; - int32_t outputCapacity; - int32_t realNumOfRows; - SArray* pTableCheckInfo; // SArray - int32_t activeIndex; - bool checkFiles; // check file stage - int8_t cachelastrow; // check if last row cached - bool loadExternalRow; // load time window external data rows - bool currentLoadExternalRows; // current load external rows - int32_t loadType; // block load type - char* idStr; // query info handle, for debug purpose + STsdb* pTsdb; + SQueryFilePos cur; // current position + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time + // SColumnDataAgg** pstatis;// the ptr array list to return to caller + int32_t numOfBlocks; + SArray* pColumns; // column list, SColumnInfoData array list + bool locateStart; + int32_t outputCapacity; + int32_t realNumOfRows; + SArray* pTableCheckInfo; // SArray + int32_t activeIndex; + bool checkFiles; // check file stage + int8_t cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + char* idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; SReadH rhelper; STableBlockInfo* pDataBlockInfo; - SDataCols* pDataCols; // in order to hold current file data block - int32_t allocSize; // allocated data block size - SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ - SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ + SDataCols* pDataCols; // in order to hold current file data block + int32_t allocSize; // allocated data block size + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ SBlockLoadSuppInfo suppInfo; - SArray* prev; // previous row which is before than time window - SArray* next; // next row which is after the query time window - SIOCostSummary cost; - STSchema* pSchema; + SArray* prev; // previous row which is before than time window + SArray* next; // next row which is after the query time window + SIOCostSummary cost; + STSchema* pSchema; } STsdbReadHandle; typedef struct STableGroupSupporter { @@ -155,7 +155,6 @@ static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); @@ -164,6 +163,8 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->uid = 0; @@ -350,6 +351,41 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } +#if 0 +int nQUERY = 0; +#endif +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { + if (vnodeIsRollup(pVnode)) { + int level = 0; +#if 1 + int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + i; + if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + break; + } + } +#endif +#if 0 + ++nQUERY; + if(nQUERY%3 == 0) { + level = 2; + } else if(nQUERY%2 == 0) { + level = 1; + } else { + level = 0; + } +#endif + if (level == TSDB_RETENTION_L0) { + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + return VND_RSMA1(pVnode); + } else { + return VND_RSMA2(pVnode); + } + } + return pVnode->pTsdb; +} static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); @@ -357,8 +393,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* goto _end; } + STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + pReadHandle->order = pCond->order; - pReadHandle->pTsdb = pVnode->pTsdb; + pReadHandle->pTsdb = pTsdb; pReadHandle->type = TSDB_QUERY_TYPE_ALL; pReadHandle->cur.fid = INT32_MIN; pReadHandle->cur.win = TSWINDOW_INITIALIZER; @@ -376,7 +414,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId); pReadHandle->idStr = strdup(buf); - if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)pVnode->pTsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, pReadHandle->pTsdb) != 0) { goto _end; } @@ -409,11 +447,13 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); - pReadHandle->suppInfo.slotIds = taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); - pReadHandle->suppInfo.plist = taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); + pReadHandle->suppInfo.slotIds = + taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); + pReadHandle->suppInfo.plist = + taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); } - pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->pVnode->config.tsdbCfg.maxRows); + pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); if (pReadHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -433,7 +473,6 @@ _end: tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; @@ -451,16 +490,16 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG return NULL; } - STableCheckInfo *pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); - int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); + int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; STSchema* pSchema = pTsdbReadHandle->pSchema; int32_t i = 0, j = 0; - while(i < numOfCols && j < pSchema->numOfCols) { + while (i < numOfCols && j < pSchema->numOfCols) { if (ids[i] == pSchema->columns[j].colId) { pTsdbReadHandle->suppInfo.slotIds[i] = j; i++; @@ -1126,7 +1165,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1297,6 +1336,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* return code; } +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); + static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -3330,7 +3371,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat int32_t* slotIds = pHandle->suppInfo.slotIds; for (int32_t i = 1; i < numOfCols; ++i) { ASSERT(colIds[i] == pHandle->pSchema->columns[slotIds[i]].colId); - if (pHandle->pSchema->columns[slotIds[i]].sma) { + if (IS_BSMA_ON(&(pHandle->pSchema->columns[slotIds[i]]))) { if (pHandle->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL pHandle->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index d131cef8a2..362e31ff61 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -105,7 +105,7 @@ typedef struct { #define RSMA_TASK_INFO_HASH_SLOT 8 struct SRSmaInfo { - void *taskInfo[TSDB_RSMA_RETENTION_2]; // qTaskInfo_t + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t }; struct SSmaStat { @@ -127,7 +127,7 @@ static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { } static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < TSDB_RSMA_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (pInfo->taskInfo[i]) { tsdbFreeTaskHandle(pInfo->taskInfo[i]); } @@ -174,6 +174,9 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -702,25 +705,25 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SInterval interval = {0}; TSKEY lastWinSKey = INT64_MIN; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } while (true) { - tGetSubmitMsgNextEx(&msgIter, &pBlock); + tGetSubmitMsgNext(&msgIter, &pBlock); if (!pBlock) break; STSmaWrapper *pSW = NULL; STSma *pTSma = NULL; SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIterEx(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { pSW = tdFreeTSmaWrapper(pSW); break; } while (true) { - STSRow *row = tGetSubmitBlkNextEx(&blkIter); + STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { tdFreeTSmaWrapper(pSW); break; @@ -1881,8 +1884,10 @@ int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, t return TSDB_CODE_SUCCESS; } + ASSERT(ppStore != NULL); + if (!(*ppStore)) { - if (tsdbUidStoreInit((STbUidStore **)ppStore) != 0) { + if (tsdbUidStoreInit(ppStore) != 0) { return TSDB_CODE_FAILED; } } @@ -1966,87 +1971,101 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&msgIter, &pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; } -int32_t tsdbExecuteRSma(STsdb *pTsdb, SMeta *pMeta, const void *pMsg, int32_t inputType, tb_uid_t *suid) { +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level) { + SArray *pResult = NULL; + tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, + suid); + + qSetStreamInput(taskInfo, pMsg, inputType); + while (1) { + SSDataBlock *output = NULL; + uint64_t ts; + if (qExecTask(taskInfo, &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + if (!pResult) { + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + taosArrayPush(pResult, output); + } + + if (taosArrayGetSize(pResult) > 0) { + blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + SSubmitReq *pReq = NULL; + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); + } else { + tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); + } + + taosArrayDestroy(pResult); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + ASSERT(uid != 0); // TODO: remove later + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), *suid); + tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), suid); return TSDB_CODE_SUCCESS; } - SArray *pResult = NULL; - - pResult = taosArrayInit(0, sizeof(SSDataBlock)); - if (!pResult) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - if (pRSmaInfo->taskInfo[0]) { - tsdbDebug("vgId:%d execute rsma task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), pRSmaInfo->taskInfo[0], - *suid); - qSetStreamInput(pRSmaInfo->taskInfo[0], pMsg, inputType); - while (1) { - SSDataBlock *output; - uint64_t ts; - if (qExecTask(pRSmaInfo->taskInfo[0], &output, &ts) < 0) { - ASSERT(false); - } - if (!output) { - break; - } - taosArrayPush(pResult, output); - } - if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); - } else { - tsdbWarn("vgId:%d no sma data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - } - - // if (pRSmaInfo->taskInfo[1]) { - // qSetStreamInput(pRSmaInfo->taskInfo[1], pMsg, inputType); - // while (1) { - // SSDataBlock *output; - // uint64_t ts; - // if (qExecTask(pRSmaInfo->taskInfo[1], &output, &ts) < 0) { - // ASSERT(false); - // } - // if (!output) { - // break; - // } - // taosArrayPush(pResult, output); - // } - // blockDebugShowData(pResult); - // } + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); } return TSDB_CODE_SUCCESS; } -int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType) { +int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists @@ -2058,12 +2077,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputTyp tsdbFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, &uidStore.suid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, pTbSuid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); pIter = taosHashIterate(uidStore.uidHash, pIter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index 74878e817f..8a553e94fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 9fb6aad472..88b637bc24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -38,11 +38,11 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * while (true) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) { + if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &affectedrows) < 0) { return -1; } - numOfRows += pBlock->numOfRows; + numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -66,20 +66,20 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // pMsg->length = htonl(pMsg->length); + // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); + // pBlock->uid = htobe64(pBlock->uid); + // pBlock->suid = htobe64(pBlock->suid); + // pBlock->sversion = htonl(pBlock->sversion); + // pBlock->dataLen = htonl(pBlock->dataLen); + // pBlock->schemaLen = htonl(pBlock->schemaLen); + // pBlock->numOfRows = htons(pBlock->numOfRows); #if 0 if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 8130a00e04..7eac0389d2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -66,7 +66,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG if (pCfg->tsdbCfg.retentions[0].freq > 0) { int32_t nRetention = 1; if (pCfg->tsdbCfg.retentions[1].freq > 0) { @@ -87,7 +86,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeRetentions, pNodeRetention); } } -#endif if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -135,11 +133,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); - int nRetention = tjsonGetArraySize(pNodeRetentions); - ASSERT(nRetention <= TSDB_RSMA_RETENTION_MAX); - + int32_t nRetention = tjsonGetArraySize(pNodeRetentions); + if (nRetention > TSDB_RETENTION_MAX) { + nRetention = TSDB_RETENTION_MAX; + } for (int32_t i = 0; i < nRetention; ++i) { SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); ASSERT(pNodeRetention != NULL); @@ -148,7 +146,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit); } -#endif if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 7026673231..6d8bcb35c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -47,9 +47,26 @@ int vnodeBegin(SVnode *pVnode) { } // begin tsdb - if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + if (vnodeIsRollup(pVnode)) { + if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA1(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA2(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + } else { + if (tsdbBegin(pVnode->pTsdb) < 0) { + vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } } return 0; @@ -212,10 +229,28 @@ int vnodeCommit(SVnode *pVnode) { ASSERT(0); return -1; } - if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); - return -1; + + if(vnodeIsRollup(pVnode)) { + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { + ASSERT(0); + return -1; + } + } else { + if (tsdbCommit(pVnode->pTsdb) < 0) { + ASSERT(0); + return -1; + } } + + if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3737bcfe3b..ae134e6496 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -96,24 +96,24 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (tsdbIsRollup(pVnode)) { + if (vnodeIsRollup(pVnode)) { if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { - vError("vgId: %d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { - vError("vgId: %d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) { - vError("vgId: %d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } } else { if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) { - vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } } @@ -160,8 +160,8 @@ _err: if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + tsdbClose(VND_RSMA1(pVnode)); + tsdbClose(VND_RSMA2(pVnode)); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4eeba06027..878dd5fca4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -445,14 +445,45 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM } static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SVDropTbReq req = {0}; - SVDropTbReq rsp = {0}; + SVDropTbBatchReq req = {0}; + SVDropTbBatchRsp rsp = {0}; + SCoder coder = {0}; + int ret; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; // decode req + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + ret = tDecodeSVDropTbBatchReq(&coder, &req); + if (ret < 0) { + terrno = TSDB_CODE_INVALID_MSG; + pRsp->code = terrno; + goto _exit; + } // process req + rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + for (int iReq = 0; iReq < req.nReqs; iReq++) { + SVDropTbReq *pDropTbReq = req.pReqs + iReq; + SVDropTbRsp dropTbRsp = {0}; - // return rsp + /* code */ + ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); + if (ret < 0) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } + + taosArrayPush(rsp.pArray, &dropTbRsp); + } + +_exit: + tCoderClear(&coder); + // encode rsp (TODO) return 0; } @@ -461,7 +492,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in SSubmitRsp rsp = {0}; pRsp->code = 0; - tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // handle the request if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { pRsp->code = terrno; @@ -470,12 +501,28 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in // pRsp->msgType = TDMT_VND_SUBMIT_RSP; // vnodeProcessSubmitReq(pVnode, ptr, pRsp); - // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); pRsp->contLen = sizeof(SSubmitRsp); + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + return 0; } + +int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2aae4303eb..2e94ec8d0c 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -13,14 +13,13 @@ * along with this program. If not, see . */ -#include "query.h" -#include "plannodes.h" #include "commandInt.h" +#include "plannodes.h" +#include "query.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); - void qExplainFreeResNode(SExplainResNode *resNode) { if (NULL == resNode) { return; @@ -28,12 +27,10 @@ void qExplainFreeResNode(SExplainResNode *resNode) { taosMemoryFreeClear(resNode->pExecInfo); - SNode* node = NULL; - FOREACH(node, resNode->pChildren) { - qExplainFreeResNode((SExplainResNode *)node); - } + SNode *node = NULL; + FOREACH(node, resNode->pChildren) { qExplainFreeResNode((SExplainResNode *)node); } nodesClearList(resNode->pChildren); - + taosMemoryFreeClear(resNode); } @@ -59,24 +56,24 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosMemoryFreeClear(rsp->subplanInfo); } } - + pIter = taosHashIterate(pCtx->groupHash, pIter); } } - + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); @@ -95,7 +92,7 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d ctx->tbuf = tbuf; ctx->rows = rows; ctx->groupHash = groupHash; - + *pCtx = ctx; return TSDB_CODE_SUCCESS; @@ -110,9 +107,9 @@ _return: } int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { - int32_t tlen = 0; + int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -120,47 +117,47 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; pPhysiChildren = pTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; pPhysiChildren = pSTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; pPhysiChildren = pPrjNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; pPhysiChildren = pJoinNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; pPhysiChildren = pAggNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; pPhysiChildren = pExchNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; pPhysiChildren = pSortNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; pPhysiChildren = pIntNode->window.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; pPhysiChildren = pSessNode->window.node.pChildren; break; @@ -178,7 +175,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } } - SNode* node = NULL; + SNode *node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); @@ -195,14 +192,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SExplainRsp *rsp = NULL; + SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); return TSDB_CODE_QRY_APP_ERROR; } - + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); } @@ -217,7 +214,7 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); @@ -226,15 +223,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai int32_t code = 0; resNode->pNode = pNode; - + if (group->nodeExecInfo) { QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(&resNode->pExecInfo, group)); } - + QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); ++group->physiPlanNum; - + *pResNode = resNode; return TSDB_CODE_SUCCESS; @@ -242,15 +239,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai _return: qExplainFreeResNode(resNode); - + QRY_RET(code); } int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = *len; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = *len; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->startupCost > maxExecInfo.startupCost) { @@ -263,20 +260,20 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { maxExecInfo.numOfRows = execInfo->numOfRows; } } - + EXPLAIN_ROW_APPEND(EXPLAIN_EXECINFO_FORMAT, maxExecInfo.startupCost, maxExecInfo.totalCost, maxExecInfo.numOfRows); *len = tlen; - + return TSDB_CODE_SUCCESS; } int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = 0; - bool gotVerbose = false; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = 0; + bool gotVerbose = false; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->verboseInfo) { @@ -289,11 +286,10 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t } *len = tlen; - + return TSDB_CODE_SUCCESS; } - int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); @@ -304,7 +300,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; + row.len = len; ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { @@ -316,21 +312,21 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t return TSDB_CODE_SUCCESS; } -static uint8_t getIntervalPrecision(SIntervalPhysiNode* pIntNode) { - return ((SColumnNode*)pIntNode->window.pTspk)->node.resType.precision; +static uint8_t getIntervalPrecision(SIntervalPhysiNode *pIntNode) { + return ((SColumnNode *)pIntNode->window.pTspk)->node.resType.precision; } int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { - int32_t tlen = 0; - bool isVerboseLine = false; - char *tbuf = ctx->tbuf; - bool verbose = ctx->verbose; - SPhysiNode* pNode = pResNode->pNode; + int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; + SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -339,7 +335,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTagScanNode->pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); @@ -350,80 +346,85 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); if (tlen) { EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } + } } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_TABLE_SCAN_FORMAT, pTblScanNode->scanSeq[0], pTblScanNode->scanSeq[1]); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, + pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -431,91 +432,96 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pSTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pPrjNode->pProjections->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pPrjNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pJoinNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET( + nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pAggNode->pAggFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->totalRowSize); @@ -523,57 +529,61 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); } - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pAggNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pExchNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -582,14 +592,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSortNode->pSortKeys->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->totalRowSize); @@ -599,29 +609,31 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSortNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -630,41 +642,39 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = getIntervalPrecision(pIntNode); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), - pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), pIntNode->slidingUnit); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - - if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, getFillModeString(pIntNode->pFill->mode)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pSessNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -672,10 +682,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -686,8 +696,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSessNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -702,7 +713,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i return TSDB_CODE_SUCCESS; } - int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); @@ -712,29 +722,28 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 int32_t code = 0; QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); - SNode* pNode = NULL; - FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); - } + SNode *pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); } return TSDB_CODE_SUCCESS; } int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; - int32_t code = 0; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); if ((EXPLAIN_MODE_ANALYZE == ctx->mode) && (group->physiPlanNum != group->physiPlanExecNum)) { - qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, groupId); + qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, + groupId); QRY_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -743,21 +752,21 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { _return: qExplainFreeResNode(node); - + QRY_RET(code); } - int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = (SExplainCtx *)ctx; - int32_t rowNum = taosArrayGetSize(pCtx->rows); + int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -768,13 +777,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); // payload length - *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + *(int32_t *)rsp->data = + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; // group id - *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0; // column length - int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); // varchar column offset segment int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); @@ -782,7 +792,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { // varchar data real payload char *data = (char *)(offset + rowNum); - char* start = data; + char *start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); offset[i] = data - start; @@ -800,11 +810,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { - int32_t code = 0; + int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; SExplainGroup *pGroup = NULL; - SExplainCtx *ctx = NULL; + SExplainCtx *ctx = NULL; if (pDag->numOfSubplans <= 0) { qError("invalid subplan num:%d", pDag->numOfSubplans); @@ -817,13 +827,15 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + SHashObj *groupHash = + taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - QRY_ERR_JRET(qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); + QRY_ERR_JRET( + qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -850,7 +862,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { SExplainGroup group = {0}; group.nodeNum = 1; group.plan = plan; - + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -872,7 +884,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { *pCtx = ctx; return TSDB_CODE_SUCCESS; - + _return: qExplainFreeCtx(ctx); @@ -886,7 +898,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { } int32_t tlen = 0; - char *tbuf = pCtx->tbuf; + char *tbuf = pCtx->tbuf; EXPLAIN_SUM_ROW_NEW(EXPLAIN_RATIO_TIME_FORMAT, pCtx->ratio); EXPLAIN_SUM_ROW_END(); @@ -911,11 +923,11 @@ int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } -int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { +int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { SExplainResNode *node = NULL; - int32_t code = 0; - bool groupDone = false; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + bool groupDone = false; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { @@ -931,30 +943,32 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { - qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), group->nodeNum); + qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), + group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { - qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, groupId); + qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, + groupId); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); - + taosWUnLockLatch(&group->lock); if (groupDone && (taosHashGetSize(pCtx->groupHash) == atomic_add_fetch_32(&pCtx->groupDoneNum, 1))) { @@ -969,14 +983,13 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i return TSDB_CODE_SUCCESS; } - int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); - + _return: qExplainFreeCtx(pCtx); QRY_RET(code); @@ -984,7 +997,7 @@ _return: int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) { QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx)); - + (*pCtx)->reqStartTs = startTs; (*pCtx)->jobStartTs = taosGetTimestampUs(); @@ -994,7 +1007,7 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { int32_t code = 0; pCtx->jobDoneTs = taosGetTimestampUs(); - + atomic_store_8((int8_t *)&pCtx->execDone, true); if (taosHashGetSize(pCtx->groupHash) == atomic_load_32(&pCtx->groupDoneNum)) { @@ -1006,6 +1019,3 @@ int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } - - - diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 3dfef9b59f..bfa54be71f 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes + PRIVATE os util common function parser planner qcom vnode scalar nodes index ) target_include_directories( @@ -19,4 +19,4 @@ target_include_directories( #if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f0cb1c1107..9f834fd659 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -73,7 +73,7 @@ typedef struct SResKeyPos { } SResKeyPos; typedef struct SResultRowInfo { - SResultRowPosition *pPosition; + SResultRowPosition *pPosition; // todo remove this int32_t size; // number of result set int32_t capacity; // max capacity SResultRowPosition cur; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 59db66becc..f2f0bc2055 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,12 @@ extern "C" { #endif +typedef struct { + char* pData; + bool isNull; + int16_t type; + int32_t bytes; +} SGroupKeys, SStateKeys; #ifdef __cplusplus } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ee8cdc6b1b..0404b4d447 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -40,6 +40,7 @@ extern "C" { #include "tpagedbuf.h" #include "vnode.h" +#include "executorInt.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -118,6 +119,17 @@ typedef struct SLimit { int64_t offset; } SLimit; +typedef struct SFileBlockLoadRecorder { + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t skipBlocks; + uint32_t filterOutBlocks; + uint64_t elapsedTime; +} SFileBlockLoadRecorder; + typedef struct STaskCostInfo { int64_t created; int64_t start; @@ -131,14 +143,10 @@ typedef struct STaskCostInfo { uint64_t loadDataInCacheSize; uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t skipBlocks; - uint32_t filterOutBlocks; + + SFileBlockLoadRecorder* pRecoder; uint64_t elapsedTime; + uint64_t firstStageMergeTime; uint64_t winInfoSize; uint64_t tableInfoSize; @@ -196,7 +204,7 @@ typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggS struct SOptrBasicInfo* pInfo, char* result, int32_t length); typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); -typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param, int32_t num); typedef int32_t (*__optr_get_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain); @@ -267,7 +275,7 @@ typedef struct SOperatorFpSet { typedef struct SOperatorInfo { uint8_t operatorType; - bool blockingOptr; // block operator or not + bool blocking; // block operator or not uint8_t status; // denote if current operator is completed int32_t numOfOutput; // number of columns of the current operator results char* name; // name, used to show the query execution plan @@ -332,17 +340,14 @@ typedef struct SScanInfo { typedef struct STableScanInfo { void* dataReader; - - int32_t numOfBlocks; // extract basic running information. - int32_t numOfSkipped; - int32_t numOfBlockStatis; + SFileBlockLoadRecorder readRecorder; int64_t numOfRows; int64_t elapsedTime; - int32_t prevGroupId; // previous table group id +// int32_t prevGroupId; // previous table group id SScanInfo scanInfo; - int32_t current; - SNode* pFilterNode; // filter operator info - SqlFunctionCtx* pCtx; // next operator query context + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer + SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context SResultRowInfo* pResultRowInfo; int32_t* rowCellInfoOffset; SExprInfo* pExpr; @@ -396,7 +401,6 @@ typedef struct SSysTableScanInfo { SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; - int32_t capacity; int64_t numOfBlocks; // extract basic running information. SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; @@ -424,7 +428,7 @@ typedef struct STimeWindowSupp { SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; -typedef struct STableIntervalOperatorInfo { +typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info @@ -439,7 +443,7 @@ typedef struct STableIntervalOperatorInfo { SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; struct SFillInfo* pFillInfo; // fill info -} STableIntervalOperatorInfo; +} SIntervalAggOperatorInfo; typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; @@ -478,16 +482,8 @@ typedef struct SFillOperatorInfo { void** p; SSDataBlock* existNewGroupBlock; bool multigroupResult; - SInterval intervalInfo; } SFillOperatorInfo; -typedef struct { - char* pData; - bool isNull; - int16_t type; - int32_t bytes; -} SGroupKeys, SStateKeys; - typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; SArray* pGroupCols; // group by columns, SArray @@ -540,6 +536,7 @@ typedef struct SSessionAggOperatorInfo { SWindowRowsSup winSup; bool reptScan; // next round scan int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; } SSessionAggOperatorInfo; @@ -557,6 +554,7 @@ typedef struct SStateWindowOperatorInfo { int32_t colIndex; // start row index bool hasKey; SStateKeys stateKey; + int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; // bool reptScan; } SStateWindowOperatorInfo; @@ -613,6 +611,9 @@ typedef struct SJoinOperatorInfo { SNode *pOnCondition; } SJoinOperatorInfo; +#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) +#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) + SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, __optr_decode_fn_t decode, __optr_get_explain_fn_t explain); @@ -623,8 +624,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); + void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, @@ -642,6 +643,16 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); +void cleanupAggSup(SAggSupporter* pAggSup); +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); + +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowCellInfoOffset); + +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, + char* pData, int16_t bytes, bool masterscan, uint64_t groupId, + SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -663,10 +674,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, @@ -676,14 +689,15 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pConditions); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SExecTaskInfo* pTaskInfo); SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); + SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); @@ -704,7 +718,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 26d066d9a9..a1f45fd665 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -27,13 +27,12 @@ extern "C" { struct SSDataBlock; typedef struct SFillColInfo { -// STColumn col; // column info - SResSchema col; - int16_t functionId; // sql function id - int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN - int16_t tagIndex; // index of current tag in SFillTagColInfo array list - int32_t offset; - union {int64_t i; double d;} val; + SExprInfo *pExpr; +// SResSchema schema; +// int16_t functionId; // sql function id + int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN + int16_t tagIndex; // index of current tag in SFillTagColInfo array list + SVariant fillVal; } SFillColInfo; typedef struct { @@ -56,9 +55,10 @@ typedef struct SFillInfo { int32_t numOfCols; // number of columns, including the tags columns int32_t rowSize; // size of each row SInterval interval; - char * prevValues; // previous row of data, to generate the interpolation results - char * nextValues; // next row of data - char** pData; // original result data block involved in filling data + + SArray *prev; + SArray *next; + SSDataBlock *pSrcBlock; int32_t alloc; // data buffer size in rows SFillColInfo* pFillCol; // column info for fill operations @@ -72,7 +72,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, @@ -80,7 +80,7 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3 struct SFillColInfo* pCol, const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t getFillInfoStart(struct SFillInfo *pFillInfo); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 516afe5553..7705744694 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -154,14 +154,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); - int64_t st = 0; - - st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &newgroup); + int64_t st = taosGetTimestampUs(); + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); uint64_t el = (taosGetTimestampUs() - st); + pTaskInfo->cost.elapsedTime += el; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d8e69c4b70..a91d204efa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -41,8 +41,6 @@ #define SET_MAIN_SCAN_FLAG(runtime) ((runtime)->scanFlag = MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) - #define SDATA_BLOCK_INITIALIZER \ (SDataBlockInfo) { {0}, 0 } @@ -54,11 +52,6 @@ enum { TS_JOIN_TAG_NOT_EQUALS = 2, }; -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -97,48 +90,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -#define TSKEY_MAX_ADD(a, b) \ - do { \ - if (a < 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) > 0 && ((b) >= INT32_MAX - (a))) { \ - a = INT32_MAX; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) > 0 && ((b) >= INT64_MAX - (a))) { \ - a = INT64_MAX; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - -#define TSKEY_MIN_SUB(a, b) \ - do { \ - if (a >= 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) < 0 && ((b) <= INT32_MIN - (a))) { \ - a = INT32_MIN; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) < 0 && ((b) <= INT64_MIN - (a))) { \ - a = INT64_MIN; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo* pExprInfo) { @@ -146,39 +97,6 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { return 0; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); static bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -190,19 +108,13 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -// static SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); @@ -218,11 +130,9 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { } } -#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) -#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) - int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } @@ -440,9 +350,9 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, * | 8 bytes | actual length | * +----------+---------------+ */ -static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = @@ -499,61 +409,6 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI return pResult; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, - bool ascQuery) { - if (ascQuery) { - getAlignQueryTimeWindow(pInterval, precision, ts, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pInterval, precision, ts, w); - - int64_t key = w->skey; - while (key < ts) { // moving towards end - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -// get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win) { - STimeWindow w = {0}; - - if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, precision, ts, &w, true); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; - } - - if (w.skey > ts || w.ekey < ts) { - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } - } - return w; -} - // get the correct time window according to the handled timestamp static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) { STimeWindow w = {0}; @@ -638,75 +493,6 @@ static bool chkWindowOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset); - -static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan, - SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); - SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, id, (char*)&win->skey, - TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); - - if (pResultRow == NULL) { - *pResult = NULL; - return TSDB_CODE_SUCCESS; - } - - // set time window for current result - pResultRow->win = (*win); - *pResult = pResultRow; - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - return TSDB_CODE_SUCCESS; -} - -static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - pResult->startInterp = true; - } else { - pResult->endInterp = true; - } -} - -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - return pResult->startInterp == true; - } else { - return pResult->endInterp == true; - } -} - -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, - int16_t pos, int16_t order, int64_t* pData) { - int32_t forwardStep = 0; - - if (order == TSDB_ORDER_ASC) { - int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); - if (end >= 0) { - forwardStep = end; - - if (pData[end + pos] == ekey) { - forwardStep += 1; - } - } - } else { - int32_t end = searchFn((char*)pData, pos + 1, ekey, order); - if (end >= 0) { - forwardStep = pos - end; - - if (pData[end] == ekey) { - forwardStep += 1; - } - } - } - - assert(forwardStep >= 0); - return forwardStep; -} - static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; @@ -776,46 +562,8 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey // } //} -static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, - TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - - int32_t num = -1; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (order == TSDB_ORDER_ASC) { - if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; - } - } else { - num = pDataBlockInfo->rows - startPos; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.ekey + step; - } - } - } else { // desc - if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; - } - } else { - num = startPos + 1; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.skey + step; - } - } - } - - assert(num >= 0); - return num; -} - // query_range_start, query_range_end, window_duration, window_start, window_end -static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); @@ -829,16 +577,6 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { - int64_t* ts = (int64_t*)pColData->pData; - int32_t delta = includeEndpoint ? 1 : 0; - - int64_t duration = pWin->ekey - pWin->skey + delta; - ts[2] = duration; // set the duration - ts[3] = pWin->skey; // window start key - ts[4] = pWin->ekey + delta; // window end key -} - void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { @@ -890,77 +628,6 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* } } -static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { - int32_t order = pInfo->order; - bool ascQuery = (order == TSDB_ORDER_ASC); - - int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); - - // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || - (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { - return -1; - } - - TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; - int32_t startPos = 0; - - // tumbling time window query, a special case of sliding time window query - if (pInterval->sliding == pInterval->interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - startPos = prevPosition + factor; - } else { - if (startKey <= pDataBlockInfo->window.skey && ascQuery) { - startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { - startPos = pDataBlockInfo->rows - 1; - } else { - startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); - } - } - - /* interp query with fill should not skip time window */ - // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - // return startPos; - // } - - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if (primaryKeys == NULL) { - if (ascQuery) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { - if (ascQuery && primaryKeys[startPos] > pNext->ekey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->skey = pNext->ekey - pInterval->interval + 1; - } - } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->ekey = pNext->skey + pInterval->interval - 1; - } - } - } - - return startPos; -} - static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* pWindow) { TSKEY ekey = -1; int32_t order = TSDB_ORDER_ASC; @@ -979,41 +646,6 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* p return ekey; } -static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].start.key = INT64_MIN; - } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; - } - } -} - -static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { - if (pDataBlock == NULL) { - return; - } - - for (int32_t k = 0; k < numOfCols; ++k) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} - -static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { - TSKEY ts = TSKEY_INITIAL_VAL; - if (tsCols == NULL) { - ts = ascQuery ? win->skey : win->ekey; - } else { - int32_t offset = ascQuery ? 0 : rows - 1; - ts = tsCols[offset]; - } - - return ts; -} - static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); @@ -1101,7 +733,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column + // the last parameter is the timestamp column + if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + pInput->pPTS = pInput->pData[j]; + } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { // todo avoid case: top(k, 12), 12 is the value parameter. @@ -1179,7 +814,7 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S } int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList) { + int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; @@ -1196,7 +831,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pResult->info.rows > 0 && !createNewColModel) { - colDataMergeCol(pColInfoData, pResult->info.rows, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); } @@ -1224,7 +859,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1234,9 +869,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc if (fmIsPseudoColumnFunc(pfCtx->functionId)) { // do nothing } else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) { - // todo set the correct timestamp column - pfCtx->input.pPTS = taosArrayGet(pSrcBlock->pDataBlock, 1); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); pfCtx->fpSet.init(&pCtx[k], pResInfo); @@ -1258,14 +890,14 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pBlockList); return code; } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1282,412 +914,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc return TSDB_CODE_SUCCESS; } -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; - - SqlFunctionCtx* pCtx = pInfo->pCtx; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { - pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); - - // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - if (functionId == FUNCTION_INTERP) { - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = prevTs; - pCtx[k].start.val = v1; - - pCtx[k].end.key = curTs; - pCtx[k].end.val = v2; - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - if (prevRowIndex == -1) { - // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; - } else { - pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; - } - - pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; - } - } - } else if (functionId == FUNCTION_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; - - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - } - } -} - -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - bool ascQuery = true; - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; - - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = ascQuery ? win->skey : win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); - return true; -} - -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, - STimeWindow* win) { - int32_t order = TSDB_ORDER_ASC; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - TSKEY actualEndKey = tsCols[endRowIndex]; - TSKEY key = order ? win->ekey : win->skey; - - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || - (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return false; - } - - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); - - TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); - return true; -} - -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, - int32_t order, bool timeWindowInterpo) { - if (!timeWindowInterpo) { - return; - } - - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (pBlock->pDataBlock == NULL) { - // tscError("pBlock->pDataBlock == NULL"); - return; - } - - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY* tsCols = (TSKEY*)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - } - - // point interpolation does not require the end key time window interpolation. - // if (pointInterpQuery) { - // return; - // } - - // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; - bool interp = - setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); - } -} - -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; - - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - SArray* pUpdated = NULL; - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, POINTER_BYTES); - } - - int32_t step = 1; - bool ascScan = true; - - // int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } - - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, &pInfo->win); - bool masterScan = true; - - SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - int32_t forwardStep = 0; - TSKEY ekey = win.ekey; - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // prev time window not interpolation yet. - // int32_t curIndex = pResultRowInfo->curPos; - -#if 0 - if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - // restore current time window - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } -#endif - - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); - if (startPos < 0) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); - } - - return pUpdated; - // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); -} - -static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { - pRowSup->win.ekey = ts; - pRowSup->prevTs = ts; - pRowSup->numOfRows += 1; -} - -static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { - pRowSup->startRowIndex = rowIndex; - pRowSup->numOfRows = 0; - pRowSup->win.skey = tsList[rowIndex]; -} - -// todo handle multiple tables cases. -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - // todo find the correct time stamp column slot - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - int64_t gid = pBlock->info.groupId; - - int64_t gap = pInfo->gap; - - if (!pInfo->reptScan) { - pInfo->reptScan = true; - pInfo->winSup.prevTs = INT64_MIN; - } - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (pInfo->winSup.prevTs == INT64_MIN) { - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { - // The gap is less than the threshold, so it belongs to current session window that has been opened already. - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // start a new session window - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - // pInfo->numOfRows data belong to the current session window - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { // todo disable this @@ -1712,12 +938,12 @@ int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = binfo->pCtx; - SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId, - pTaskInfo, false, pAggSup); + SResultRow* pResultRow = + doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); assert(pResultRow != NULL); setResultRowKey(pResultRow, pData, type); - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -2367,8 +1593,8 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc STaskCostInfo* pCost = &pTaskInfo->cost; - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; +// pCost->totalBlocks += 1; +// pCost->totalRows += pBlock->info.rows; #if 0 // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. @@ -2489,7 +1715,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // if (pQueryAttr->pFilters != NULL) { // filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); // } - + // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { // filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); // } @@ -2498,69 +1724,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - if (num <= 0) { - return -1; - } - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY*)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - /* * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer @@ -2728,8 +1891,8 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t int64_t tid = 0; int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, tid, (char*)&tid, sizeof(tid), true, - groupId, pTaskInfo, false, pSup); + SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, + pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); @@ -2870,53 +2033,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } -// todo merged with the build group result. -void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, - int32_t* rowCellInfoOffset) { - size_t num = taosArrayGetSize(pUpdateList); - - for (int32_t i = 0; i < num; ++i) { - SResKeyPos* pPos = taosArrayGetP(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); - // - for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - // - struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - // if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - // continue; - // } - // - // if (pCtx[j].fpSet.process) { // TODO set the dummy function. - //// pCtx[j].fpSet.finalize(&pCtx[j]); - // pResInfo->initialized = true; - // } - // - if (pRow->numOfRows < pResInfo->numOfRes) { - pRow->numOfRows = pResInfo->numOfRes; - } - } - - releaseBufPage(pBuf, bufPage); - } -} - -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; - - // set more initial size of interval/groupby query - // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { - int32_t initialSize = 128; - // int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); - // if (code != TSDB_CODE_SUCCESS) { - // return NULL; - // } - // } else { // in other aggregate query, do not initialize the windowResInfo - // } - return pTableQueryInfo; } @@ -2929,8 +2048,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { // cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset) { +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -3015,7 +2133,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx; int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, + SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); assert(pResultRow != NULL); @@ -3031,7 +2149,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u } } - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo, SAggOperatorInfo* pAggInfo) { @@ -3045,47 +2163,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -/* - * There are two cases to handle: - * - * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including - * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. - * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be - * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there - * is a previous result generated or not. - */ -void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { - // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - // if (pResultRowInfo->curPos != -1) { - // return; - // } - - // pTableQueryInfo->win.skey = key; - // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - // STimeWindow w = TSWINDOW_INITIALIZER; - // - // TSKEY sk = TMIN(win.skey, win.ekey); - // TSKEY ek = TMAX(win.skey, win.ekey); - // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - - // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { - // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - // assert(win.ekey == pQueryAttr->window.ekey); - // } - // - // pResultRowInfo->prevSKey = w.skey; - // } - - // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; -} - /** * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, @@ -3158,10 +2235,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; + SSDataBlock* pBlock = pbInfo->pRes; + SqlFunctionCtx* pCtx = pbInfo->pCtx; + blockDataCleanup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; @@ -3203,16 +2283,16 @@ static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pOutput, int32_t capacity, void** p) { +int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t capacity) { // for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { // SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); // p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); // } - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); - pOutput->info.rows += numOfRows; + int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, pBlock, capacity - pBlock->info.rows); + pBlock->info.rows += numOfRows; - return pOutput->info.rows; + return pBlock->info.rows; } void publishOperatorProfEvent(SOperatorInfo* pOperator, EQueryProfEventType eventType) { @@ -3332,12 +2412,13 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { // // calculateOperatorProfResults(pQInfo); - qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 - " us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, - pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - // + SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; + if (pSummary->pRecoder != NULL) { + qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks, + pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + } // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); @@ -3626,30 +2707,6 @@ static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQuer #endif } -// SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { -// SQueryTableDataCond cond = { -// .colList = pQueryAttr->tableCols, -// .order = pQueryAttr->order.order, -// .numOfCols = pQueryAttr->numOfCols, -// .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, -// .loadExternalRows = false, -// }; -// -// TIME_WINDOW_COPY(cond.twindow, *win); -// return cond; -// } - -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { - STableIdInfo tidInfo; - // STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - // - // tidInfo.uid = id->uid; - // tidInfo.tid = id->tid; - // tidInfo.key = pTableQueryInfo->lastKey; - - return tidInfo; -} - // static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t // order) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); @@ -3959,9 +3016,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { goto _error; } @@ -4136,7 +3192,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4155,8 +3211,6 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { @@ -4230,7 +3284,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pBlock->info.numOfCols; @@ -4277,7 +3331,6 @@ _error: static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); -static void cleanupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*)param; @@ -4292,21 +3345,6 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); } -static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); - memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); - - // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); - // for (int32_t j = 0; j < src->base.numOfParams; ++j) { - // taosVariantAssign(&dst->base.param[j], &src->base.param[j]); - // } -} - // TODO merge aggregate super table static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -4347,8 +3385,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - bool newgroup = false; - return pOperator->fpSet.getNextFn(pOperator, &newgroup); + return pOperator->fpSet.getNextFn(pOperator); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { @@ -4519,7 +3556,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL; } -static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -4638,7 +3675,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->name = "SortedMerge"; // pOperator->operatorType = OP_SortedMerge; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = num; @@ -4666,7 +3703,7 @@ _error: return NULL; } -static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSort(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -4721,7 +3758,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; @@ -4769,7 +3806,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -4781,8 +3818,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->pScalarExprInfo != NULL) { - int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, pAggInfo->numOfScalarExpr, - NULL); + int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, + pAggInfo->numOfScalarExpr, NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; longjmp(pTaskInfo->env, pTaskInfo->code); @@ -4819,7 +3856,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; @@ -4834,8 +3871,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, - pInfo->rowCellInfoOffset, pInfo->pCtx); + doBuildResultDatablock(pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5037,7 +4073,7 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) } } -static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo* pInfo = &pProjectInfo->binfo; @@ -5075,21 +4111,18 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - bool prevVal = *newgroup; - // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - *newgroup = prevVal; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } // Return result of the previous group in the firstly. - if (*newgroup) { + if (false) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; break; @@ -5127,430 +4160,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; } -static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableIntervalOperatorInfo* pInfo = pOperator->info; - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = {0}; - bool newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); - -#if 0 // test for encode/decode result info - if(pOperator->encodeResultRow){ - char *result = NULL; - int32_t length = 0; - SAggSupporter *pSup = &pInfo->aggSup; - pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); - taosHashClear(pSup->pResultRowHashTable); - pInfo->binfo.resultRowInfo.size = 0; - pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); - if(result){ - taosMemoryFree(result); - } - } -#endif - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - OPTR_SET_OPENED(pOperator); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSDataBlock* pBlock = pInfo->binfo.pRes; - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->fpSet.getStreamResFn(pOperator, newgroup); - } else { - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBlock->info.rows == 0 ? NULL : pBlock; - } -} - -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - int32_t order = TSDB_ORDER_ASC; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - // STimeWindow win = {0}; - *newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - SArray* pUpdated = NULL; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the - // caller. Note that all the time window are not close till now. - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); - } - - finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); - - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - ASSERT(pInfo->binfo.pRes->info.rows > 0); - pOperator->status = OP_RES_TO_RETURN; - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STimeSliceOperatorInfo* pSliceInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pSliceInfo->binfo.pRes; - } - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = pQueryAttr->window; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); - // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); - } - - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - - // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); - - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; -} - -static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - STableIntervalOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - int64_t st = taosGetTimestampUs(); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - // hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pBlock->info.groupId); - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - - // initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); - OPTR_SET_OPENED(pOperator); - - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - int64_t gid = pBlock->info.groupId; - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - - int16_t bytes = pStateColInfoData->info.bytes; - int16_t type = pStateColInfoData->info.type; - - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { - continue; - } - - char* val = colDataGetData(pStateColInfoData, j); - - if (!pInfo->hasKey) { - memcpy(pInfo->stateKey.pData, val, bytes); - pInfo->hasKey = true; - - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // a new state window started - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - -static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - return pBInfo->pRes; - } - - int32_t order = TSDB_ORDER_ASC; - STimeWindow win = pTaskInfo->window; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doStateWindowAggImpl(pOperator, pInfo, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} - -static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - return pBInfo->pRes; - } - - int32_t order = TSDB_ORDER_ASC; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doSessionWindowAggImpl(pOperator, pInfo, pBlock); - } - - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} - static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, bool* newgroup, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; @@ -5562,7 +4171,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity); pInfo->existNewGroupBlock = NULL; *newgroup = true; } @@ -5571,7 +4180,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity); if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult)) { return; } @@ -5583,7 +4192,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf } } -static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doFill(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -5595,6 +4204,9 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } + // todo handle different group data interpolation + bool n = false; + bool *newgroup = &n; doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) { return pResBlock; @@ -5603,7 +4215,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream, newgroup); + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -5632,7 +4244,8 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity, pInfo->p); + blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity); // current group has no more result to return if (pResBlock->info.rows > 0) { @@ -5715,11 +4328,9 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n return TSDB_CODE_SUCCESS; } -static void cleanupAggSup(SAggSupporter* pAggSup) { +void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); - // taosHashCleanup(pAggSup->pResultRowListSet); - // taosArrayDestroy(pAggSup->pResultRowArrayList); destroyDiskbasedBuf(pAggSup->pResultBuf); } @@ -5770,7 +4381,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf } STimeWindow win = {0, INT64_MAX}; - createTableQueryInfo(pTableQueryInfo, false, win); + createTableQueryInfo(pTableQueryInfo, win); return pTableQueryInfo; } @@ -5810,7 +4421,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; @@ -5849,28 +4460,11 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(pInfo, numOfOutput); } -void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - taosMemoryFreeClear(pInfo->stateKey.pData); -} - void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - cleanupAggSup(&pInfo->aggSup); -} - -void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); @@ -5940,7 +4534,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; @@ -5962,263 +4556,17 @@ _error: return NULL; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, - destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, - destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { - STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pOperator == NULL || pInfo == NULL) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pOperator->name = "TimeSliceOperator"; - // pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, - NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, - SExecTaskInfo* pTaskInfo) { - SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->colIndex = -1; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, 4096); - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pInfo->twAggSup = *pTwAggSup; - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, - destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - pTaskInfo->code = TSDB_CODE_SUCCESS; - return NULL; -} - -SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, - SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->twAggSup = *pTwAggSupp; - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->winSup.prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, - destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - pOperator->pTaskInfo = pTaskInfo; - - code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySWindowOperatorInfo(pInfo, numOfCols); - } - - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, NULL); + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); - // TODO set correct time precision STimeWindow w = TSWINDOW_INITIALIZER; - getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, &w); + getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey, &w); int32_t order = TSDB_ORDER_ASC; pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } else { @@ -6227,14 +4575,13 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pInfo->pRes = pResBlock; pInfo->multigroupResult = multigroupResult; - pInfo->intervalInfo = *pInterval; int32_t type = TSDB_FILL_NONE; switch (fillType) { @@ -6263,16 +4610,16 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(pOperator, 4096); - int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*)fillVal, pTaskInfo->window, pResultInfo->capacity, + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type); if (code != TSDB_CODE_SUCCESS) { goto _error; } pOperator->name = "FillOperator"; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Fill; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; @@ -6483,6 +4830,7 @@ static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); +static void setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode); static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SInterval interval = { @@ -6516,15 +4864,19 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + + int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; } SInterval interval = extractIntervalInfo(pTableScanNode); - return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, + SOperatorInfo* pOperator = createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + STableScanInfo* pScanInfo = pOperator->info; + pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; + return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6532,16 +4884,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); int32_t numOfCols = 0; SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, + pScanPhyNode->node.pConditions); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -6618,14 +4969,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType}; - int32_t primaryTsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, - pTableGroupInfo, pTaskInfo); - - if (pIntervalPhyNode->pFill != NULL) { - pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, - false, pTaskInfo); - } + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTableGroupInfo, + pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; @@ -6637,12 +4983,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType}; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, &as, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -6657,13 +5004,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; + + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &num); pOptr = createJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { + SFillPhysiNode* pFillNode = (SFillPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); + + SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval; + pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); } else { ASSERT(0); } @@ -6927,7 +5283,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { goto _error; - } + } #if 0 return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); #endif @@ -7165,9 +5521,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { SJoinOperatorInfo* pJoinInfo = pOperator->info; - // SOptrBasicInfo* pInfo = &pJoinInfo->binfo; SSDataBlock* pRes = pJoinInfo->pRes; blockDataCleanup(pRes); @@ -7176,12 +5531,10 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) int32_t nrows = 0; while (1) { - bool prevVal = *newgroup; - if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { SOperatorInfo* ds1 = pOperator->pDownstream[0]; publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1, newgroup); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->leftPos = 0; @@ -7194,7 +5547,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { SOperatorInfo* ds2 = pOperator->pDownstream[1]; publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2, newgroup); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->rightPos = 0; @@ -7272,25 +5625,29 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf goto _error; } - pOperator->resultInfo.capacity = 4096; - pOperator->resultInfo.threshold = 4096 * 0.75; + initResultSizeInfo(pOperator, 4096); - // initResultRowInf - // o(&pInfo->binfo.resultRowInfo, 8); - pInfo->pRes = pResBlock; - - pOperator->name = "JoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; + SOperatorNode* pNode = (SOperatorNode*)pOnCondition; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; _error: @@ -7299,3 +5656,11 @@ _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + +void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->precision = pColumnNode->node.resType.precision; + pColumn->scale = pColumnNode->node.resType.scale; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index dd9cd52870..f5dd67217e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -256,7 +256,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } -static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -277,7 +277,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); @@ -353,7 +353,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; pOperator->pExpr = pExprInfo; @@ -537,11 +537,12 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pageIndex += 1; + blockDataUpdateTsWindow(pInfo->binfo.pRes); pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId; return pInfo->binfo.pRes; } -static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -558,7 +559,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -611,7 +612,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* } pOperator->name = "PartitionOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; pInfo->binfo.pRes = pResultBlock; diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 1f8e73b80a..3b62bdd664 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -15,7 +15,9 @@ #include "indexoperator.h" #include "executorimpl.h" +#include "index.h" #include "nodes.h" +#include "tdatablock.h" typedef struct SIFCtx { int32_t code; @@ -48,11 +50,38 @@ typedef struct SIFCtx { } while (0) typedef struct SIFParam { - SArray * result; SHashObj *pFilter; + + SArray *result; + char * condValue; + + uint8_t colValType; + col_id_t colId; + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; -typedef int32_t (*sif_func_t)(SNode *left, SNode *rigth, SIFParam *output); +static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { + if (src == OP_TYPE_GREATER_THAN) { + *dst = QUERY_GREATER_THAN; + } else if (src == OP_TYPE_GREATER_EQUAL) { + *dst = QUERY_GREATER_EQUAL; + } else if (src == OP_TYPE_LOWER_THAN) { + *dst = QUERY_LESS_THAN; + } else if (src == OP_TYPE_LOWER_EQUAL) { + *dst = QUERY_LESS_EQUAL; + } else if (src == OP_TYPE_EQUAL) { + *dst = QUERY_TERM; + } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { + *dst = QUERY_REGEX; + } else { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + +typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; @@ -60,7 +89,10 @@ static void destroyTagFilterOperatorInfo(void *param) { static void sifFreeParam(SIFParam *param) { if (param == NULL) return; + taosArrayDestroy(param->result); + taosMemoryFree(param->condValue); + taosHashCleanup(param->pFilter); } static int32_t sifGetOperParamNum(EOperatorType ty) { @@ -71,16 +103,72 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } +static int32_t sifValidateColumn(SColumnNode *cn) { + // add more check + if (cn == NULL) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + if (cn->colType != COLUMN_TYPE_TAG) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t sifGetValueFromNode(SNode *node, char **value) { + // covert data From snode; + SValueNode *vn = (SValueNode *)node; + + char * pData = nodesGetValueFromNode(vn); + SDataType *pType = &vn->node.resType; + int32_t type = pType->type; + int32_t valLen = 0; + + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + valLen = dataLen; + } else { + valLen = pType->bytes; + } + char *tv = taosMemoryCalloc(1, valLen + 1); + if (tv == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(tv, pData, valLen); + *value = tv; + + return TSDB_CODE_SUCCESS; +} + static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; - + SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); + param->colId = -1; break; } case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; + /*only support tag column*/ + SIF_ERR_RET(sifValidateColumn(cn)); + param->colId = cn->colId; + param->colValType = cn->node.resType.type; + memcpy(param->dbName, cn->dbName, sizeof(cn->dbName)); + memcpy(param->colName, cn->colName, sizeof(cn->colName)); break; } case QUERY_NODE_NODE_LIST: { @@ -89,7 +177,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->dataType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); @@ -163,58 +251,75 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu qError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } +static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, + strlen(left->colName), right->condValue, strlen(right->condValue)); + if (tm == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); -static int32_t sifLessThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLessEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; + EIndexQueryType qtype = 0; + SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + + indexMultiTermQueryAdd(mtm, tm, qtype); + int ret = indexSearch(NULL, mtm, output->result); + indexMultiTermQueryDestroy(mtm); + return ret; } -static int32_t sifEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_THAN; + return sifDoIndex(left, right, id, output); } -static int32_t sifNotEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifInFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotInFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_EQUAL; + return sifDoIndex(left, right, id, output); } -static int32_t sifMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_THAN; + return sifDoIndex(left, right, id, output); } -static int32_t sifNotMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_EQUAL; + return sifDoIndex(left, right, id, output); } -static int32_t sifDefaultFunc(SNode *left, SNode *rigth, SIFParam *output) { + +static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_EQUAL; + return sifDoIndex(left, right, id, output); +} +static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_EQUAL; + return sifDoIndex(left, right, id, output); +} +static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_IN; + return sifDoIndex(left, right, id, output); +} +static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_IN; + return sifDoIndex(left, right, id, output); +} +static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LIKE; + return sifDoIndex(left, right, id, output); +} +static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_LIKE; + return sifDoIndex(left, right, id, output); +} + +static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_MATCH; + return sifDoIndex(left, right, id, output); +} +static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NMATCH; + return sifDoIndex(left, right, id, output); +} +static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } @@ -252,17 +357,18 @@ static sif_func_t sifGetOperFn(int32_t funcId) { return sifDefaultFunc; } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { - int32_t code = 0; - SIFParam *params = NULL; - SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); - + int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + + SIFParam *params = NULL; + SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + sif_func_t operFn = sifGetOperFn(node->opType); - return operFn(node->pLeft, node->pRight, output); + return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: taosMemoryFree(params); SIF_RET(code); @@ -335,7 +441,6 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { if (ctx->code) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -387,6 +492,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { qError("index-filter failed to taosHashInit"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); SIF_ERR_RET(ctx.code); @@ -425,6 +531,7 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } + // impl later return SFLT_ACCURATE_INDEX; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c2a101e012..7e721455c3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "common/ttime.h" +#include "ttime.h" #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -36,10 +36,10 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo); +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -158,11 +158,11 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } -int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; - STaskCostInfo* pCost = &pTaskInfo->cost; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; @@ -188,11 +188,11 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - bool allHave = true; + bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allHave); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (allHave == true) { + if (allColumnsHaveAgg == true) { int32_t numOfCols = pBlock->info.numOfCols; // todo create this buffer during creating operator @@ -236,6 +236,7 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + // todo record the filter time cost doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; @@ -257,18 +258,15 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction pTableScanInfo->cond.order = TSDB_ORDER_DESC; } -static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { if (isTaskKilled(pOperator->pTaskInfo)) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); uint32_t status = 0; @@ -289,7 +287,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } -static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -298,17 +296,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - - while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + // do the ascending order traverse in the first place. + while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; @@ -323,7 +320,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { } int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; - if (pTableScanInfo->current < total) { + if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -333,21 +330,20 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { qDebug("%s start to descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - while (pTableScanInfo->current < total) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + while (pTableScanInfo->scanTimes < total) { + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 - "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -376,58 +372,59 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon pInfo->cond = *pCond; pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]}; - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; pInfo->dataBlockLoadFlag = dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pDataReader; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pDataReader; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->fpSet.getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->name = "TableScanOperator"; // for dubug purpose + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, NULL, NULL, NULL, NULL); static int32_t cost = 0; - pOperator->cost.openCost = ++cost; + + // for non-blocking operator, the open cost is always 0 + pOperator->cost.openCost = 0; pOperator->cost.totalCost = ++cost; pOperator->resultInfo.totalRows = ++cost; return pOperator; } -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { +SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - - pInfo->dataReader = pTsdbReadHandle; - pInfo->current = 0; - pInfo->prevGroupId = -1; - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet.getNextFn = doTableScanImpl; + pInfo->dataReader = pReadHandle; +// pInfo->prevGroupId = -1; + + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL); return pOperator; } -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } STableScanInfo* pTableScanInfo = pOperator->info; - *newgroup = false; STableBlockDistInfo tableBlockDist = {0}; tableBlockDist.numOfTables = 1; // TODO set the correct number of tables @@ -487,7 +484,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doBlockInfoScan; @@ -514,7 +511,7 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); } -static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; @@ -641,20 +638,22 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* } pInfo->readerHandle = streamReadHandle; - pInfo->pRes = pResBlock; - pInfo->pCondition = pCondition; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; - pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doStreamBlockScan; - pOperator->fpSet.closeFn = operatorDummyCloseFn; + pOperator->fpSet.closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); + return pOperator; } @@ -676,14 +675,12 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { switch (nType) { case QUERY_NODE_OPERATOR: { SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { *(int32_t*)pContext = 1; return DEAL_RES_CONTINUE; } *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; } case QUERY_NODE_COLUMN: { @@ -714,19 +711,17 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { default: break; } - return DEAL_RES_CONTINUE; } -void getDBNameFromCondition(SNode* pCondition, char* dbName) { +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { if (NULL == pCondition) { return; } - - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*) dbName); } -static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { +static int32_t loadSysTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { SOperatorInfo* operator=(SOperatorInfo*) param; SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; if (TSDB_CODE_SUCCESS == code) { @@ -751,13 +746,14 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } SFilterInfo* filter = NULL; - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); + + int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); SSDataBlock* px = createOneDataBlock(pInfo->pRes, false); @@ -800,66 +796,36 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { static SSDataBlock* buildSysTableMetaBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = 10; - pBlock->info.hasVarCol = true; + + size_t size = 0; + const SSysTableMeta *pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for(int32_t i = 0; i < size; ++i) { + if(strcmp(pMeta[i].name, TSDB_INS_TABLE_USER_TABLES) == 0) { + index = i; + break; + } + } pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = 1; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + for(int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = {0}; + colInfoData.info.colId = i + 1; + colInfoData.info.type = pMeta[index].schema[i].type; + colInfoData.info.bytes = pMeta[index].schema[i].bytes; + taosArrayPush(pBlock->pDataBlock, &colInfoData); + } - colInfoData.info.colId = 2; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 3; - colInfoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfoData.info.bytes = 8; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 4; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 5; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 6; - colInfoData.info.type = TSDB_DATA_TYPE_BIGINT; - colInfoData.info.bytes = 8; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 7; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 8; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 9; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = 512 + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 10; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = 20 + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + pBlock->info.numOfCols = pMeta[index].colNum; + pBlock->info.hasVarCol = true; return pBlock; } -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -873,7 +839,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - buildSysDbTableInfo(pInfo); + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); doFilterResult(pInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -901,7 +867,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { varDataSetLen(dbname, strlen(varDataVal(dbname))); SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pInfo->capacity); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; while (metaTbCursorNext(pInfo->pCur) == 0) { @@ -982,7 +948,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { pColInfoData = taosArrayGet(p->pDataBlock, 9); colDataAppend(pColInfoData, numOfRows, str, false); - if (++numOfRows >= pInfo->capacity) { + if (++numOfRows >= pOperator->resultInfo.capacity) { break; } } @@ -1027,7 +993,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { pMsgSendInfo->msgInfo.pData = buf1; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadSysTableContentCb; + pMsgSendInfo->fp = loadSysTableCallback; int64_t transporterId = 0; int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); @@ -1065,9 +1031,9 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { } } -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo) { +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pInfo->capacity); + blockDataEnsureCapacity(p, capacity); size_t size = 0; const SSysTableMeta* pSysDbTableMeta = NULL; @@ -1138,18 +1104,19 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return NULL; } - pInfo->accountId = accountId; + pInfo->accountId = accountId; pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; + + initResultSizeInfo(pOperator, 4096); tNameAssign(&pInfo->name, pName); const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*) readHandle; - blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { tsem_init(&pInfo->ready, 0, 0); pInfo->epSet = epset; @@ -1178,12 +1145,12 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe #endif } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; @@ -1191,9 +1158,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return pOperator; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { #if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -1327,29 +1293,29 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = pReaderHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; + pInfo->pReader = readHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; return pOperator; + _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index aeed07c636..018f0016a2 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -13,20 +13,21 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" -#include "querynodes.h" - #include "taosdef.h" #include "tmsg.h" #include "ttypes.h" -#include "tfill.h" -#include "function.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" +#include "function.h" +#include "tdatablock.h" +#include "executorInt.h" +#include "querynodes.h" +#include "tfill.h" + #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) @@ -37,168 +38,208 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { continue; } - char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + char* val1 = elePtrAt(data[j], pSchema->bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - -// assert (pTag->col.colId == pCol->col.colId); - assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); + assignVal(val1, pTag->tagVal, pSchema->bytes, pSchema->type); } } -static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { +static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < numOfCol; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); - setNull(output, pCol->col.type, pCol->col.bytes); + for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); + colDataAppendNULL(p, rowIndex); } } -static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { - char* prev = pFillInfo->prevValues; - char* next = pFillInfo->nextValues; +#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) +#define GET_SRC_SLOT_ID(_p) ((_p)->pExpr->base.pParam[0].pCol->slotId) +static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); + +static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSDataBlock* pSrcBlock, int64_t ts, bool outOfBound) { SPoint point1, point2, point; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); + SColumnInfoData *pCol0 = taosArrayGet(pBlock->pDataBlock, 0); + char* val = colDataGetData(pCol0, index); + *(TSKEY*) val = pFillInfo->currentKey; // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { - char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev : pFillInfo->next; - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - // TODO : linear interpolation supports NULL value - if (prev != NULL && !outOfBound) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - int16_t type = pCol->col.type; - int16_t bytes = pCol->col.bytes; - - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); - if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { - setNull(val1, pCol->col.type, bytes); - continue; - } - - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->offset}; - point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; - point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); - } - } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else { // fill the default value */ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)) { continue; } - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); + } + } else if (pFillInfo->type == TSDB_FILL_NEXT) { + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next : pFillInfo->prev; + + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); + } + } else if (pFillInfo->type == TSDB_FILL_LINEAR) { + // TODO : linear interpolation supports NULL value + if (outOfBound) { + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); + + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + + int16_t type = pCol->pExpr->base.resSchema.type; + SGroupKeys* pKey = taosArrayGet(pFillInfo->prev, i); + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { + colDataAppendNULL(pDstCol, index); + continue; + } + + SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev, 0); + int64_t prevTs = *(int64_t*)pKey1->pData; + + SColumnInfoData* pSrcCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + char* data = colDataGetData(pSrcCol, pFillInfo->index); + + point1 = (SPoint){.key = prevTs, .val = pKey->pData}; + point2 = (SPoint){.key = ts, .val = data}; + + int64_t out = 0; + point = (SPoint){.key = pFillInfo->currentKey, .val = &out}; + taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); + + colDataAppend(pDstCol, index, (const char*)&out, false); + } + } + } else if (pFillInfo->type == TSDB_FILL_NULL) { // fill with NULL + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { // fill with user specified value for each column + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { + continue; + } + + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } } } - setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, - pFillInfo->interval.precision); +// setTagsValue(pFillInfo, data, index); + SInterval* pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); pFillInfo->numOfCurrent++; } -static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { - if (*next != NULL) { +void doSetVal(SColumnInfoData* pDstCol, int32_t rowIndex, const SGroupKeys* pKey) { + if (pKey->isNull) { + colDataAppendNULL(pDstCol, rowIndex); + } else { + colDataAppend(pDstCol, rowIndex, pKey->pData, false); + } +} + +static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { + if (taosArrayGetSize(pFillInfo->next) > 0) { return; } - *next = taosMemoryCalloc(1, pFillInfo->rowSize); - for (int i = 1; i < pFillInfo->numOfCols; i++) { + for (int i = 0; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->offset, pCol->col.type, pCol->col.bytes); + + SGroupKeys key = {0}; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + key.pData = taosMemoryMalloc(pSchema->bytes); + key.isNull = true; + key.bytes = pSchema->bytes; + key.type = pSchema->type; + + taosArrayPush(pFillInfo->next, &key); + + key.pData = taosMemoryMalloc(pSchema->bytes); + taosArrayPush(pFillInfo->prev, &key); } } -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) { - int32_t rowIndex = pFillInfo->index; +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); + +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); + int32_t srcSlotId = GET_SRC_SLOT_ID(&pFillInfo->pFillCol[i]); + + SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + bool isNull = colDataIsNull_s(pSrcCol, rowIndex); + char* p = colDataGetData(pSrcCol, rowIndex); + saveColData(pRow, i, p, isNull); } } -static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { +static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t outputRows) { pFillInfo->numOfCurrent = 0; - char** srcData = pFillInfo->pData; - char** prev = &pFillInfo->prevValues; - char** next = &pFillInfo->nextValues; + // todo make sure the first column is always the primary timestamp column? + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); + bool ascFill = FILL_IS_ASC_FILL(pFillInfo); - if (FILL_IS_ASC_FILL(pFillInfo)) { - assert(pFillInfo->currentKey >= pFillInfo->start); - } else { - assert(pFillInfo->currentKey <= pFillInfo->start); - } +#if 0 + ASSERT(ascFill && (pFillInfo->currentKey >= pFillInfo->start) || (!ascFill && (pFillInfo->currentKey <= pFillInfo->start))); +#endif while (pFillInfo->numOfCurrent < outputRows) { - int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; + int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index]; // set the next value for interpolation - if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - initBeforeAfterDataBuf(pFillInfo, next); - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); } - if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - - // fill the gap between two actual input rows - while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, data, srcData, ts, false); + if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + // fill the gap between two input rows + while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); } // output buffer is full, abort @@ -208,61 +249,66 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR } } else { assert(pFillInfo->currentKey == ts); - initBeforeAfterDataBuf(pFillInfo, prev); + if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { - initBeforeAfterDataBuf(pFillInfo, next); ++pFillInfo->index; - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); --pFillInfo->index; } // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } - char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); - char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); - if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) || - (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlotId); + SColumnInfoData* pSrc = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + char* src = colDataGetData(pSrc, pFillInfo->index); + if (i == 0 || (/*pCol->functionId != FUNCTION_COUNT &&*/ !colDataIsNull_s(pSrc, pFillInfo->index)) /*|| + (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)*/) { + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->offset, pCol->col.bytes, pCol->col.type); + SGroupKeys *pKey = taosArrayGet(pFillInfo->prev, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); + } else if (pFillInfo->type == TSDB_FILL_NULL) { + colDataAppendNULL(pDst, pFillInfo->numOfCurrent); } else if (pFillInfo->type == TSDB_FILL_NEXT) { - if (*next) { - assignVal(output, *next + pCol->offset, pCol->col.bytes, pCol->col.type); - } else { - setNull(output, pCol->col.type, pCol->col.bytes); - } + SGroupKeys *pKey = taosArrayGet(pFillInfo->next, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else { - assignVal(output, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + colDataAppend(pDst, pFillInfo->numOfCurrent, (char*)&pVar->i, false); } } } // set the tag value for final result - setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); +// setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); + SInterval *pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ - if (pFillInfo->index >= pFillInfo->numOfRows) { - taosMemoryFreeClear(*next); - } - +// if (pFillInfo->index >= pFillInfo->numOfRows) { +// taosMemoryFreeClear(*next); +// } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; return pFillInfo->numOfCurrent; } @@ -271,14 +317,24 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR return pFillInfo->numOfCurrent; } -static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull) { + SGroupKeys *pKey = taosArrayGet(rowBuf, columnIndex); + if (isNull) { + pKey->isNull = true; + } else { + memcpy(pKey->pData, src, pKey->bytes); + pKey->isNull = false; + } +} + +static int64_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. */ pFillInfo->numOfCurrent = 0; while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true); + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, pFillInfo->start, true); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -295,15 +351,15 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t int32_t k = 0; for (int32_t i = 0; i < numOfCols; ++i) { SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = NULL; + SResSchema* pSchema = &pColInfo->pExpr->base.resSchema; - if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) { + if (TSDB_COL_IS_TAG(pColInfo->flag) || pSchema->type == TSDB_DATA_TYPE_BINARY) { numOfTags += 1; bool exists = false; int32_t index = -1; for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.slotId) { + if (pFillInfo->pTags[j].col.colId == pSchema->slotId) { exists = true; index = j; break; @@ -311,12 +367,12 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.slotId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; + SSchema* pSchema1 = &pFillInfo->pTags[k].col; + pSchema1->colId = pSchema->slotId; + pSchema1->type = pSchema->type; + pSchema1->bytes = pSchema->bytes; - pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pSchema->bytes); pColInfo->tagIndex = k; k += 1; @@ -325,7 +381,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } } - rowsize += pColInfo->col.bytes; + rowsize += pSchema->bytes; } pFillInfo->numOfTags = numOfTags; @@ -355,7 +411,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } taosResetFillInfo(pFillInfo, skey); - pFillInfo->order = order; switch(fillType) { @@ -364,6 +419,7 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag case FILL_MODE_NULL: pFillInfo->type = TSDB_FILL_NULL; break; case FILL_MODE_LINEAR: pFillInfo->type = TSDB_FILL_LINEAR;break; case FILL_MODE_NEXT: pFillInfo->type = TSDB_FILL_NEXT; break; + case FILL_MODE_VALUE: pFillInfo->type = TSDB_FILL_SET_VALUE; break; default: terrno = TSDB_CODE_INVALID_PARA; return NULL; @@ -376,7 +432,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->alloc = capacity; pFillInfo->id = id; pFillInfo->interval = *pInterval; - pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); @@ -385,6 +440,11 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } // } + pFillInfo->next = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + pFillInfo->prev = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + + initBeforeAfterDataBuf(pFillInfo); + pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); return pFillInfo; @@ -405,18 +465,15 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - taosMemoryFreeClear(pFillInfo->prevValues); - taosMemoryFreeClear(pFillInfo->nextValues); + taosArrayDestroy(pFillInfo->prev); + taosArrayDestroy(pFillInfo->next); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } taosMemoryFreeClear(pFillInfo->pTags); - - taosMemoryFreeClear(pFillInfo->pData); taosMemoryFreeClear(pFillInfo->pFillCol); - taosMemoryFreeClear(pFillInfo); return NULL; } @@ -436,18 +493,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) } void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); - pFillInfo->pData[i] = pColData->pData; - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.slotId); - memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? - } - } + pFillInfo->pSrcBlock = (SSDataBlock*) pInput; } bool taosFillHasMoreResults(SFillInfo* pFillInfo) { @@ -465,8 +511,9 @@ bool taosFillHasMoreResults(SFillInfo* pFillInfo) { } int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; + SColumnInfoData* pCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); + int64_t* tsList = (int64_t*) pCol->pData; int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; @@ -513,7 +560,7 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* return TSDB_CODE_SUCCESS; } -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); @@ -521,9 +568,9 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t cap // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - appendFilledResult(pFillInfo, output, numOfRes); + appendFilledResult(pFillInfo, p, numOfRes); } else { - fillResultImpl(pFillInfo, output, (int32_t) numOfRes); + fillResultImpl(pFillInfo, p, (int32_t) numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); } @@ -538,28 +585,30 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { return pFillInfo->start; } -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val) { - int32_t offset = 0; - - struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); +SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* pValNode) { + SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } + size_t len = (pValNode != NULL)? LIST_LENGTH(pValNode->pNodeList):0; for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; + SExprInfo* pExprInfo = &pExpr[i]; + pFillCol[i].pExpr = pExprInfo; + pFillCol[i].tagIndex = -2; - pFillCol[i].col = pExprInfo->base.resSchema; - pFillCol[i].offset = offset; - pFillCol[i].tagIndex = -2; + // todo refactor + if (len > 0) { + // if the user specified value is less than the column, alway use the last one as the fill value + int32_t index = (i >= len)? (len - 1):i; + + SValueNode* pv = (SValueNode*)nodesListGetNode(pValNode->pNodeList, index); + valueNodeToVariant(pv, &pFillCol[i].fillVal); + } if (pExprInfo->base.numOfParams > 0) { pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query } -// pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; -// pFillCol[i].val.d = *val; - - offset += pExprInfo->base.resSchema.bytes; } return pFillCol; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c new file mode 100644 index 0000000000..580614dc02 --- /dev/null +++ b/source/libs/executor/src/timewindowoperator.c @@ -0,0 +1,1462 @@ +#include "ttime.h" +#include "tdatablock.h" +#include "executorimpl.h" + +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including + * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { + // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + // if (pResultRowInfo->curPos != -1) { + // return; + // } + + // pTableQueryInfo->win.skey = key; + // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + // STimeWindow w = TSWINDOW_INITIALIZER; + // + // TSKEY sk = TMIN(win.skey, win.ekey); + // TSKEY ek = TMAX(win.skey, win.ekey); + // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); + + // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + // assert(win.ekey == pQueryAttr->window.ekey); + // } + // + // pResultRowInfo->prevSKey = w.skey; + // } + + // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +} + +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { + TSKEY ts = TSKEY_INITIAL_VAL; + if (tsCols == NULL) { + ts = ascQuery ? win->skey : win->ekey; + } else { + int32_t offset = ascQuery ? 0 : rows - 1; + ts = tsCols[offset]; + } + + return ts; +} + +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, + bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, precision, ts, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pInterval, precision, ts, w); + + int64_t key = w->skey; + while (key < ts) { // moving towards end + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, + SInterval* pInterval, int32_t precision, STimeWindow* win) { + STimeWindow w = {0}; + + if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pInterval, precision, ts, &w, true); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + } + + if (w.skey > ts || w.ekey < ts) { + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + } + return w; +} + +static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan, + SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + assert(win->skey <= win->ekey); + SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, + masterscan, tableGroupId, pTaskInfo, true, pAggSup); + + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + return TSDB_CODE_SUCCESS; +} + +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { + int64_t* ts = (int64_t*)pColData->pData; + int32_t delta = includeEndpoint ? 1 : 0; + + int64_t duration = pWin->ekey - pWin->skey + delta; + ts[2] = duration; // set the duration + ts[3] = pWin->skey; // window start key + ts[4] = pWin->ekey + delta; // window end key +} + +static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { + pRowSup->win.ekey = ts; + pRowSup->prevTs = ts; + pRowSup->numOfRows += 1; +} + +static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { + pRowSup->startRowIndex = rowIndex; + pRowSup->numOfRows = 0; + pRowSup->win.skey = tsList[rowIndex]; +} + +static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, + int16_t pos, int16_t order, int64_t* pData) { + int32_t forwardStep = 0; + + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; + + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } + } + } + + assert(forwardStep >= 0); + return forwardStep; +} + +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + if (num <= 0) { + return -1; + } + + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY*)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, + TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, + int32_t order) { + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + + int32_t num = -1; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (order == TSDB_ORDER_ASC) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } else { + num = pDataBlockInfo->rows - startPos; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } else { + num = startPos + 1; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = tw->skey + pInterval->interval - 1; + return; + } + + int64_t key = tw->skey, interval = pInterval->interval; + // convert key to second + key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + if (pInterval->intervalUnit == 'y') { + interval *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + mon = (int)(mon + interval); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + tw->ekey -= 1; +} + +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SExprInfo* pExpr = pOperator->pExpr; + + SqlFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; + int16_t index = pColIndex->colIndex; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + + // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == FUNCTION_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (prevRowIndex == -1) { + // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; + } else { + pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; + } + + pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; + } + } + } else if (functionId == FUNCTION_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } + } + } +} + +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, + STimeWindow* win) { + bool ascQuery = true; + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery ? win->skey : win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, + RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, + SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, + STimeWindow* win) { + int32_t order = TSDB_ORDER_ASC; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + TSKEY actualEndKey = tsCols[endRowIndex]; + TSKEY key = order ? win->ekey : win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || + (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { + int32_t order = pInfo->order; + bool ascQuery = (order == TSDB_ORDER_ASC); + + int32_t precision = pInterval->precision; + getNextTimeWindow(pInterval, precision, order, pNext); + + // next time window is not in current block + if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || + (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { + return -1; + } + + TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; + int32_t startPos = 0; + + // tumbling time window query, a special case of sliding time window query + if (pInterval->sliding == pInterval->interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + startPos = prevPosition + factor; + } else { + if (startKey <= pDataBlockInfo->window.skey && ascQuery) { + startPos = 0; + } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { + startPos = pDataBlockInfo->rows - 1; + } else { + startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); + } + } + + /* interp query with fill should not skip time window */ + // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { + // return startPos; + // } + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (primaryKeys == NULL) { + if (ascQuery) { + assert(pDataBlockInfo->window.skey <= pNext->ekey); + } else { + assert(pDataBlockInfo->window.ekey >= pNext->skey); + } + } else { + if (ascQuery && primaryKeys[startPos] > pNext->ekey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->skey = pNext->ekey - pInterval->interval + 1; + } + } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->ekey = pNext->skey + pInterval->interval - 1; + } + } + } + + return startPos; +} + +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, + int32_t order, bool timeWindowInterpo) { + if (!timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (pBlock->pDataBlock == NULL) { + // tscError("pBlock->pDataBlock == NULL"); + return; + } + + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* tsCols = (TSKEY*)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, + pBlock->pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + // if (pointInterpQuery) { + // return; + // } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + bool interp = + setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); + } +} + +static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { + if (pDataBlock == NULL) { + return; + } + + for (int32_t k = 0; k < numOfCols; ++k) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} + +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + SArray* pUpdated = NULL; + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + pUpdated = taosArrayInit(4, POINTER_BYTES); + } + + int32_t step = 1; + bool ascScan = true; + + // int32_t prevIndex = pResultRowInfo->curPos; + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } + + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, &pInfo->win); + bool masterScan = true; + SResultRow* pResult = NULL; + + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + int32_t forwardStep = 0; + TSKEY ekey = win.ekey; + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // prev time window not interpolation yet. + // int32_t curIndex = pResultRowInfo->curPos; + +#if 0 + if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = getResultRow(pResultRowInfo, j); + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + // restore current time window + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } +#endif + + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + if (pInfo->timeWindowInterpo) { + int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; + saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); + } + + return pUpdated; + // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); +} + +static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SIntervalAggOperatorInfo* pInfo = pOperator->info; + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); + +#if 0 // test for encode/decode result info + if(pOperator->encodeResultRow){ + char *result = NULL; + int32_t length = 0; + SAggSupporter *pSup = &pInfo->aggSup; + pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); + taosHashClear(pSup->pResultRowHashTable); + pInfo->binfo.resultRowInfo.size = 0; + pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); + if(result){ + taosMemoryFree(result); + } + } +#endif + } + + closeAllResultRows(&pInfo->binfo.resultRowInfo); + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, + &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + int64_t gid = pBlock->info.groupId; + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfOutput; + int16_t bytes = pStateColInfoData->info.bytes; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { + continue; + } + + char* val = colDataGetData(pStateColInfoData, j); + + if (!pInfo->hasKey) { + memcpy(pInfo->stateKey.pData, val, bytes); + pInfo->hasKey = true; + + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // a new state window started + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStateWindowOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + STimeWindow win = pTaskInfo->window; + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doStateWindowAggImpl(pOperator, pInfo, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { + SIntervalAggOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pBlock = pInfo->binfo.pRes; + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + return pOperator->fpSet.getStreamResFn(pOperator); + } else { + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + + if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBlock->info.rows == 0 ? NULL : pBlock; + } +} + +// todo merged with the build group result. +static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, + int32_t* rowCellInfoOffset) { + size_t num = taosArrayGetSize(pUpdateList); + + for (int32_t i = 0; i < num; ++i) { + SResKeyPos* pPos = taosArrayGetP(pUpdateList, i); + + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); + + for (int32_t j = 0; j < numOfOutput; ++j) { + SResultRowEntryInfo* pEntry = getResultCell(pRow, j, rowCellInfoOffset); + if (pRow->numOfRows < pEntry->numOfRes) { + pRow->numOfRows = pEntry->numOfRes; + } + } + + releaseBufPage(pBuf, bufPage); + } +} + +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + SIntervalAggOperatorInfo* pInfo = pOperator->info; + int32_t order = TSDB_ORDER_ASC; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + // STimeWindow win = {0}; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + SArray* pUpdated = NULL; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + } + + finalizeUpdatedResult(pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + + ASSERT(pInfo->binfo.pRes->info.rows > 0); + pOperator->status = OP_RES_TO_RETURN; + + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} + +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + taosMemoryFreeClear(pInfo->stateKey.pData); +} + +void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); +} + +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamTimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +// todo handle multiple tables cases. +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfOutput; + int64_t gid = pBlock->info.groupId; + + int64_t gap = pInfo->gap; + + if (!pInfo->reptScan) { + pInfo->reptScan = true; + pInfo->winSup.prevTs = INT64_MIN; + } + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (pInfo->winSup.prevTs == INT64_MIN) { + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { + // The gap is less than the threshold, so it belongs to current session window that has been opened already. + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // start a new session window + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + // pInfo->numOfRows data belong to the current session window + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSessionAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doSessionWindowAggImpl(pOperator, pInfo, pBlock); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STimeSliceOperatorInfo* pSliceInfo = pOperator->info; + if (pOperator->status == OP_RES_TO_RETURN) { + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pSliceInfo->binfo.pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); + // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); + + // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); + + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; +} + +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { + STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL || pInfo == NULL) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pOperator->name = "TimeSliceOperator"; + // pOperator->operatorType = OP_AllTimeWindow; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId, + SExecTaskInfo* pTaskInfo) { + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->colIndex = -1; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, 4096); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pInfo->twAggSup = *pTwAggSup; + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, + destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_SUCCESS; + return NULL; +} + +void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->twAggSup = *pTwAggSupp; + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->tsSlotId = tsSlotId; + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->winSup.prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, + destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->pTaskInfo = pTaskInfo; + + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySWindowOperatorInfo(pInfo, numOfCols); + } + + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} \ No newline at end of file diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index de8df7b916..f534cf0917 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -55,7 +55,7 @@ typedef struct SDummyInputInfo { SSDataBlock* pBlock; } SDummyInputInfo; -SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; @@ -121,7 +121,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { return pBlock; } -SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index e19a332e66..3869a5d7b2 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -39,6 +39,7 @@ extern "C" { #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) #define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) +#define FUNC_MGT_SELECT_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(13) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 8f3c88900d..38922833f9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -438,6 +438,11 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -465,7 +470,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "min", .type = FUNCTION_TYPE_MIN, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, @@ -476,7 +481,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "max", .type = FUNCTION_TYPE_MAX, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, @@ -908,7 +913,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, @@ -974,6 +979,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = NULL, .sprocessFunc = toJsonFunction, .finalizeFunc = NULL + }, + { + .name = "_select_value", + .type = FUNCTION_TYPE_SELECT_VALUE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .translateFunc = translateSelectValue, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL } }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index bb01af420d..0eba442e66 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -78,6 +78,8 @@ typedef struct SDiffInfo { int64_t i64; double d64; } prev; + + int64_t prevTs; } SDiffInfo; typedef struct SSpreadInfo { @@ -1196,9 +1198,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { bool isFirstBlock = (pDiffInfo->hasPrev == false); int32_t numOfElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - // int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; TSKEY* tsList = (int64_t*)pInput->pPTS->pData; @@ -1206,44 +1205,86 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { switch (pInputCol->info.type) { case TSDB_DATA_TYPE_INT: { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { - int32_t pos = startOffset + (isFirstBlock ? (numOfElems - 1) : numOfElems); - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); - if (tsList != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + if (pCtx->order == TSDB_ORDER_ASC) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t pos = startOffset + (isFirstBlock ? (numOfElems - 1) : numOfElems); + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (pDiffInfo->includeNull) { + colDataSetNull_f(pOutput->nullbitmap, pos); + if (tsList != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + + numOfElems += 1; + } + continue; + } + + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); } - numOfElems += 1; + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } } - continue; - } - int32_t v = *(int32_t*)colDataGetData(pInputCol, i); - if (pDiffInfo->hasPrev) { - int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null - if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); - } else { + pDiffInfo->prev.i64 = v; + pDiffInfo->hasPrev = true; + numOfElems++; + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + int32_t pos = startOffset + numOfElems; + + // there is a row of previous data block to be handled in the first place. + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(pDiffInfo->prev.i64 - v); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); + } + + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &pDiffInfo->prevTs); + } + pDiffInfo->hasPrev = false; + } + + // it is not the last row of current block + if (i < pInput->numOfRows + pInput->startRowIndex - 1) { + int32_t next = *(int32_t*)colDataGetData(pInputCol, i + 1); + + int32_t delta = v - next; // direct previous may be null colDataAppendInt32(pOutput, pos, &delta); - } - if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + } else { + pDiffInfo->prev.i64 = v; + if (pTsOutput != NULL) { + pDiffInfo->prevTs = tsList[i]; + } + pDiffInfo->hasPrev = true; } + numOfElems++; } - pDiffInfo->prev.i64 = v; - pDiffInfo->hasPrev = true; - numOfElems++; } break; } case TSDB_DATA_TYPE_BIGINT: { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } @@ -1378,7 +1419,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { } // initial value is not set yet - if (!pDiffInfo->hasPrev || numOfElems <= 0) { + if (numOfElems <= 0) { /* * 1. current block and blocks before are full of null * 2. current block may be null value @@ -1386,15 +1427,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { assert(pCtx->hasNull); return 0; } else { - // for (int t = 0; t < pCtx->tagInfo.numOfTagCols; ++t) { - // SqlFunctionCtx* tagCtx = pCtx->tagInfo.pTagCtxList[t]; - // if (tagCtx->functionId == TSDB_FUNC_TAG_DUMMY) { - // aAggs[TSDB_FUNC_TAGPRJ].xFunction(tagCtx); - // } - // } - - int32_t forwardStep = (isFirstBlock) ? numOfElems - 1 : numOfElems; - return forwardStep; + return (isFirstBlock) ? numOfElems - 1 : numOfElems; } } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index b505f2e8ec..f8ef0f7d20 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -145,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } + bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index df46999379..9aac2bfe0c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1176,7 +1176,7 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte // input: interBuf // output: resultData int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { - int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int8_t callType = TSDB_UDF_CALL_AGG_FIN; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; } @@ -1243,12 +1243,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } SUdfUvSession *session = (SUdfUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; memset(udfRes, 0, envSize); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->session = (SUdfUvSession *)handle; SUdfInterBuf buf = {0}; if (callUdfAggInit(handle, &buf) != 0) { @@ -1260,7 +1260,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; int32_t numOfCols = pInput->numOfInputCols; @@ -1320,13 +1319,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf, - .bufLen = session->outputLen, - .numOfResult = udfRes->finalResNum}; + SUdfInterBuf resultBuf = {0}; SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; callUdfAggFinalize(session, &state, &resultBuf); + + udfRes->finalResBuf = resultBuf.buf; + udfRes->finalResNum = resultBuf.numOfResult; + teardownUdf(session); if (resultBuf.numOfResult == 1) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ae24a832c8..06fa49e1c2 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -124,7 +124,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char *finishSuffix = "_finish"; strncpy(finishFuncName, processFuncName, strlen(processFuncName)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc)); + uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); //TODO: merge } return 0; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 83187c5855..69ed515d2b 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -27,7 +27,7 @@ int32_t udf2_start(SUdfInterBuf *buf) { int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { int64_t sumSquares = *(int64_t*)interBuf->buf; for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { SUdfColumn* col = block->udfCols[i]; //TODO: check the bitmap for null value int32_t* rows = (int32_t*)col->colData.fixLenCol.data; @@ -35,7 +35,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte } } - *(int64_t*)newInterBuf = sumSquares; + *(int64_t*)(newInterBuf->buf) = sumSquares; newInterBuf->bufLen = sizeof(int64_t); //TODO: if all null value, numOfResult = 0; newInterBuf->numOfResult = 1; diff --git a/source/libs/index/inc/indexFst.h b/source/libs/index/inc/indexFst.h index 39ad5ffa8c..0a360c1c72 100644 --- a/source/libs/index/inc/indexFst.h +++ b/source/libs/index/inc/indexFst.h @@ -52,7 +52,6 @@ typedef struct FstRange { uint64_t end; } FstRange; -typedef enum { GE, GT, LE, LT } RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; @@ -174,9 +173,9 @@ Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) -#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) -#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) -#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) +#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) +#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) +#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) typedef struct FstLastTransition { uint8_t inp; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 37318767c7..5c7b8b9afe 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -34,6 +34,7 @@ extern "C" { #endif +typedef enum { LT, LE, GT, GE } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { @@ -57,7 +58,7 @@ struct SIndex { char* path; - SIndexStat stat; + SIndexStat stat; TdThreadMutex mtx; }; @@ -165,7 +166,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 814d61afd7..f1676ed411 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -68,7 +68,7 @@ extern "C" { */ void iIntersection(SArray *interResults, SArray *finalResult); -/* multi sorted result intersection +/* multi sorted result union * input: [1, 2, 4, 5] * [2, 3, 4, 5] * [1, 4, 5] @@ -76,7 +76,7 @@ void iIntersection(SArray *interResults, SArray *finalResult); */ void iUnion(SArray *interResults, SArray *finalResult); -/* sorted array +/* see example * total: [1, 2, 4, 5, 7, 8] * except: [4, 5] * return: [1, 2, 7, 8] saved in total diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 7d52abcd1b..83b5025ad0 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -175,55 +175,19 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { -#ifdef USE_LUCENE - EIndexOperatorType opera = multiQuerys->opera; - - int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); - char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); - int* types = taosMemoryMalloc(sizeof(int) * nQuery); - - for (int i = 0; i < nQuery; i++) { - SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); - SIndexTerm* term = p->field_value; - - fields[i] = taosMemoryCalloc(1, term->nKey + 1); - keys[i] = taosMemoryCalloc(1, term->nVal + 1); - - memcpy(fields[i], term->key, term->nKey); - memcpy(keys[i], term->val, term->nVal); - types[i] = (int)(p->type); - } - int* tResult = NULL; - int tsz = 0; - index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } - - for (int i = 0; i < nQuery; i++) { - taosMemoryFree(fields[i]); - taosMemoryFree(keys[i]); - } - taosMemoryFree(fields); - taosMemoryFree(keys); - taosMemoryFree(types); -#endif - #ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys - SArray* interResults = taosArrayInit(4, POINTER_BYTES); + SArray* iRslts = taosArrayInit(4, POINTER_BYTES); int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); - SArray* tResult = NULL; - indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void*)&tResult); + SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i); + SArray* trslt = NULL; + indexTermSearch(index, qterm, &trslt); + taosArrayPush(iRslts, (void*)&trslt); } - indexMergeFinalResults(interResults, opera, result); - indexInterResultsDestroy(interResults); + indexMergeFinalResults(iRslts, opera, result); + indexInterResultsDestroy(iRslts); #endif return 0; @@ -258,13 +222,13 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { + SIndexMultiTermQuery* mtq = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); + if (mtq == NULL) { return NULL; } - p->opera = opera; - p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); - return p; + mtq->opera = opera; + mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + return mtq; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { @@ -280,25 +244,26 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { + SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); + if (tm == NULL) { return NULL; } - t->suid = suid; - t->operType = oper; - t->colType = colType; + tm->suid = suid; + tm->operType = oper; + tm->colType = colType; - t->colName = (char*)taosMemoryCalloc(1, nColName + 1); - memcpy(t->colName, colName, nColName); - t->nColName = nColName; + tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); + memcpy(tm->colName, colName, nColName); + tm->nColName = nColName; - t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); - memcpy(t->colVal, colVal, nColVal); - t->nColVal = nColVal; - return t; + tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); + memcpy(tm->colVal, colVal, nColVal); + tm->nColVal = nColVal; + + return tm; } void indexTermDestroy(SIndexTerm* p) { taosMemoryFree(p->colName); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index df3c0b6e7b..13768ce682 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,9 +34,129 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +/*comm func of compare, used in (LE/LT/GE/GT compare)*/ +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type); + +typedef enum { MATCH, CONTINUE, BREAK } TExeCond; +typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); + +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } + +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, + tCompareGreaterThan, tCompareGreaterEqual}; + +static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, + cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; + static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + if (cache == NULL) { + return 0; + } + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, ct->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + tSkipListDestroyIter(iter); + return 0; +} +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIter(mem->mem); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + } else if (cond == BREAK) { + break; + } + } + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, ct, tr, s, LT); +} +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, ct, tr, s, LE); +} +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, ct, tr, s, GT); +} +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, ct, tr, s, GE); +} +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { @@ -263,33 +383,7 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SI if (mem == NULL) { return 0; } - char* key = indexCacheTermGet(ct); - - SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - if (node != NULL) { - CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (qtype == QUERY_TERM) { - if (0 == strcmp(c->colVal, ct->colVal)) { - if (c->operaType == ADD_VALUE) { - INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) - // taosArrayPush(result, &c->uid); - *s = kTypeValue; - } else if (c->operaType == DEL_VALUE) { - INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) - } - } else { - break; - } - } else if (qtype == QUERY_PREFIX) { - } else if (qtype == QUERY_SUFFIX) { - } else if (qtype == QUERY_RANGE) { - } - } - } - tSkipListDestroyIter(iter); - return 0; + return cacheSearch[qtype](mem, ct, tr, s); } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 554b8092a2..5aed2bd6b0 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -57,6 +57,24 @@ static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version); static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version); static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); +/* + * search from tfile + */ +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); + +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); + +static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { + tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, + tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -183,59 +201,194 @@ void tfileReaderDestroy(TFileReader* reader) { writerCtxDestroy(reader->ctx, reader->remove); taosMemoryFree(reader); } +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + uint64_t offset; + if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + int64_t et = taosGetTimestampUs(); + int64_t cost = et - st; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, cost); + + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + cost = taosGetTimestampUs() - et; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + tem->colName, tem->colVal, cost); + } + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} + +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + + int32_t ret = 0; + for (int i = 0; i < taosArrayGetSize(offsets); i++) { + uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + if (ret != 0) { + indexError("failed to find target tablelist"); + return TSDB_CODE_TDB_FILE_CORRUPTED; + } + } + if (hasJson) { + taosMemoryFree(p); + } + return 0; +} +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} + +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + + FstSlice h = fstSliceCreate((uint8_t*)p, sz); + fstStreamBuilderSetRange(sb, &h, type); + fstSliceDestroy(&h); + + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc(reader, tem, tr, LT); +} +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc(reader, tem, tr, LE); +} +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc(reader, tem, tr, GT); +} +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc(reader, tem, tr, GE); +} +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + // uint64_t offset; + // if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + // int64_t et = taosGetTimestampUs(); + // int64_t cost = et - st; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + // tem->suid, tem->colName, tem->colVal, cost); + + // ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + // cost = taosGetTimestampUs() - et; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + // tem->colName, tem->colVal, cost); + //} + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); EIndexQueryType qtype = query->qType; - - // SArray* result = taosArrayInit(16, sizeof(uint64_t)); - int ret = -1; - // refactor to callback later - if (qtype == QUERY_TERM) { - uint64_t offset; - char* p = term->colVal; - uint64_t sz = term->nColVal; - if (hasJson) { - p = indexPackJsonData(term); - sz = strlen(p); - } - int64_t st = taosGetTimestampUs(); - FstSlice key = fstSliceCreate(p, sz); - if (fstGet(reader->fst, &key, &offset)) { - int64_t et = taosGetTimestampUs(); - int64_t cost = et - st; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", - term->suid, term->colName, term->colVal, cost); - - ret = tfileReaderLoadTableIds(reader, offset, tr->total); - cost = taosGetTimestampUs() - et; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", term->suid, - term->colName, term->colVal, cost); - } else { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, - term->colVal); - } - fstSliceDestroy(&key); - if (hasJson) { - taosMemoryFree(p); - } - } else if (qtype == QUERY_PREFIX) { - // handle later - // - } else if (qtype == QUERY_SUFFIX) { - // handle later - } else if (qtype == QUERY_REGEX) { - // handle later - } else if (qtype == QUERY_RANGE) { - // handle later + if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, + term->colVal); + return -1; + } else { + return tfSearch[qtype](reader, term, tr); } tfileReaderUnRef(reader); - - // taosArrayAddAll(tr->total, result); - // taosArrayDestroy(result); - - return ret; + return 0; } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 0162b754ee..d8ea6a8233 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,9 +483,9 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM}; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,9 +623,9 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -638,9 +638,9 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index f789d23136..e1e5004701 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f1fe68c55a..5e9e4e1d57 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -154,6 +154,7 @@ static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicCond } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); @@ -190,6 +191,7 @@ static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); + CLONE_NODE_FIELD(pWStartTs); return (SNode*)pDst; } @@ -268,11 +270,18 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pFuncs); - CLONE_NODE_FIELD(pFill); CLONE_NODE_FIELD(pTspk); return (SNode*)pDst; } +static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_FIELD(pWStartTs); + CLONE_NODE_FIELD(pValues); + return (SNode*)pDst; +} + static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); @@ -369,6 +378,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillCopy((const SFillLogicNode*)pNode, (SFillLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PARTITION: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index fa862df2b1..6c85d9dff5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -192,6 +192,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicExchange"; case QUERY_NODE_LOGIC_PLAN_WINDOW: return "LogicWindow"; + case QUERY_NODE_LOGIC_PLAN_FILL: + return "LogicFill"; case QUERY_NODE_LOGIC_PLAN_SORT: return "LogicSort"; case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -222,6 +224,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -564,6 +568,58 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkFillLogicPlanMode = "Mode"; +static const char* jkFillLogicPlanWStartTs = "WStartTs"; +static const char* jkFillLogicPlanValues = "Values"; +static const char* jkFillLogicPlanStartTime = "StartTime"; +static const char* jkFillLogicPlanEndTime = "EndTime"; + +static int32_t logicFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillLogicNode* pNode = (const SFillLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanValues, nodeToJson, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanEndTime, pNode->timeRange.ekey); + } + + return code; +} + +static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { + SFillLogicNode* pNode = (SFillLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanValues, &pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanEndTime, &pNode->timeRange.ekey); + } + + return code; +} + static const char* jkSortLogicPlanSortKeys = "SortKeys"; static int32_t logicSortNodeToJson(const void* pObj, SJson* pJson) { @@ -1382,7 +1438,6 @@ static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; -static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -1403,9 +1458,6 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanSlidingUnit, pNode->slidingUnit); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); - } return code; } @@ -1429,8 +1481,64 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkIntervalPhysiPlanSlidingUnit, &pNode->slidingUnit); } + + return code; +} + +static const char* jkFillPhysiPlanMode = "Mode"; +static const char* jkFillPhysiPlanWStartTs = "WStartTs"; +static const char* jkFillPhysiPlanValues = "Values"; +static const char* jkFillPhysiPlanTargets = "Targets"; +static const char* jkFillPhysiPlanStartTime = "StartTime"; +static const char* jkFillPhysiPlanEndTime = "EndTime"; + +static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillPhysiPlanWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillPhysiPlanValues, nodeToJson, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkFillPhysiPlanTargets, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanEndTime, pNode->timeRange.ekey); + } + + return code; +} + +static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { + SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanValues, &pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkFillPhysiPlanTargets, &pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanEndTime, &pNode->timeRange.ekey); } return code; @@ -2077,6 +2185,7 @@ static const char* jkFunctionName = "Name"; static const char* jkFunctionId = "Id"; static const char* jkFunctionType = "Type"; static const char* jkFunctionParameter = "Parameters"; +static const char* jkFunctionUdfBufSize = "UdfBufSize"; static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { const SFunctionNode* pNode = (const SFunctionNode*)pObj; @@ -2094,6 +2203,9 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkFunctionParameter, pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionUdfBufSize, pNode->udfBufSize); + } return code; } @@ -2114,6 +2226,9 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkFunctionParameter, &pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionUdfBufSize, &pNode->udfBufSize); + } return code; } @@ -2321,6 +2436,9 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) { static const char* jkFillMode = "Mode"; static const char* jkFillValues = "Values"; +static const char* jkFillWStartTs = "WStartTs"; +static const char* jkFillStartTime = "StartTime"; +static const char* jkFillEndTime = "EndTime"; static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { const SFillNode* pNode = (const SFillNode*)pObj; @@ -2329,6 +2447,15 @@ static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillEndTime, pNode->timeRange.ekey); + } return code; } @@ -2340,6 +2467,15 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillEndTime, &pNode->timeRange.ekey); + } return code; } @@ -2700,6 +2836,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicProjectNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: break; + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -2728,6 +2866,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return physiFillNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -2788,6 +2928,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_FILL: + return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: return jsonToLogicSortNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -2814,6 +2956,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return jsonToPhysiFillNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 88f5d914e4..f2c043b9ea 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -132,9 +132,14 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa case QUERY_NODE_NODE_LIST: res = walkExprs(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); break; - case QUERY_NODE_FILL: - res = walkExpr(((SFillNode*)pNode)->pValues, order, walker, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = walkExpr(pFill->pValues, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pFill->pWStartTs, order, walker, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = walkExpr(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; @@ -272,9 +277,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_NODE_LIST: res = rewriteExprs(((SNodeListNode*)pNode)->pNodeList, order, rewriter, pContext); break; - case QUERY_NODE_FILL: - res = rewriteExpr(&(((SFillNode*)pNode)->pValues), order, rewriter, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = rewriteExpr(&pFill->pValues, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&(pFill->pWStartTs), order, rewriter, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = rewriteExpr(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; @@ -333,6 +343,9 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_PARTITION_BY: nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesWalkExpr(((SIntervalWindowNode*)pSelect->pWindow)->pFill, walker, pContext); + } nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: nodesWalkExpr(pSelect->pHaving, walker, pContext); @@ -362,6 +375,9 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit case SQL_CLAUSE_PARTITION_BY: nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesRewriteExpr(&(((SIntervalWindowNode*)pSelect->pWindow)->pFill), rewriter, pContext); + } nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); @@ -496,14 +512,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)pNode; + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); - if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { - res = walkPhysiPlan((SNode*)pInterval->pFill, order, walker, pContext); - } break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d7dba10718..0bceb84f4a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -213,6 +213,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); + case QUERY_NODE_LOGIC_PLAN_FILL: + return makeNode(type, sizeof(SFillLogicNode)); case QUERY_NODE_LOGIC_PLAN_SORT: return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -243,6 +245,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -373,9 +377,12 @@ void nodesDestroyNode(SNodeptr pNode) { case QUERY_NODE_NODE_LIST: nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); break; - case QUERY_NODE_FILL: - nodesDestroyNode(((SFillNode*)pNode)->pValues); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + nodesDestroyNode(pFill->pValues); + nodesDestroyNode(pFill->pWStartTs); break; + } case QUERY_NODE_RAW_EXPR: nodesDestroyNode(((SRawExprNode*)pNode)->pNode); break; @@ -554,7 +561,6 @@ void nodesDestroyNode(SNodeptr pNode) { SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); - nodesDestroyNode(pLogicNode->pFill); nodesDestroyNode(pLogicNode->pTspk); break; } @@ -630,12 +636,9 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pPhyNode->pSortKeys); break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pPhyNode = (SIntervalPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pFill); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; @@ -894,7 +897,7 @@ void* nodesGetValueFromNode(SValueNode* pNode) { return NULL; } -int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value) { +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: pNode->datum.b = *(bool*)value; @@ -1192,7 +1195,7 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs) { +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { return TSDB_CODE_FAILED; } @@ -1203,7 +1206,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_OUT_OF_MEMORY; } *pFuncs = NULL; - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectFuncs, &cxt); + nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pFuncs); return cxt.errCode; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 1701976f8d..c021f65090 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -501,6 +501,12 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { CHECK_OUT_OF_MEM(fill); fill->mode = mode; fill->pValues = pValues; + fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == fill->pWStartTs) { + nodesDestroyNode(fill); + CHECK_OUT_OF_MEM(fill->pWStartTs); + } + strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstartts"); return (SNode*)fill; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index ed123d50bd..44d68a7886 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -26,193 +26,192 @@ typedef struct SKeyword { uint8_t len; // length } SKeyword; +// clang-format off // keywords in sql string static SKeyword keywordTable[] = { - {"ACCOUNT", TK_ACCOUNT}, - {"ACCOUNTS", TK_ACCOUNTS}, - {"ADD", TK_ADD}, - {"AGGREGATE", TK_AGGREGATE}, - {"ALL", TK_ALL}, - {"ALTER", TK_ALTER}, - {"ANALYZE", TK_ANALYZE}, - {"AND", TK_AND}, - {"APPS", TK_APPS}, - {"AS", TK_AS}, - {"ASC", TK_ASC}, - {"AT_ONCE", TK_AT_ONCE}, - {"BETWEEN", TK_BETWEEN}, - {"BINARY", TK_BINARY}, - {"BIGINT", TK_BIGINT}, - // {"BLOCKS", TK_BLOCKS}, - {"BNODE", TK_BNODE}, - {"BNODES", TK_BNODES}, - {"BOOL", TK_BOOL}, - {"BUFFER", TK_BUFFER}, - {"BUFSIZE", TK_BUFSIZE}, - {"BY", TK_BY}, - {"CACHE", TK_CACHE}, - {"CACHELAST", TK_CACHELAST}, - {"CAST", TK_CAST}, - {"CLUSTER", TK_CLUSTER}, - {"COLUMN", TK_COLUMN}, - {"COMMENT", TK_COMMENT}, - {"COMP", TK_COMP}, - {"COMPACT", TK_COMPACT}, - {"CONNS", TK_CONNS}, - {"CONNECTION", TK_CONNECTION}, - {"CONNECTIONS", TK_CONNECTIONS}, - {"COUNT", TK_COUNT}, - {"CREATE", TK_CREATE}, - {"DATABASE", TK_DATABASE}, - {"DATABASES", TK_DATABASES}, - {"DAYS", TK_DAYS}, - {"DBS", TK_DBS}, - {"DELAY", TK_DELAY}, - {"DESC", TK_DESC}, - {"DESCRIBE", TK_DESCRIBE}, - {"DISTINCT", TK_DISTINCT}, - {"DNODE", TK_DNODE}, - {"DNODES", TK_DNODES}, - {"DOUBLE", TK_DOUBLE}, - {"DROP", TK_DROP}, - {"EXISTS", TK_EXISTS}, - {"EXPLAIN", TK_EXPLAIN}, - {"FILE_FACTOR", TK_FILE_FACTOR}, - {"FILL", TK_FILL}, - {"FIRST", TK_FIRST}, - {"FLOAT", TK_FLOAT}, - {"FROM", TK_FROM}, - {"FSYNC", TK_FSYNC}, - {"FUNCTION", TK_FUNCTION}, - {"FUNCTIONS", TK_FUNCTIONS}, - {"GRANTS", TK_GRANTS}, - {"GROUP", TK_GROUP}, - {"HAVING", TK_HAVING}, - {"IF", TK_IF}, - {"IMPORT", TK_IMPORT}, - {"IN", TK_IN}, - {"INDEX", TK_INDEX}, - {"INDEXES", TK_INDEXES}, - {"INNER", TK_INNER}, - {"INT", TK_INT}, - {"INSERT", TK_INSERT}, - {"INTEGER", TK_INTEGER}, - {"INTERVAL", TK_INTERVAL}, - {"INTO", TK_INTO}, - {"IS", TK_IS}, - {"JOIN", TK_JOIN}, - {"JSON", TK_JSON}, - {"KEEP", TK_KEEP}, - {"KILL", TK_KILL}, - {"LAST", TK_LAST}, - {"LAST_ROW", TK_LAST_ROW}, - {"LICENCE", TK_LICENCE}, - {"LIKE", TK_LIKE}, - {"LIMIT", TK_LIMIT}, - {"LINEAR", TK_LINEAR}, - {"LOCAL", TK_LOCAL}, - {"MATCH", TK_MATCH}, - {"MAXROWS", TK_MAXROWS}, - {"MINROWS", TK_MINROWS}, - {"MINUS", TK_MINUS}, - {"MNODE", TK_MNODE}, - {"MNODES", TK_MNODES}, - {"MODIFY", TK_MODIFY}, - {"MODULES", TK_MODULES}, - {"NCHAR", TK_NCHAR}, - {"NMATCH", TK_NMATCH}, - {"NONE", TK_NONE}, - {"NOT", TK_NOT}, - {"NOW", TK_NOW}, - {"NULL", TK_NULL}, - {"NULLS", TK_NULLS}, - {"OFFSET", TK_OFFSET}, - {"ON", TK_ON}, - {"OR", TK_OR}, - {"ORDER", TK_ORDER}, - {"OUTPUTTYPE", TK_OUTPUTTYPE}, - {"PARTITION", TK_PARTITION}, - {"PASS", TK_PASS}, - {"PAGES", TK_PAGES}, - {"PAGESIZE", TK_PAGESIZE}, - {"PORT", TK_PORT}, - {"PPS", TK_PPS}, - {"PRECISION", TK_PRECISION}, - {"PRIVILEGE", TK_PRIVILEGE}, - {"PREV", TK_PREV}, - {"QNODE", TK_QNODE}, - {"QNODES", TK_QNODES}, - {"QTIME", TK_QTIME}, - {"QUERIES", TK_QUERIES}, - {"QUERY", TK_QUERY}, - // {"QUORUM", TK_QUORUM}, - {"RATIO", TK_RATIO}, - {"REPLICA", TK_REPLICA}, - {"RESET", TK_RESET}, - {"RETENTIONS", TK_RETENTIONS}, - {"ROLLUP", TK_ROLLUP}, - {"SCHEMA", TK_SCHEMA}, - {"SCORES", TK_SCORES}, - {"SELECT", TK_SELECT}, - {"SESSION", TK_SESSION}, - {"SET", TK_SET}, - {"SHOW", TK_SHOW}, + {"ACCOUNT", TK_ACCOUNT}, + {"ACCOUNTS", TK_ACCOUNTS}, + {"ADD", TK_ADD}, + {"AGGREGATE", TK_AGGREGATE}, + {"ALL", TK_ALL}, + {"ALTER", TK_ALTER}, + {"ANALYZE", TK_ANALYZE}, + {"AND", TK_AND}, + {"APPS", TK_APPS}, + {"AS", TK_AS}, + {"ASC", TK_ASC}, + {"AT_ONCE", TK_AT_ONCE}, + {"BETWEEN", TK_BETWEEN}, + {"BINARY", TK_BINARY}, + {"BIGINT", TK_BIGINT}, + {"BNODE", TK_BNODE}, + {"BNODES", TK_BNODES}, + {"BOOL", TK_BOOL}, + {"BUFFER", TK_BUFFER}, + {"BUFSIZE", TK_BUFSIZE}, + {"BY", TK_BY}, + {"CACHE", TK_CACHE}, + {"CACHELAST", TK_CACHELAST}, + {"CAST", TK_CAST}, + {"CLUSTER", TK_CLUSTER}, + {"COLUMN", TK_COLUMN}, + {"COMMENT", TK_COMMENT}, + {"COMP", TK_COMP}, + {"COMPACT", TK_COMPACT}, + {"CONNS", TK_CONNS}, + {"CONNECTION", TK_CONNECTION}, + {"CONNECTIONS", TK_CONNECTIONS}, + {"COUNT", TK_COUNT}, + {"CREATE", TK_CREATE}, + {"DATABASE", TK_DATABASE}, + {"DATABASES", TK_DATABASES}, + {"DAYS", TK_DAYS}, + {"DBS", TK_DBS}, + {"DELAY", TK_DELAY}, + {"DESC", TK_DESC}, + {"DESCRIBE", TK_DESCRIBE}, + {"DISTINCT", TK_DISTINCT}, + {"DNODE", TK_DNODE}, + {"DNODES", TK_DNODES}, + {"DOUBLE", TK_DOUBLE}, + {"DROP", TK_DROP}, + {"EXISTS", TK_EXISTS}, + {"EXPLAIN", TK_EXPLAIN}, + {"FILE_FACTOR", TK_FILE_FACTOR}, + {"FILL", TK_FILL}, + {"FIRST", TK_FIRST}, + {"FLOAT", TK_FLOAT}, + {"FROM", TK_FROM}, + {"FSYNC", TK_FSYNC}, + {"FUNCTION", TK_FUNCTION}, + {"FUNCTIONS", TK_FUNCTIONS}, + {"GRANTS", TK_GRANTS}, + {"GROUP", TK_GROUP}, + {"HAVING", TK_HAVING}, + {"IF", TK_IF}, + {"IMPORT", TK_IMPORT}, + {"IN", TK_IN}, + {"INDEX", TK_INDEX}, + {"INDEXES", TK_INDEXES}, + {"INNER", TK_INNER}, + {"INT", TK_INT}, + {"INSERT", TK_INSERT}, + {"INTEGER", TK_INTEGER}, + {"INTERVAL", TK_INTERVAL}, + {"INTO", TK_INTO}, + {"IS", TK_IS}, + {"JOIN", TK_JOIN}, + {"JSON", TK_JSON}, + {"KEEP", TK_KEEP}, + {"KILL", TK_KILL}, + {"LAST", TK_LAST}, + {"LAST_ROW", TK_LAST_ROW}, + {"LICENCE", TK_LICENCE}, + {"LIKE", TK_LIKE}, + {"LIMIT", TK_LIMIT}, + {"LINEAR", TK_LINEAR}, + {"LOCAL", TK_LOCAL}, + {"MATCH", TK_MATCH}, + {"MAXROWS", TK_MAXROWS}, + {"MINROWS", TK_MINROWS}, + {"MINUS", TK_MINUS}, + {"MNODE", TK_MNODE}, + {"MNODES", TK_MNODES}, + {"MODIFY", TK_MODIFY}, + {"MODULES", TK_MODULES}, + {"NCHAR", TK_NCHAR}, + {"NEXT", TK_NEXT}, + {"NMATCH", TK_NMATCH}, + {"NONE", TK_NONE}, + {"NOT", TK_NOT}, + {"NOW", TK_NOW}, + {"NULL", TK_NULL}, + {"NULLS", TK_NULLS}, + {"OFFSET", TK_OFFSET}, + {"ON", TK_ON}, + {"OR", TK_OR}, + {"ORDER", TK_ORDER}, + {"OUTPUTTYPE", TK_OUTPUTTYPE}, + {"PARTITION", TK_PARTITION}, + {"PASS", TK_PASS}, + {"PAGES", TK_PAGES}, + {"PAGESIZE", TK_PAGESIZE}, + {"PORT", TK_PORT}, + {"PPS", TK_PPS}, + {"PRECISION", TK_PRECISION}, + {"PRIVILEGE", TK_PRIVILEGE}, + {"PREV", TK_PREV}, + {"QNODE", TK_QNODE}, + {"QNODES", TK_QNODES}, + {"QTIME", TK_QTIME}, + {"QUERIES", TK_QUERIES}, + {"QUERY", TK_QUERY}, + {"RATIO", TK_RATIO}, + {"REPLICA", TK_REPLICA}, + {"RESET", TK_RESET}, + {"RETENTIONS", TK_RETENTIONS}, + {"ROLLUP", TK_ROLLUP}, + {"SCHEMA", TK_SCHEMA}, + {"SCORES", TK_SCORES}, + {"SELECT", TK_SELECT}, + {"SESSION", TK_SESSION}, + {"SET", TK_SET}, + {"SHOW", TK_SHOW}, {"SINGLE_STABLE", TK_SINGLE_STABLE}, - {"SLIDING", TK_SLIDING}, - {"SLIMIT", TK_SLIMIT}, - {"SMA", TK_SMA}, - {"SMALLINT", TK_SMALLINT}, - {"SNODE", TK_SNODE}, - {"SNODES", TK_SNODES}, - {"SOFFSET", TK_SOFFSET}, - {"STABLE", TK_STABLE}, - {"STABLES", TK_STABLES}, - {"STATE", TK_STATE}, - {"STATE_WINDOW", TK_STATE_WINDOW}, - {"STORAGE", TK_STORAGE}, - {"STREAM", TK_STREAM}, - {"STREAMS", TK_STREAMS}, - // {"STREAM_MODE", TK_STREAM_MODE}, - {"STRICT", TK_STRICT}, - {"SYNCDB", TK_SYNCDB}, - {"TABLE", TK_TABLE}, - {"TABLES", TK_TABLES}, - {"TAG", TK_TAG}, - {"TAGS", TK_TAGS}, - {"TBNAME", TK_TBNAME}, - {"TIMESTAMP", TK_TIMESTAMP}, - {"TIMEZONE", TK_TIMEZONE}, - {"TINYINT", TK_TINYINT}, - {"TODAY", TK_TODAY}, - {"TOPIC", TK_TOPIC}, - {"TOPICS", TK_TOPICS}, - {"TRIGGER", TK_TRIGGER}, - {"TSERIES", TK_TSERIES}, - {"TTL", TK_TTL}, - {"UNION", TK_UNION}, - {"UNSIGNED", TK_UNSIGNED}, - {"USE", TK_USE}, - {"USER", TK_USER}, - {"USERS", TK_USERS}, - {"USING", TK_USING}, - {"VALUE", TK_VALUE}, - {"VALUES", TK_VALUES}, - {"VARCHAR", TK_VARCHAR}, - {"VARIABLES", TK_VARIABLES}, - {"VERBOSE", TK_VERBOSE}, - {"VGROUPS", TK_VGROUPS}, - {"VNODES", TK_VNODES}, - {"WAL", TK_WAL}, - {"WATERMARK", TK_WATERMARK}, - {"WHERE", TK_WHERE}, - {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, - {"WITH", TK_WITH}, - {"_QENDTS", TK_QENDTS}, - {"_QSTARTTS", TK_QSTARTTS}, - {"_ROWTS", TK_ROWTS}, - {"_WDURATION", TK_WDURATION}, - {"_WENDTS", TK_WENDTS}, - {"_WSTARTTS", TK_WSTARTTS}, + {"SLIDING", TK_SLIDING}, + {"SLIMIT", TK_SLIMIT}, + {"SMA", TK_SMA}, + {"SMALLINT", TK_SMALLINT}, + {"SNODE", TK_SNODE}, + {"SNODES", TK_SNODES}, + {"SOFFSET", TK_SOFFSET}, + {"STABLE", TK_STABLE}, + {"STABLES", TK_STABLES}, + {"STATE", TK_STATE}, + {"STATE_WINDOW", TK_STATE_WINDOW}, + {"STORAGE", TK_STORAGE}, + {"STREAM", TK_STREAM}, + {"STREAMS", TK_STREAMS}, + {"STRICT", TK_STRICT}, + {"SYNCDB", TK_SYNCDB}, + {"TABLE", TK_TABLE}, + {"TABLES", TK_TABLES}, + {"TAG", TK_TAG}, + {"TAGS", TK_TAGS}, + {"TBNAME", TK_TBNAME}, + {"TIMESTAMP", TK_TIMESTAMP}, + {"TIMEZONE", TK_TIMEZONE}, + {"TINYINT", TK_TINYINT}, + {"TODAY", TK_TODAY}, + {"TOPIC", TK_TOPIC}, + {"TOPICS", TK_TOPICS}, + {"TRIGGER", TK_TRIGGER}, + {"TSERIES", TK_TSERIES}, + {"TTL", TK_TTL}, + {"UNION", TK_UNION}, + {"UNSIGNED", TK_UNSIGNED}, + {"USE", TK_USE}, + {"USER", TK_USER}, + {"USERS", TK_USERS}, + {"USING", TK_USING}, + {"VALUE", TK_VALUE}, + {"VALUES", TK_VALUES}, + {"VARCHAR", TK_VARCHAR}, + {"VARIABLES", TK_VARIABLES}, + {"VERBOSE", TK_VERBOSE}, + {"VGROUPS", TK_VGROUPS}, + {"VNODES", TK_VNODES}, + {"WAL", TK_WAL}, + {"WATERMARK", TK_WATERMARK}, + {"WHERE", TK_WHERE}, + {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, + {"WITH", TK_WITH}, + {"_QENDTS", TK_QENDTS}, + {"_QSTARTTS", TK_QSTARTTS}, + {"_ROWTS", TK_ROWTS}, + {"_WDURATION", TK_WDURATION}, + {"_WENDTS", TK_WENDTS}, + {"_WSTARTTS", TK_WSTARTTS}, // {"ID", TK_ID}, // {"STRING", TK_STRING}, // {"EQ", TK_EQ}, @@ -279,6 +278,7 @@ static SKeyword keywordTable[] = { // {"PARTITIONS", TK_PARTITIONS}, // {"MODE", TK_MODE}, }; +// clang-format on static const char isIdChar[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2d08342214..30e55420d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -17,6 +17,7 @@ #include "catalog.h" #include "cmdnodes.h" +#include "filter.h" #include "functionMgt.h" #include "parUtil.h" #include "scalar.h" @@ -255,6 +256,26 @@ static void destroyTranslateContext(STranslateContext* pCxt) { taosHashCleanup(pCxt->pTables); } +static bool isAliasColumn(const SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0])); +} + +static bool isAggFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isSelectFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isTimelineFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { int cmp = 0; if ('\0' != pCol->dbName[0]) { @@ -468,19 +489,19 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); *(bool*)&pVal->typeData = pVal->datum.b; break; - case TSDB_DATA_TYPE_TINYINT:{ + case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_SMALLINT:{ + case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_INT:{ + case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; @@ -489,43 +510,43 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); - *(int64_t*)&pVal->typeData = pVal->datum.i; + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_UTINYINT:{ + case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint8_t*)&pVal->typeData = pVal->datum.u; + *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_USMALLINT:{ + case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint16_t*)&pVal->typeData = pVal->datum.u; + *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_UINT:{ + case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint32_t*)&pVal->typeData = pVal->datum.u; + *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint64_t*)&pVal->typeData = pVal->datum.u; + *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_FLOAT:{ + case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); - *(float*)&pVal->typeData = pVal->datum.d; + *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); - *(double*)&pVal->typeData = pVal->datum.d; + *(double*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_VARCHAR: @@ -543,7 +564,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } - *(int64_t*)&pVal->typeData = pVal->datum.i; + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_NCHAR: @@ -616,13 +637,45 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveAggFunction(SNode* pNode, void* pContext) { - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + if (isAggFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } return DEAL_RES_CONTINUE; } +static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { + *pOutput = pTable; + return TSDB_CODE_SUCCESS; + } + } + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); +} + +static bool isCountStar(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); +} + +// count(*) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { + SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } + return code; +} + static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, @@ -630,10 +683,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) .pErrBuf = pCxt->msgBuf.buf, .errBufLen = pCxt->msgBuf.len}; pCxt->errCode = fmGetFuncInfo(¶m, pFunc); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - return DEAL_RES_ERROR; - } - if (fmIsAggFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) { if (beforeHaving(pCxt->currClause)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); } @@ -642,11 +692,14 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (haveAggFunc) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } + pCxt->pCurrStmt->hasAggFuncs = true; pCxt->pCurrStmt->isTimeOrderQuery = false; + if (isCountStar(pFunc)) { + pCxt->errCode = rewriteCountStar(pCxt, pFunc); + } } - - return DEAL_RES_CONTINUE; + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { @@ -690,12 +743,6 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } -static bool isAliasColumn(SColumnNode* pCol) { return ('\0' == pCol->tableAlias[0]); } - -static bool isDistinctOrderBy(STranslateContext* pCxt) { - return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); -} - static SNodeList* getGroupByList(STranslateContext* pCxt) { if (isDistinctOrderBy(pCxt)) { return pCxt->pCurrStmt->pProjectionList; @@ -717,31 +764,71 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } -static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { - STranslateContext* pCxt = (STranslateContext*)pContext; - if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { +typedef struct SCheckExprForGroupByCxt { + STranslateContext* pTranslateCxt; + int32_t selectFuncNum; + bool hasSelectValFunc; +} SCheckExprForGroupByCxt; + +static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSelectValFunc, SNode** pNode) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pFunc->functionName, "_select_value"); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + translateFunction(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + if (NULL != pHasSelectValFunc) { + *pHasSelectValFunc = true; + } + } else { + nodesDestroyNode(pFunc); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; +} + +static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { + SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; + if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - !isDistinctOrderBy(pCxt)) { + pCxt->selectFuncNum += isSelectFunc(*pNode) ? 1 : 0; + if (pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } + if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; - FOREACH(pGroupNode, getGroupByList(pCxt)) { - if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { + FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(pNode) || - (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - isDistinctOrderBy(pCxt))) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (pCxt->selectFuncNum > 1) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } else { + return rewriteColToSelectValFunc(pCxt->pTranslateCxt, &pCxt->hasSelectValFunc, pNode); + } + } + if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } return DEAL_RES_CONTINUE; } -static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); +static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } return pCxt->errCode; } @@ -749,7 +836,29 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } + return pCxt->errCode; +} + +static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { + if (isAggFunc(*pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteColsToSelectValFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteColsToSelectValFuncImpl, pCxt); + } return pCxt->errCode; } @@ -757,11 +866,13 @@ typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; bool existCol; + int32_t selectFuncNum; } CheckAggColCoexistCxt; static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->selectFuncNum += isSelectFunc(pNode) ? 1 : 0; + if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } @@ -780,7 +891,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (!pSelect->isDistinct) { nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if ((cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { + if (1 == cxt.selectFuncNum) { + return rewriteColsToSelectValFunc(pCxt, pSelect); + } else if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } return TSDB_CODE_SUCCESS; @@ -989,19 +1102,6 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { return (SNode*)pFunc; } -static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { - SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - size_t nums = taosArrayGetSize(pTables); - for (size_t i = 0; i < nums; ++i) { - STableNode* pTable = taosArrayGetP(pTables, i); - if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { - *pOutput = pTable; - return TSDB_CODE_SUCCESS; - } - } - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); -} - static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) { STableNode* pTable = NULL; int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); @@ -1227,7 +1327,7 @@ static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, pSelect->pHaving); if (TSDB_CODE_SUCCESS == code) { - code = checkExprForGroupBy(pCxt, pSelect->pHaving); + code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; } @@ -1244,6 +1344,113 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static EDealRes isPrimaryKeyCondImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool isPrimaryKeyCond(SNode* pNode) { + bool isPrimaryKeyCond = false; + nodesWalkExpr(pNode, isPrimaryKeyCondImpl, &isPrimaryKeyCond); + return isPrimaryKeyCond; +} + +static int32_t getTimeRangeFromLogicCond(STranslateContext* pCxt, SLogicConditionNode* pLogicCond, + STimeWindow* pTimeRange) { + SNodeList* pPrimaryKeyConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (isPrimaryKeyCond(pCond)) { + if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pPrimaryKeyConds, pCond)) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + } + + if (NULL == pPrimaryKeyConds) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + SLogicConditionNode* pPrimaryKeyLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pPrimaryKeyLogicCond) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + pPrimaryKeyLogicCond->condType = LOGIC_COND_TYPE_AND; + pPrimaryKeyLogicCond->pParameterList = pPrimaryKeyConds; + bool isStrict = false; + int32_t code = filterGetTimeRange((SNode*)pPrimaryKeyLogicCond, pTimeRange, &isStrict); + nodesClearList(pPrimaryKeyConds); + pPrimaryKeyLogicCond->pParameterList = NULL; + nodesDestroyNode(pPrimaryKeyLogicCond); + return code; +} + +static int32_t getTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { + if (NULL == pWhere) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pWhere)->condType) { + return getTimeRangeFromLogicCond(pCxt, (SLogicConditionNode*)pWhere, pTimeRange); + } + + if (isPrimaryKeyCond(pWhere)) { + bool isStrict = false; + return filterGetTimeRange(pWhere, pTimeRange, &isStrict); + } else { + *pTimeRange = TSWINDOW_INITIALIZER; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t checkFill(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + SFillNode* pFill = (SFillNode*)pInterval->pFill; + if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || + TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); + int64_t intervalRange = 0; + SValueNode* pInter = (SValueNode*)pInterval->pInterval; + if (TIME_IS_VAR_DURATION(pInter->unit)) { + int64_t f = 1; + if (pInter->unit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pInter->unit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + intervalRange = pInter->datum.i * f; + } else { + intervalRange = pInter->datum.i; + } + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFill(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { + if (NULL == pInterval->pFill) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = getTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange)); + if (TSDB_CODE_SUCCESS == code) { + code = checkFill(pCxt, pInterval); + } + return code; +} + static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); switch (unit) { @@ -1266,7 +1473,7 @@ static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni return -1; } -static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { +static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; @@ -1308,7 +1515,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* } } - return TSDB_CODE_SUCCESS; + return translateFill(pCxt, pWhere, pInterval); } static EDealRes checkStateExpr(SNode* pNode, void* pContext) { @@ -1345,28 +1552,28 @@ static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* p return TSDB_CODE_SUCCESS; } -static int32_t checkWindow(STranslateContext* pCxt, SNode* pWindow) { - switch (nodeType(pWindow)) { +static int32_t checkWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return checkStateWindow(pCxt, (SStateWindowNode*)pWindow); + return checkStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); case QUERY_NODE_SESSION_WINDOW: - return checkSessionWindow(pCxt, (SSessionWindowNode*)pWindow); + return checkSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: - return checkIntervalWindow(pCxt, (SIntervalWindowNode*)pWindow); + return checkIntervalWindow(pCxt, pSelect->pWhere, (SIntervalWindowNode*)pSelect->pWindow); default: break; } return TSDB_CODE_SUCCESS; } -static int32_t translateWindow(STranslateContext* pCxt, SNode* pWindow) { - if (NULL == pWindow) { +static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_WINDOW; - int32_t code = translateExpr(pCxt, pWindow); + int32_t code = translateExpr(pCxt, pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { - code = checkWindow(pCxt, pWindow); + code = checkWindow(pCxt, pSelect); } return code; } @@ -1403,25 +1610,6 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -static bool isCountStar(SFunctionNode* pFunc) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); -} - -// count(*) is rewritten as count(ts) for scannning optimization -static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { - SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } - return code; -} - static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { @@ -1445,33 +1633,22 @@ static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) return code; } -static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - SNode* pPrimaryKey = NULL; - int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); - } - return code; -} - -EDealRes rewriteFuncForSelectImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - STranslateContext* pCxt = pContext; - SFunctionNode* pFunc = (SFunctionNode*)pNode; - if (isCountStar(pFunc)) { - pCxt->errCode = rewriteCountStar(pCxt, pFunc); - } else if (fmIsTimelineFunc(pFunc->funcId)) { - pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); - } - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - return DEAL_RES_ERROR; +static EDealRes rewriteTimelineFuncImpl(SNode* pNode, void* pContext) { + STranslateContext* pCxt = pContext; + if (isTimelineFunc(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + SNode* pPrimaryKey = NULL; + pCxt->errCode = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; } -static int32_t rewriteFuncForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteFuncForSelectImpl, pCxt); +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); return pCxt->errCode; } @@ -1485,7 +1662,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translatePartitionBy(pCxt, pSelect->pPartitionByList); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindow(pCxt, pSelect->pWindow); + code = translateWindow(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateGroupBy(pCxt, pSelect); @@ -1506,7 +1683,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = checkLimit(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteFuncForSelect(pCxt, pSelect); + code = rewriteTimelineFunc(pCxt, pSelect); } return code; } @@ -1770,9 +1947,6 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - if (!TIME_IS_VAR_DURATION(pVal->unit)) { - pVal->datum.i = convertTimeFromPrecisionToUnit(pVal->datum.i, pVal->node.resType.precision, pVal->unit); - } } } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 2bf4071d33..e839536218 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -128,6 +128,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid topic query"; case TSDB_CODE_PAR_INVALID_DROP_STABLE: return "Cannot drop super table in batch"; + case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE: + return "start(end) time of query range required or time range too large"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -140,7 +142,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) { va_start(vArgList, errCode); vsnprintf(pBuf->buf, pBuf->len, getSyntaxErrFormat(errCode), vArgList); va_end(vArgList); - terrno = errCode; return errCode; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index ab02ff9558..2d9a907d9b 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -100,6 +100,17 @@ void generateInformationSchema(MockCatalogService* mcs) { } } +/* + * Table:t1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * c3 | column | BIGINT | 8 | + * c4 | column | DOUBLE | 8 | + * c5 | column | DOUBLE | 8 | + */ void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) @@ -113,6 +124,17 @@ void generateTestT1(MockCatalogService* mcs) { builder.done(); } +/* + * Super Table: st1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * tag1 | tag | INT | 4 | + * tag2 | tag | VARCHAR | 20 | + * Child Table: st1s1, st1s2 + */ void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 2c85bfdf2e..f86cecb9e3 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -103,7 +103,6 @@ class MockCatalogServiceImpl { const char* tname = tNameGetTableName(pTableName); int32_t code = copyTableSchemaMeta(db, tname, &table); if (TSDB_CODE_SUCCESS != code) { - std::cout << "db : " << db << ", table :" << tname << std::endl; return code; } *pTableMeta = table.release(); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 4c3a4e8ab9..5a385ba25e 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -24,170 +24,211 @@ class ParserSelectTest : public ParserTestBase {}; TEST_F(ParserSelectTest, basic) { useDb("root", "test"); - run("select * from t1"); + run("SELECT * FROM t1"); - run("select * from test.t1"); + run("SELECT * FROM test.t1"); - run("select ts, c1 from t1"); + run("SELECT ts, c1 FROM t1"); - run("select ts, t.c1 from (select * from t1) t"); + run("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); - run("select * from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1"); + run("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); } TEST_F(ParserSelectTest, constant) { useDb("root", "test"); - run("select 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s from t1"); + run("SELECT 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s FROM t1"); - run("select 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " - "timestamp '2022-02-09 17:30:20', true, false, 15s from t1"); + run("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " + "timestamp '2022-02-09 17:30:20', true, false, 15s FROM t1"); - run("select 123 + 45 from t1 where 2 - 1"); + run("SELECT 123 + 45 FROM t1 WHERE 2 - 1"); } TEST_F(ParserSelectTest, expression) { useDb("root", "test"); - run("select ts + 10s, c1 + 10, concat(c2, 'abc') from t1"); + run("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); - run("select ts > 0, c1 < 20 and c2 = 'qaz' from t1"); + run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1"); - run("select ts > 0, c1 between 10 and 20 and c2 = 'qaz' from t1"); + run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1"); } TEST_F(ParserSelectTest, condition) { useDb("root", "test"); - run("select c1 from t1 where ts in (true, false)"); + run("SELECT c1 FROM t1 WHERE ts in (true, false)"); - run("select * from t1 where c1 > 10 and c1 is not null"); + run("SELECT * FROM t1 WHERE c1 > 10 and c1 is not null"); } TEST_F(ParserSelectTest, pseudoColumn) { useDb("root", "test"); - run("select _wstartts, _wendts, count(*) from t1 interval(10s)"); + run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(ParserSelectTest, multiResFunc) { useDb("root", "test"); - run("select last(*), first(*), last_row(*) from t1"); + run("SELECT LAST(*), FIRST(*), LAST_ROW(*) FROM t1"); - run("select last(c1, c2), first(t1.*), last_row(c3) from t1"); + run("SELECT LAST(c1, c2), FIRST(t1.*), LAST_ROW(c3) FROM t1"); - run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("SELECT LAST(t2.*), FIRST(t1.c1, t2.*), LAST_ROW(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); } TEST_F(ParserSelectTest, timelineFunc) { useDb("root", "test"); - run("select last(*), first(*) from t1"); + run("SELECT LAST(*), FIRST(*) FROM t1"); - run("select last(*), first(*) from t1 group by c1"); + run("SELECT FIRST(ts), FIRST(c1), FIRST(c2), FIRST(c3) FROM t1"); - run("select last(*), first(*) from t1 interval(10s)"); + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); - run("select diff(c1) from t1"); + run("SELECT LAST(*), FIRST(*) FROM t1 INTERVAL(10s)"); + + run("SELECT diff(c1) FROM t1"); +} + +TEST_F(ParserSelectTest, selectFunc) { + useDb("root", "test"); + + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); + // select function along with the columns of select row, and with window clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); } TEST_F(ParserSelectTest, clause) { useDb("root", "test"); - // group by clause - run("select count(*) cnt from t1 where c1 > 0"); + // GROUP BY clause + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0"); - run("select count(*), c2 cnt from t1 where c1 > 0 group by c2"); + run("SELECT COUNT(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); - run("select count(*) cnt from t1 where c1 > 0 group by c2 having count(c1) > 10"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 having COUNT(c1) > 10"); - run("select count(*), c1, c2 + 10, c1 + c2 cnt from t1 where c1 > 0 group by c2, c1"); + run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); - run("select count(*), c1 + 10, c2 cnt from t1 where c1 > 0 group by c1 + 10, c2"); + run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); // order by clause - run("select count(*) cnt from t1 where c1 > 0 group by c2 order by cnt"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by cnt"); - run("select count(*) cnt from t1 where c1 > 0 group by c2 order by 1"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by 1"); // distinct clause - // run("select distinct c1, c2 from t1 where c1 > 0 order by c1"); + // run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by c1"); - // run("select distinct c1 + 10, c2 from t1 where c1 > 0 order by c1 + 10, c2"); + // run("SELECT distinct c1 + 10, c2 FROM t1 WHERE c1 > 0 order by c1 + 10, c2"); - // run("select distinct c1 + 10 cc1, c2 cc2 from t1 where c1 > 0 order by cc1, c2"); + // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 order by cc1, c2"); - // run("select distinct count(c2) from t1 where c1 > 0 group by c1 order by count(c2)"); + // run("SELECT distinct COUNT(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 order by COUNT(c2)"); } -TEST_F(ParserSelectTest, window) { +// INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] +// fill_mod_and_val = { NONE | PREV | NULL | LINEAR | NEXT | value_mod } +// value_mod = VALUE , val ... +TEST_F(ParserSelectTest, interval) { + useDb("root", "test"); + // INTERVAL(interval_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); + // INTERVAL(interval_val, interval_offset) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s)"); + // INTERVAL(interval_val, interval_offset) SLIDING (sliding_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s) SLIDING(7s)"); + // INTERVAL(interval_val) FILL(NONE) + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(NONE)"); +} + +TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("select count(*) from t1 interval(10s)"); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, + PARSER_STAGE_TRANSLATE); } TEST_F(ParserSelectTest, semanticError) { useDb("root", "test"); // TSDB_CODE_PAR_INVALID_COLUMN - run("select c1, cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT c1, cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); - run("select t1.c1, t1.cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT t1.c1, t1.cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_TABLE_NOT_EXIST - run("select * from t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT * FROM t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); - run("select * from test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT * FROM test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); - run("select t2.c1 from t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT t2.c1 FROM t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_AMBIGUOUS_COLUMN - run("select c2 from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_VALUE_TYPE - run("select timestamp '2010' from t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION - run("select c2 from t1 tt1 join t1 tt2 on count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, + run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 where count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 WHERE COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 group by count(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 GROUP BY COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT - run("select c2 from t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION - run("select count(*) cnt from t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*) cnt FROM t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*) cnt from t1 group by c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*), c1 cnt from t1 group by c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*), c1 cnt FROM t1 GROUP BY c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*) cnt from t1 group by c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SINGLE_GROUP - run("select count(*), c1 from t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*), c1 FROM t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); - run("select count(*) from t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*) FROM t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); - run("select c1 from t1 order by count(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT c1 FROM t1 order by COUNT(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION - run("select distinct c1, c2 from t1 where c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select distinct c1 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select distinct c2 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c2 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index e6534f710d..4640ed99bd 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "planner.h" +#include "taoserror.h" #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 @@ -33,6 +34,8 @@ extern "C" { #define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); + int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 00f2808294..0bc700696b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -274,7 +274,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } if (TSDB_CODE_SUCCESS == code) { - code = nodesCollectFuncs(pSelect, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); } pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); @@ -440,7 +440,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAgg->pAggFuncs); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); } // rewrite the expression in subsequent clauses @@ -474,7 +474,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { - int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs); + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (pCxt->pPlanCxt->streamQuery) { pWindow->triggerType = pCxt->pPlanCxt->triggerType; @@ -559,14 +559,6 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL != pInterval->pFill) { - pWindow->pFill = nodesCloneNode(pInterval->pFill); - if (NULL == pWindow->pFill) { - nodesDestroyNode(pWindow); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } @@ -589,6 +581,38 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } +static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || + NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { + return TSDB_CODE_SUCCESS; + } + + SFillNode* pFillNode = (SFillNode*)(((SIntervalWindowNode*)pSelect->pWindow)->pFill); + + SFillLogicNode* pFill = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); + + pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; + pFill->pValues = nodesCloneNode(pFillNode->pValues); + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if ((NULL != pFillNode->pValues && NULL == pFill->pValues) || NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pOrderByList) { return TSDB_CODE_SUCCESS; @@ -753,6 +777,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createFillLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 47dcd9ce86..67609355d7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -554,22 +554,19 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { if (LOGIC_COND_TYPE_AND != pOnCond->condType) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } SNode* pCond = NULL; FOREACH(pCond, pOnCond->pParameterList) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } } return TSDB_CODE_SUCCESS; @@ -577,8 +574,7 @@ static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 901afce718..05b48d2d6a 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -17,8 +17,8 @@ #include "catalog.h" #include "functionMgt.h" -#include "tglobal.h" #include "systable.h" +#include "tglobal.h" typedef struct SSlotIdInfo { int16_t slotId; @@ -880,12 +880,6 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil pInterval->intervalUnit = pWindowLogicNode->intervalUnit; pInterval->slidingUnit = pWindowLogicNode->slidingUnit; - pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); - if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { - nodesDestroyNode(pInterval); - return TSDB_CODE_OUT_OF_MEMORY; - } - return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); } @@ -1035,6 +1029,46 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi return code; } +static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, + SPhysiNode** pPhyNode) { + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode, + QUERY_NODE_PHYSICAL_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->node.pTargets, &pFill->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pFill->pTargets, pFill->node.pOutputDataBlockDesc); + } + + if (TSDB_CODE_SUCCESS == code) { + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if (NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pFillNode->pValues) { + pFill->pValues = nodesCloneNode(pFillNode->pValues); + if (NULL == pFill->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { @@ -1054,6 +1088,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_PARTITION: return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_FILL: + return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c new file mode 100644 index 0000000000..36625b28fb --- /dev/null +++ b/source/libs/planner/src/planUtil.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +static char* getUsageErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PLAN_EXPECTED_TS_EQUAL: + return "l.ts = r.ts is expected in join expression"; + case TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN: + return "not support cross join"; + default: + break; + } + return "Unknown error"; +} + +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...) { + va_list vArgList; + va_start(vArgList, errCode); + vsnprintf(pBuf, len, getUsageErrFormat(errCode), vArgList); + va_end(vArgList); + return errCode; +} diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 05a009d79d..639af2875e 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -1,7 +1,7 @@ /* * Copyright (c) 2019 TAOS Data, Inc. * - * This program is free software: you can use, redistribute, and/or modify + * This program is free software: you can use, redistribute, AND/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. * @@ -20,30 +20,32 @@ using namespace std; class PlanBasicTest : public PlannerTestBase {}; -TEST_F(PlanBasicTest, select) { +TEST_F(PlanBasicTest, selectClause) { useDb("root", "test"); - run("select * from t1"); - run("select 1 from t1"); - run("select * from st1"); - run("select 1 from st1"); + run("SELECT * FROM t1"); + run("SELECT 1 FROM t1"); + run("SELECT * FROM st1"); + run("SELECT 1 FROM st1"); } -TEST_F(PlanBasicTest, where) { +TEST_F(PlanBasicTest, whereClause) { useDb("root", "test"); - run("select * from t1 where c1 > 10"); + run("SELECT * FROM t1 WHERE c1 > 10"); + + run("SELECT * FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59'"); } -TEST_F(PlanBasicTest, join) { +TEST_F(PlanBasicTest, joinClause) { useDb("root", "test"); - run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); } TEST_F(PlanBasicTest, func) { useDb("root", "test"); - run("select diff(c1) from t1"); + run("SELECT DIFF(c1) FROM t1"); } diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 05ffe41fe7..9ca1001f4c 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -23,30 +23,45 @@ class PlanGroupByTest : public PlannerTestBase {}; TEST_F(PlanGroupByTest, basic) { useDb("root", "test"); - run("select count(*) from t1"); + run("SELECT COUNT(*) FROM t1"); - run("select c1, max(c3), min(c3), count(*) from t1 group by c1"); + run("SELECT c1, MAX(c3), MIN(c3), COUNT(*) FROM t1 GROUP BY c1"); - run("select c1 + c3, c1 + count(*) from t1 where c2 = 'abc' group by c1, c3"); + run("SELECT c1 + c3, c1 + COUNT(*) FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3"); - run("select c1 + c3, sum(c4 * c5) from t1 where concat(c2, 'wwww') = 'abcwww' group by c1 + c3"); + run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); - run("select sum(ceil(c1)) from t1 group by ceil(c1)"); + run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); } TEST_F(PlanGroupByTest, withOrderBy) { useDb("root", "test"); - // order by aggfunc - run("select count(*), sum(c1) from t1 order by sum(c1)"); - // order by alias of aggfunc - // run("select count(*), sum(c1) a from t1 order by a"); + // ORDER BY aggfunc + run("SELECT COUNT(*), SUM(c1) FROM t1 ORDER BY SUM(c1)"); + // ORDER BY alias of aggfunc + // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); } TEST_F(PlanGroupByTest, aggFunc) { useDb("root", "test"); - run("select last(*), first(*) from t1"); + run("SELECT LAST(*), FIRST(*) FROM t1"); - run("select last(*), first(*) from t1 group by c1"); + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); +} + +TEST_F(PlanGroupByTest, selectFunc) { + useDb("root", "test"); + + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index 6cc67a3446..c9bae46ca9 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -23,19 +23,31 @@ class PlanIntervalTest : public PlannerTestBase {}; TEST_F(PlanIntervalTest, basic) { useDb("root", "test"); - run("select count(*) from t1 interval(10s)"); + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(PlanIntervalTest, pseudoCol) { useDb("root", "test"); - run("select _wstartts, _wduration, _wendts, count(*) from t1 interval(10s)"); + run("SELECT _WSTARTTS, _WDURATION, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(PlanIntervalTest, fill) { useDb("root", "test"); - run("select count(*) from t1 interval(10s) fill(linear)"); + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(LINEAR)"); - run("select count(*), sum(c1) from t1 interval(10s) fill(value, 10, 20)"); + run("SELECT COUNT(*), SUM(c1) FROM t1 " + "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(VALUE, 10, 20)"); } + +TEST_F(PlanIntervalTest, selectFunc) { + useDb("root", "test"); + + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row, and with INTERVAL clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); +} \ No newline at end of file diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 0b60eaea32..6e349f8f9c 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -23,4 +23,6 @@ TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); + + run("select tbname, tag1, tag2 from st1"); } diff --git a/source/libs/planner/test/planSessionTest.cpp b/source/libs/planner/test/planSessionTest.cpp index 3ec9c4c387..7d5d826925 100644 --- a/source/libs/planner/test/planSessionTest.cpp +++ b/source/libs/planner/test/planSessionTest.cpp @@ -25,3 +25,12 @@ TEST_F(PlanSessionTest, basic) { run("select count(*) from t1 session(ts, 10s)"); } + +TEST_F(PlanSessionTest, selectFunc) { + useDb("root", "test"); + + // select function for SESSION clause + run("SELECT MAX(c1), MIN(c1) FROM t1 SESSION(ts, 10s)"); + // select function along with the columns of select row, and with SESSION clause + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 46988d6706..83c9621916 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -31,3 +31,12 @@ TEST_F(PlanStateTest, stateExpr) { run("select count(*) from t1 state_window(c1 + 10)"); } + +TEST_F(PlanStateTest, selectFunc) { + useDb("root", "test"); + + // select function for STATE_WINDOW clause + run("SELECT MAX(c1), MIN(c1) FROM t1 STATE_WINDOW(c3)"); + // select function along with the columns of select row, and with STATE_WINDOW clause + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); +} diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index a7726973f0..464c636b66 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -36,11 +36,11 @@ class PlannerEnv : public testing::Environment { static void parseArg(int argc, char* argv[]) { int opt = 0; const char* optstring = ""; - static struct option long_options[] = {{"dump", no_argument, NULL, 'd'}, {0, 0, 0, 0}}; + static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': - g_isDump = true; + setDumpModule(optarg); break; default: break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 9ddf2a4959..e434a6be96 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -34,7 +34,41 @@ using namespace testing; } \ } while (0); -bool g_isDump = false; +enum DumpModule { + DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_PARSER, + DUMP_MODULE_LOGIC, + DUMP_MODULE_OPTIMIZED, + DUMP_MODULE_SPLIT, + DUMP_MODULE_SCALED, + DUMP_MODULE_PHYSICAL, + DUMP_MODULE_SUBPLAN, + DUMP_MODULE_ALL +}; + +DumpModule g_dumpModule = DUMP_MODULE_NOTHING; + +void setDumpModule(const char* pModule) { + if (NULL == pModule) { + g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PARSER; + } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_LOGIC; + } else if (0 == strncasecmp(pModule, "optimized", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_OPTIMIZED; + } else if (0 == strncasecmp(pModule, "split", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SPLIT; + } else if (0 == strncasecmp(pModule, "scaled", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SCALED; + } else if (0 == strncasecmp(pModule, "physical", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } else if (0 == strncasecmp(pModule, "subplan", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SUBPLAN; + } else if (0 == strncasecmp(pModule, "all", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } +} class PlannerTestBaseImpl { public: @@ -66,11 +100,9 @@ class PlannerTestBaseImpl { SQueryPlan* pPlan = nullptr; doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); - if (g_isDump) { - dump(); - } + dump(g_dumpModule); } catch (...) { - dump(); + dump(DUMP_MODULE_ALL); throw; } } @@ -109,23 +141,48 @@ class PlannerTestBaseImpl { res_.physiSubplans_.clear(); } - void dump() { + void dump(DumpModule module) { + if (DUMP_MODULE_NOTHING == module) { + return; + } + cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; - cout << "syntax tree : " << endl; - cout << res_.ast_ << endl; - cout << "raw logic plan : " << endl; - cout << res_.rawLogicPlan_ << endl; - cout << "optimized logic plan : " << endl; - cout << res_.optimizedLogicPlan_ << endl; - cout << "split logic plan : " << endl; - cout << res_.splitLogicPlan_ << endl; - cout << "scaled logic plan : " << endl; - cout << res_.scaledLogicPlan_ << endl; - cout << "physical plan : " << endl; - cout << res_.physiPlan_ << endl; - cout << "physical subplan : " << endl; - for (const auto& subplan : res_.physiSubplans_) { - cout << subplan << endl; + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { + cout << "syntax tree : " << endl; + cout << res_.ast_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { + cout << "raw logic plan : " << endl; + cout << res_.rawLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) { + cout << "optimized logic plan : " << endl; + cout << res_.optimizedLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) { + cout << "split logic plan : " << endl; + cout << res_.splitLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) { + cout << "scaled logic plan : " << endl; + cout << res_.scaledLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) { + cout << "physical plan : " << endl; + cout << res_.physiPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) { + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7aabff811f..7913ef531f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,6 +32,6 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern bool g_isDump; +extern void setDumpModule(const char* pModule); #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index a2b1353093..ca471262ff 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "osDef.h" #include "qworker.h" #include "tlockfree.h" #include "ttimer.h" @@ -301,9 +302,9 @@ typedef struct SQWorkerMgmt { extern SQWorkerMgmt gQwMgmt; -FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } +static FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } -FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } +static FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } #ifdef __cplusplus diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e84f387dbe..bd7ee6321a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -945,7 +945,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); + code = TSDB_CODE_INVALID_MSG; + QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a75b2521bd..84aa5559d0 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -297,6 +297,22 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } +static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) { + int32_t inputLen = varDataLen(buf); + + char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + taosMemoryFree(t); + return; + } + varDataSetLen(t, len); + + colDataAppend(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); +} + + //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t bufSize = pIn->columnData->info.bytes; @@ -313,6 +329,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; + } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + func = ncharToVar; + vton = true; } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; @@ -608,7 +628,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, -/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 7, 7, 7, 0, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 4b2bc4130a..945d59646b 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -37,7 +37,7 @@ bool syncEnvIsStart() { int32_t syncEnvStart() { int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); - //gSyncEnv = doSyncEnvStart(gSyncEnv); + // gSyncEnv = doSyncEnvStart(gSyncEnv); gSyncEnv = doSyncEnvStart(); assert(gSyncEnv != NULL); sTrace("syncEnvStart ok!"); @@ -97,14 +97,14 @@ static SSyncEnv *doSyncEnvStart() { // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - atomic_store_8(&(pSyncEnv->isStart), 1); + atomic_store_8(&(pSyncEnv->isStart), 1); return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { assert(pSyncEnv == gSyncEnv); if (pSyncEnv != NULL) { - atomic_store_8(&(pSyncEnv->isStart), 0); + atomic_store_8(&(pSyncEnv->isStart), 0); taosTmrCleanUp(pSyncEnv->pTimerManager); taosMemoryFree(pSyncEnv); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 262c9d5530..9a8dcc57d9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -141,6 +141,18 @@ const char* syncGetMyRoleStr(int64_t rid) { return s; } +int32_t syncGetVgId(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + int32_t vgId = pSyncNode->vgId; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return vgId; +} + SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -153,6 +165,29 @@ SyncTerm syncGetMyTerm(int64_t rid) { return term; } +void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + memset(pEpSet, 0, sizeof(*pEpSet)); + return; + } + assert(rid == pSyncNode->rid); + pEpSet->numOfEps = 0; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); + pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + (pEpSet->numOfEps)++; + + sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + + } + pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; + + sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -260,6 +295,8 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + sTrace("syncPropose msgType:%d ", pMsg->msgType); + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -459,18 +496,17 @@ void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { syncNodeBecomeLeader(pSyncNode); - + syncNodeLog2("==state change become leader immediately==", pSyncNode); - + // Raft 3.6.2 Committing entries from previous terms - + // use this now syncNodeAppendNoop(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica return; } - syncNodeBecomeFollower(pSyncNode); // for test diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 49da6fd8ed..18f8ddec5c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -40,7 +40,9 @@ int tdbCommit(TENV *pEnv, TXN *pTxn); int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -53,11 +55,9 @@ int tdbDbcMoveToLast(TDBC *pDbc); int tdbDbcMoveToNext(TDBC *pDbc); int tdbDbcMoveToPrev(TDBC *pDbc); int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); - -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); -int tdbDbcDrop(TDBC *pDbc); +int tdbDbcDelete(TDBC *pDbc); int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2665dd1cf6..0d13ec4d2b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -138,67 +138,90 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in } if (btc.idx == -1) { - idx = 0; + btc.idx = 0; } else { if (c > 0) { - idx = btc.idx + 1; - } else if (c < 0) { - idx = btc.idx; - } else { - // TDB does NOT allow same key - tdbBtcClose(&btc); + btc.idx++; + } else if (c == 0) { + // dup key not allowed ASSERT(0); return -1; } } - // make sure enough space to hold the cell - szBuf = kLen + vLen + 14; - pBuf = tdbRealloc(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); - if (pBuf == NULL) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - pBt->pBuf = pBuf; - pCell = (SCell *)pBt->pBuf; - - // encode cell - ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - tdbBtcClose(&btc); ASSERT(0); - return -1; - } - - // mark the page dirty - ret = tdbPagerWrite(pBt->pPager, btc.pPage); - if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); return -1; } - // insert the cell - ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - - // check if need balance - if (btc.pPage->nOverflow > 0) { - ret = tdbBtreeBalance(&btc); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - } - tdbBtcClose(&btc); + return 0; +} +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); + if (ret < 0) { + tdbBtcClose(&btc); + ASSERT(0); + return -1; + } + + if (btc.idx < 0 || c != 0) { + tdbBtcClose(&btc); + return -1; + } + + // delete the key + if (tdbBtcDelete(&btc) < 0) { + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + if (btc.idx == -1) { + btc.idx = 0; + c = 1; + } else { + if (c > 0) { + btc.idx = btc.idx + 1; + } + } + + ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); return 0; } @@ -552,14 +575,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx SCell *pCell; int szLCell, szRCell; + // balance page (iNew) and (iNew-1) for (;;) { pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); - if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + if (!childNotLeaf) { + szRCell = szLCell; } else { - szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; SPage *pPage; @@ -736,6 +759,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } + if (TDB_BTREE_PAGE_IS_ROOT(pParent) && TDB_PAGE_TOTAL_CELLS(pParent) == 0) { + i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]); + // copy content to the parent page + tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); + tdbPageCopy(pNews[0], pParent); + } + for (int i = 0; i < 3; i++) { if (pDivCell[i]) { tdbOsFree(pDivCell[i]); @@ -1357,7 +1387,143 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int if (ppVal) { *ppVal = (void *)pBtc->coder.pVal; - *kLen = pBtc->coder.vLen; + *vLen = pBtc->coder.vLen; + } + + return 0; +} + +int tdbBtcDelete(SBTC *pBtc) { + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + SPager *pPager = pBtc->pBt->pPager; + const void *pKey; + i8 iPage; + SPage *pPage; + SPgno pgno; + SCell *pCell; + int szCell; + int nKey; + int ret; + + ASSERT(idx >= 0 && idx < nCells); + + // drop the cell on the leaf + ret = tdbPagerWrite(pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tdbPageDropCell(pBtc->pPage, idx); + + // update interior page or do balance + if (idx == nCells - 1) { + if (idx) { + pBtc->idx--; + tdbBtcGet(pBtc, &pKey, &nKey, NULL, NULL); + + // loop to update the interial page + pgno = TDB_PAGE_PGNO(pBtc->pPage); + for (iPage = pBtc->iPage - 1; iPage >= 0; iPage--) { + pPage = pBtc->pgStack[iPage]; + idx = pBtc->idxStack[iPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + + if (idx < nCells) { + ret = tdbPagerWrite(pPager, pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // update the cell with new key + pCell = tdbOsMalloc(nKey + 9); + tdbBtreeEncodeCell(pPage, pKey, nKey, &pgno, sizeof(pgno), pCell, &szCell); + + ret = tdbPageUpdateCell(pPage, idx, pCell, szCell); + if (ret < 0) { + tdbOsFree(pCell); + ASSERT(0); + return -1; + } + tdbOsFree(pCell); + break; + } else { + pgno = TDB_PAGE_PGNO(pPage); + } + } + } else { + // delete the leaf page and do balance + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + + return 0; +} + +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert) { + SCell *pCell; + int szCell; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + int szBuf; + void *pBuf; + int ret; + + ASSERT(pBtc->idx >= 0); + + // alloc space + szBuf = kLen + nData + 14; + pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); + if (pBuf == NULL) { + ASSERT(0); + return -1; + } + pBtc->pBt->pBuf = pBuf; + pCell = (SCell *)pBtc->pBt->pBuf; + + // encode cell + ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // mark dirty + ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // insert or update + if (insert) { + ASSERT(pBtc->idx <= nCells); + + ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); + } else { + ASSERT(pBtc->idx < nCells); + + ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell); + } + if (ret < 0) { + ASSERT(0); + return -1; + } + + // check balance + if (pBtc->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } } return 0; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index aadb9238b0..553bb2c646 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -75,10 +75,16 @@ int tdbDbDrop(TDB *pDb) { return 0; } -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } + +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); +} + int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } @@ -117,28 +123,16 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); } -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { - // TODO - ASSERT(0); - return 0; -} - -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) { - // TODO - ASSERT(0); - return 0; -} - -int tdbDbcDrop(TDBC *pDbc) { - // TODO - ASSERT(0); - return 0; -} +int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); +} + int tdbDbcClose(TDBC *pDbc) { if (pDbc) { tdbBtcClose(&pDbc->btc); diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index f0fdd82944..92272fb438 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl return 0; } +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + tdbPageDropCell(pPage, idx); + return tdbPageInsertCell(pPage, idx, pCell, szCell, 0); +} + int tdbPageDropCell(SPage *pPage, int idx) { int lidx; SCell *pCell; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 4c19dee03e..76dacf7b84 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -128,6 +128,8 @@ struct SBTC { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -141,6 +143,8 @@ int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); +int tdbBtcDelete(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert); // tdbPager.c ==================================== @@ -278,6 +282,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); int tdbPageCapacity(int pageSize, int amHdrSize); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1f6f28261f..e575ac156f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -115,12 +115,12 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -TEST(tdb_test, simple_test) { +TEST(tdb_test, simple_insert1) { int ret; TENV *pEnv; TDB *pDb; tdb_cmpr_fn_t compFunc; - int nData = 10000000; + int nData = 1000000; TXN txn; taosRemoveDir("tdb"); @@ -152,7 +152,7 @@ TEST(tdb_test, simple_test) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -202,6 +202,8 @@ TEST(tdb_test, simple_test) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; @@ -233,7 +235,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_test2) { +TEST(tdb_test, simple_insert2) { int ret; TENV *pEnv; TDB *pDb; @@ -269,7 +271,7 @@ TEST(tdb_test, simple_test2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -283,13 +285,15 @@ TEST(tdb_test, simple_test2) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; - std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; - std::cout.write((char *)pVal, vLen) /* << " " << vLen */; - std::cout << std::endl; + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; count++; } @@ -316,4 +320,164 @@ TEST(tdb_test, simple_test2) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); +} + +TEST(tdb_test, simple_delete1) { + int ret; + TDB *pDb; + char key[128]; + char data[128]; + TXN txn; + TENV *pEnv; + SPoolMem *pPool; + void *pKey = NULL; + void *pData = NULL; + int nKey; + TDBC *pDbc; + int nData; + int nKV = 69; + + taosRemoveDir("tdb"); + + pPool = openPool(); + + // open env + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + // loop to insert batch data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); + } + + // loop to delete some data + for (int iData = nKV - 1; iData > 30; iData--) { + sprintf(key, "key%d", iData); + + ret = tdbDbDelete(pDb, key, strlen(key), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + if (iData <= 30) { + GTEST_ASSERT_EQ(ret, 0); + } else { + GTEST_ASSERT_EQ(ret, -1); + } + } + + // loop to iterate the data + tdbDbcOpen(pDb, &pDbc, NULL); + + ret = tdbDbcMoveToFirst(pDbc); + GTEST_ASSERT_EQ(ret, 0); + + pKey = NULL; + pData = NULL; + for (;;) { + ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + if (ret < 0) break; + + std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; + std::cout.write((char *)pData, nData) /* << " " << vLen */; + std::cout << std::endl; + } + + tdbDbcClose(pDbc); + + tdbCommit(pEnv, &txn); + + closePool(pPool); + + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} + +TEST(tdb_test, simple_upsert1) { + int ret; + TENV *pEnv; + TDB *pDb; + int nData = 100000; + char key[64]; + char data[64]; + void *pData = NULL; + SPoolMem *pPool; + TXN txn; + + taosRemoveDir("tdb"); + + // open env + ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + // insert some data + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + // upsert some data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, &txn); + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + tdbDbClose(pDb); + tdbEnvClose(pEnv); } \ No newline at end of file diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 80b5d88a90..718be6aa64 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -891,7 +891,6 @@ static void doDelayTask(void* param) { SCliMsg* pMsg = arg->param1; SCliThrdObj* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); taosMemoryFree(arg); @@ -937,8 +936,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); pCtx->epSet = emsg.epSet; } - addConnToPool(pThrd, pConn); - tTrace("use remote epset, current in use: %d, retry count%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, + addConnToPool(pThrd->pool, pConn); + tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); @@ -950,9 +949,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } if (pCtx->pSem != NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + tTrace("%s cli conn %p(sync) handle resp", pTransInst->label, pConn); + if (pCtx->pRsp == NULL) { + tTrace("%s cli conn %p(sync) failed to resp, ignore", pTransInst->label, pConn); + } else { + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + } tsem_post(pCtx->pSem); + pCtx->pRsp = NULL; } else { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, pEpSet); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 01a20a466a..1bd7d0857e 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -425,10 +425,19 @@ void transDQDestroy(SDelayQueue* queue) { } int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { + uint64_t now = taosGetTimestampMs(); SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask)); task->func = func; task->arg = arg; - task->execTime = taosGetTimestampMs() + timeoutMs; + task->execTime = now + timeoutMs; + + HeapNode* minNode = heapMin(queue->heap); + if (minNode) { + SDelayTask* minTask = container_of(minNode, SDelayTask, node); + if (minTask->execTime < task->execTime) { + timeoutMs = minTask->execTime <= now ? 0 : now - minTask->execTime; + } + } tTrace("timer %p put task into queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 916bcb8862..7378ca3241 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -810,12 +810,14 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; thrd->pTransInst = shandle; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); + uv_os_sock_t fds[2]; if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 709efa3125..00fe8bd0e9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -271,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c new file mode 100644 index 0000000000..77f5ed5051 --- /dev/null +++ b/source/util/src/tskiplist2.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tskiplist2.h" + +struct SSLNode { + int8_t level; + SSLNode *forwards[]; +}; + +struct SSkipList2 { + int8_t level; + uint32_t seed; + int32_t size; + const SSLCfg *pCfg; + SSLNode *pHead[]; +}; + +static void *slMalloc(void *pPool, int32_t size); +static void slFree(void *pPool, void *p); +static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); + +const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, + .nKey = -1, + .nData = -1, + .cmprFn = slCmprFn, + .pPool = NULL, + .xMalloc = slMalloc, + .xFree = slFree}; + +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { + SSkipList2 *pSl = NULL; + int32_t size; + + *ppSl = NULL; + if (pCfg == NULL) pCfg = &slDefaultCfg; + + // check config (TODO) + + // malloc handle + size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; + pSl = pCfg->xMalloc(pCfg->pPool, size); + if (pSl == NULL) { + return -1; + } + + pSl->level = 0; + pSl->seed = taosRand(); + pSl->size = 0; + pSl->pCfg = pCfg; + + // init an empty skiplist + for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { + pSl->pHead[i] = NULL; + } + + *ppSl = pSl; + return 0; +} + +int32_t slClose(SSkipList2 *pSl) { + if (pSl) { + slClear(pSl); + if (pSl->pCfg->xFree) { + pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); + } + } + + return 0; +} + +int32_t slClear(SSkipList2 *pSl) { + // loop to clear sl + for (;;) { + // (TODO) + } + + // init sl (TODO) + + return 0; +} + +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + pSlc->pSl = pSl; + + for (int i = 0; i < SL_MAX_LEVEL; i++) { + if (i < pSl->pCfg->maxLevel) { + } else { + pSlc->forwards[i] = NULL; + } + } + + // TODO + return 0; +} + +int32_t slcClose(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { + // TODO + return 0; +} + +int32_t slcMoveToNext(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToPrev(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToFirst(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToLast(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { + // TODO + return 0; +} + +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { + // TODO + return 0; +} + +int32_t slcDrop(SSLCursor *pSlc) { + // TODO + return 0; +} + +static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } + +static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } + +static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { + ASSERT(nKey1 >= 0 && nKey2 >= 0); + + int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; + int32_t c; + + c = memcmp(pKey1, pKey2, nKey); + if (c == 0) { + if (nKey1 > nKey2) { + c = 1; + } else if (nKey1 < nKey2) { + c = -1; + } + } + + return c; +} \ No newline at end of file diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f14885b72e..828216d1d3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,8 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR, TSDB_DATA_TYPE_BOOL}; -int32_t optrIdxList[] = {2, 11, 6}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t optrIdxList[] = {0, 1, 2}; typedef struct { char* oper; @@ -169,8 +169,8 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 -CaseCtrl gCaseCtrl = { +#if 0 +CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -186,23 +186,15 @@ CaseCtrl gCaseCtrl = { .checkParamNum = false, .printRes = true, .runTimes = 0, -// .caseIdx = -1, -// .caseNum = -1, + .caseIdx = -1, + .caseNum = -1, .caseRunIdx = -1, -// .caseRunNum = -1, - - -// .optrIdxListNum = tListLen(optrIdxList), -// .optrIdxList = optrIdxList, -// .bindColTypeNum = tListLen(bindColTypeList), -// .bindColTypeList = bindColTypeList, - .caseIdx = 22, - .caseNum = 1, - .caseRunNum = 1, - + .caseRunNum = -1, }; -#else -CaseCtrl gCaseCtrl = { +#endif + +#if 0 +CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -211,17 +203,50 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, .checkParamNum = false, .printRes = true, .runTimes = 0, - .caseIdx = 2, - .caseNum = 1, .caseRunIdx = -1, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .caseIdx = 22, + .caseNum = 1, .caseRunNum = 1, }; +#endif +#if 1 +CaseCtrl gCaseCtrl = { // query case with specified col&oper + .bindNullNum = 0, + .prepareStb = false, + .printCreateTblSql = false, + .printQuerySql = true, + .printStmtSql = true, + .rowNum = 0, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = true, + .printRes = true, + .runTimes = 0, + .caseRunIdx = -1, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseIdx = 22, + .caseNum = 1, + .caseRunNum = 1, +}; #endif int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -437,6 +462,29 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { } } +void generateErrorSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + data->sql = taosMemoryCalloc(1, 1024); + + switch(tblIdx) { + case 0: + len = sprintf(data->sql, "insert into %s%d values (?, 1)", bpTbPrefix, tblIdx); + break; + case 1: + len = sprintf(data->sql, "select * from ?"); + break; + case 2: + len = sprintf(data->sql, "select * from %s%d where ? = ?", bpTbPrefix, tblIdx); + break; + default: + len = sprintf(data->sql, "select count(*) from %s%d group by ?", bpTbPrefix, tblIdx); + break; + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { @@ -687,6 +735,8 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } + + void destroyData(BindData *data) { taosMemoryFree(data->tsData); taosMemoryFree(data->boolData); @@ -1329,6 +1379,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1374,6 +1425,74 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* value in query sql */ +int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + prepareQueryData(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + for (int32_t n = 0; n< (gCurCase->rowNum/gCurCase->bindRowNum); ++n) { + bpCheckIsInsert(stmt, 0); + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckQueryResult(stmt, taos, data.sql, data.pBind + n * gCurCase->bindColNum); + } + + bpCheckIsInsert(stmt, 0); + + destroyData(&data); + } + + return 0; +} + + +int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + generateErrorSQL(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("*taos_stmt_prepare error as expected, error:%s\n", taos_stmt_errstr(stmt)); + } else { + printf("!!!taos_stmt_prepare succeed, which should be error\n"); + exit(1); + } + + destroyData(&data); + } + + return 0; +} + #if 0 @@ -1398,153 +1517,6 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2358,153 +2330,6 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2575,152 +2400,6 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - unsigned long long starttime = taosGetTimestampUs(); @@ -2790,153 +2469,6 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -3017,153 +2549,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -3218,1035 +2603,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { return 0; } -//samets -int stmt_funcb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - -//1table 18000 reocrds -int stmt_funcb5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); - - for (int i = 0; i < 18000; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 18000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 18000; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 18000; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 18000; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 18000; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 18000; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 18000; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 18000; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 18000; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 18000; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 1; zz++) { - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1table 200000 reocrds -int stmt_funcb_ssz1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int b[30000]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - - int *lb = taosMemoryMalloc(30000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = taosMemoryMalloc(sizeof(int) * 200000); - - for (int i = 0; i < 30000; ++i) { - lb[i] = 40; - no_null[i] = 0; - v.b[i] = (int8_t)(i % 2); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[30000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 30000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_INT; - params[i+1].buffer_length = sizeof(int); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 30000; - } - - int64_t tts = 0; - for (int64_t i = 0; i < 90000000LL; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(no_null); - - return 0; -} - - -//one table 60 records one time -int stmt_funcb_s1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - - - -//300 tables 60 records single column bind -int stmt_funcb_sc1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1 tables 60 records single column bind -int stmt_funcb_sc2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//10 tables [1...10] records single column bind -int stmt_funcb_sc3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - int g = 0; - for (int i = 0; i < 600; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = g%10+1; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = g%10+1; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = g%10+1; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = g%10+1; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = g%10+1; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = g%10+1; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = g%10+1; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = g%10+1; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = g%10+1; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = g%10+1; - ++g; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 60; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} #endif diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index a82f3a6f59..937ff78c71 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -101,7 +101,7 @@ class TDTestCase: # tdSql.error("select distinct c1, ts from stb1 group by c2") tdSql.error("select distinct c1, ts from t1 group by c2") # tdSql.error("select distinct c1, max(c2) from stb1 ") - tdSql.error("select distinct c1, max(c2) from t1 ") + # tdSql.error("select distinct c1, max(c2) from t1 ") # tdSql.error("select max(c2), distinct c1 from stb1 ") tdSql.error("select max(c2), distinct c1 from t1 ") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 group by t0") diff --git a/tools/taos-tools b/tools/taos-tools index bf6c766986..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c