diff --git a/include/client/taos.h b/include/client/taos.h index 111cd8ad3b..0fd2fd8df9 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -54,6 +54,7 @@ typedef void TAOS_SUB; #define TSDB_DATA_TYPE_BLOB 18 // binary #define TSDB_DATA_TYPE_MEDIUMBLOB 19 #define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string +#define TSDB_DATA_TYPE_MAX 20 typedef enum { TSDB_OPTION_LOCALE, @@ -188,7 +189,7 @@ DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -DLL_EXPORT void taos_reset_current_db(TAOS *taos); +DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 797ebf29c5..5001a99c2a 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -76,6 +76,13 @@ typedef enum { TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA } ETsdbSmaType; +typedef enum { + TSDB_BSMA_TYPE_NONE = 0, // no block-wise SMA + TSDB_BSMA_TYPE_I = 1, // sum/min/max(default) +} ETsdbBSmaType; + +#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I + extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 8332e9d09a..f2fec58d8f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -53,12 +53,12 @@ typedef struct { typedef struct SColumnDataAgg { int16_t colId; - int64_t sum; - int64_t max; - int64_t min; int16_t maxIndex; int16_t minIndex; int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; } SColumnDataAgg; typedef struct SDataBlockInfo { @@ -193,20 +193,19 @@ typedef struct SColumn { uint8_t scale; } SColumn; -typedef struct SLimit { - int64_t limit; - int64_t offset; -} SLimit; - -typedef struct SOrder { - uint32_t order; - SColumn col; -} SOrder; - -typedef struct SGroupbyExpr { - SArray* columnInfo; // SArray, group by columns information - bool groupbyTag; // group by tag or column -} SGroupbyExpr; +typedef struct STableBlockDistInfo { + uint16_t rowSize; + uint16_t numOfFiles; + uint32_t numOfTables; + uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; + int32_t firstSeekTimeUs; + uint32_t numOfRowsInMemTable; + uint32_t numOfSmallBlocks; + SArray *dataBlockInfos; +} STableBlockDistInfo; enum { FUNC_PARAM_TYPE_VALUE = 0x1, @@ -241,15 +240,6 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; -typedef struct SStateWindow { - SColumn col; -} SStateWindow; - -typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) - SColumn col; -} SSessionWindow; - #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 4bbe42bc50..7978430027 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -113,6 +113,20 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin pColumnInfoData->hasNull = true; } +static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + for(int32_t i = start; i < start + nRows; ++i) { + pColumnInfoData->varmeta.offset[i] = -1; // it is a null value of VAR type. + } + } else { + for(int32_t i = start; i < start + nRows; ++i) { + colDataSetNull_f(pColumnInfoData->nullbitmap, i); + } + } + + pColumnInfoData->hasNull = true; +} + static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL); @@ -179,12 +193,14 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); void* blockDataDestroy(SSDataBlock* pBlock); +int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n); + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); void blockDebugShowData(const SArray* dataBlocks); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 4a3ce2db86..da724206d9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -70,11 +70,13 @@ typedef struct { #pragma pack(pop) #define colType(col) ((col)->type) +#define colSma(col) ((col)->sma) #define colColId(col) ((col)->colId) #define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) #define colSetType(col, t) (colType(col) = (t)) +#define colSetSma(col, s) (colSma(col) = (s)) #define colSetColId(col, id) (colColId(col) = (id)) #define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) @@ -139,7 +141,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 31b316861b..6ec7d8c75b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -265,6 +265,20 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +typedef struct { + int8_t type; + int8_t sma; // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; +} SSchemaEx; + +#define SSCHMEA_TYPE(s) ((s)->type) +#define SSCHMEA_SMA(s) ((s)->sma) +#define SSCHMEA_COLID(s) ((s)->colId) +#define SSCHMEA_BYTES(s) ((s)->bytes) +#define SSCHMEA_NAME(s) ((s)->name) + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1381,11 +1395,10 @@ typedef struct { } SDDropTopicReq; typedef struct { - float xFilesFactor; - int8_t delayUnit; - int8_t nFuncIds; - int32_t* pFuncIds; - int64_t delay; + float xFilesFactor; + int32_t delay; + int8_t nFuncIds; + func_id_t* pFuncIds; } SRSmaParam; typedef struct SVCreateTbReq { @@ -1403,13 +1416,12 @@ typedef struct SVCreateTbReq { }; union { struct { - tb_uid_t suid; - uint32_t nCols; - SSchema* pSchema; - uint32_t nTagCols; - SSchema* pTagSchema; - col_id_t nBSmaCols; - col_id_t* pBSmaCols; + tb_uid_t suid; + col_id_t nCols; + col_id_t nBSmaCols; + SSchemaEx* pSchema; + col_id_t nTagCols; + SSchema* pTagSchema; SRSmaParam* pRSmaParam; } stbCfg; struct { @@ -1417,10 +1429,9 @@ typedef struct SVCreateTbReq { SKVRow pTag; } ctbCfg; struct { - uint32_t nCols; - SSchema* pSchema; - col_id_t nBSmaCols; - col_id_t* pBSmaCols; + col_id_t nCols; + col_id_t nBSmaCols; + SSchemaEx* pSchema; SRSmaParam* pRSmaParam; } ntbCfg; }; @@ -1899,7 +1910,10 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq) typedef struct { uint32_t nCols; - SSchema* pSchema; + union { + SSchema* pSchema; + SSchemaEx* pSchemaEx; + }; } SSchemaWrapper; static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index cb59599d9a..6c3671a8d6 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -50,7 +50,6 @@ typedef struct { PutToQueueFp queueFps[QUEUE_MAX]; GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; SendRspFp sendRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; @@ -60,7 +59,6 @@ void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); -int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq); void tmsgSendRsp(const SRpcMsg* pRsp); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 8d996be21e..afd2a01540 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -172,7 +172,7 @@ #define TK_SYNCDB 154 #define TK_NULL 155 #define TK_NK_VARIABLE 156 -#define TK_NK_UNDERLINE 157 +#define TK_NOW 157 #define TK_ROWTS 158 #define TK_TBNAME 159 #define TK_QSTARTTS 160 @@ -229,7 +229,6 @@ #define TK_NK_COLON 500 #define TK_NK_BITNOT 501 #define TK_INSERT 502 -#define TK_NOW 504 #define TK_VALUES 507 #define TK_IMPORT 509 #define TK_NK_SEMI 508 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 19442af206..1cb398fb85 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -31,6 +31,7 @@ typedef int16_t col_id_t; typedef int8_t col_type_t; typedef int32_t col_bytes_t; typedef uint16_t schema_ver_t; +typedef int32_t func_id_t; #pragma pack(push, 1) typedef struct { diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 5b88a9d6af..08ab63e55a 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -29,8 +29,7 @@ extern "C" { typedef struct SMnode SMnode; typedef struct { - int32_t dnodeId; - int64_t clusterId; + bool deploy; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 278d9d8b7c..fe9edc323d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -52,7 +52,12 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; -#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results +typedef struct SFileBlockInfo { + int32_t numBlocksOfStep; +} SFileBlockInfo; + +#define TSDB_BLOCK_DIST_STEP_ROWS 8 +#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define FUNCTION_TYPE_SCALAR 1 #define FUNCTION_TYPE_AGG 2 @@ -101,10 +106,6 @@ typedef struct SFuncExecFuncs { #define FUNCTION_DERIVATIVE 32 #define FUNCTION_BLKINFO 33 -#define FUNCTION_HISTOGRAM 34 -#define FUNCTION_HLL 35 -#define FUNCTION_MODE 36 -#define FUNCTION_SAMPLE 37 #define FUNCTION_COV 38 @@ -183,11 +184,11 @@ typedef struct SqlFunctionCtx { int32_t columnIndex; // TODO remove it uint8_t currentStage; // record current running step, default: 0 bool isAggSet; + int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it ///////////////////////////////////////////////////////////////// bool stableQuery; int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data - int64_t startTs; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7d46b543cb..57a3862a9a 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -72,10 +72,15 @@ typedef enum EFunctionType { FUNCTION_TYPE_ATAN, // string function - FUNCTION_TYPE_CHAR_LENGTH = 1500, + FUNCTION_TYPE_LENGTH = 1500, + FUNCTION_TYPE_CHAR_LENGTH, FUNCTION_TYPE_CONCAT, FUNCTION_TYPE_CONCAT_WS, - FUNCTION_TYPE_LENGTH, + FUNCTION_TYPE_LOWER, + FUNCTION_TYPE_UPPER, + FUNCTION_TYPE_LTRIM, + FUNCTION_TYPE_RTRIM, + FUNCTION_TYPE_SUBSTR, // conversion function FUNCTION_TYPE_CAST = 2000, @@ -127,6 +132,7 @@ bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); +bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9668082696..d2217f7e01 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -213,16 +213,16 @@ typedef enum EDealRes { } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNodeptr a, const SNodeptr b); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index ddcbaa0bee..a8ff48072f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -79,6 +79,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool genByCalc; union { bool b; int64_t i; @@ -277,7 +278,6 @@ typedef struct SVnodeModifOpStmt { ENodeType nodeType; ENodeType sqlNodeType; SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position @@ -306,6 +306,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod bool nodesIsExprNode(const SNode* pNode); +bool nodesIsUnaryOp(const SOperatorNode* pOp); bool nodesIsArithmeticOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 7bc0ee42e9..b5acc64f0b 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut); +/* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -58,6 +59,17 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +/* String functions */ +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 157e0cb721..3e2f596784 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -58,6 +58,9 @@ typedef struct { void *pNode; } SNodeMsg; +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); +typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -74,10 +77,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + RpcCfp cfp; // call back to retrieve the client auth info, for server app only - int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + RpcAfp afp;; void *parent; } SRpcInit; diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 34e2422167..e516000c66 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -38,8 +38,10 @@ int32_t taosMemorySize(void *ptr); #define taosMemoryFreeClear(ptr) \ do { \ - taosMemoryFree(ptr); \ - (ptr)=NULL; \ + if (ptr) { \ + taosMemoryFree(ptr); \ + (ptr) = NULL; \ + } \ } while (0) #ifdef __cplusplus diff --git a/include/os/osShm.h b/include/os/osShm.h index d26a99e277..61ffc0f6cc 100644 --- a/include/os/osShm.h +++ b/include/os/osShm.h @@ -26,7 +26,7 @@ typedef struct { void* ptr; } SShm; -int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ; +int32_t taosCreateShm(SShm *pShm, int32_t key, int32_t shmsize) ; void taosDropShm(SShm *pShm); int32_t taosAttachShm(SShm *pShm); diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 838b175a28..3f00c79f46 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -57,6 +57,8 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { return POINTER_SHIFT(buf, sizeof(*value)); } +static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); } + // ---- Fixed U16 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { if (buf != NULL) { diff --git a/include/util/tdef.h b/include/util/tdef.h index 517cb8baee..f1af9eca1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -128,18 +128,20 @@ extern const int32_t TYPE_BYTES[15]; } while (0) typedef enum EOperatorType { - // arithmetic operator + // binary arithmetic operator OP_TYPE_ADD = 1, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, OP_TYPE_MOD, + // unary arithmetic operator + OP_TYPE_MINUS, // bit operator OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, - // comparison operator + // binary comparison operator OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, @@ -152,6 +154,7 @@ typedef enum EOperatorType { OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, + // unary comparison operator OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, @@ -304,7 +307,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_DAYS_PER_FILE (1 * 1440) // unit minute +#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) @@ -477,9 +480,9 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MND_VGID -1 -#define QND_VGID 1 -#define VND_VGID 0 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE 1 +#define DEFAULT_HANDLE 0 #define MAX_NUM_STR_SIZE 40 diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 3a47450eec..bc0c8fe5a4 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -22,7 +22,7 @@ extern "C" { #endif -typedef enum { PROC_REQ, PROC_RSP, PROC_REG, PROC_RELEASE } ProcFuncType; +typedef enum { PROC_QUEUE, PROC_REQ, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType; typedef struct SProcQueue SProcQueue; typedef struct SProcObj SProcObj; @@ -53,7 +53,7 @@ void taosProcCleanup(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc); int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, ProcFuncType ftype); -int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, +void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, ProcFuncType ftype); #ifdef __cplusplus diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 49cb12cccd..0749ce9e7d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -136,7 +136,7 @@ typedef struct STscObj { uint32_t connId; int32_t connType; uint64_t id; // ref ID returned by taosAddRef - TdThreadMutex mutex; // used to protect the operation on db + TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; } STscObj; @@ -152,7 +152,8 @@ typedef struct SResultColumn { typedef struct SReqResultInfo { const char* pRspMsg; const char* pData; - TAOS_FIELD* fields; + TAOS_FIELD* fields; // todo, column names are not needed. + TAOS_FIELD* userFields; // the fields info that return to user uint32_t numOfCols; int32_t* length; char** convertBuf; @@ -221,6 +222,7 @@ void destroyRequest(SRequestObj* pRequest); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); +void resetConnectDB(STscObj* pTscObj); void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char* str); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 53cc77af1a..769870ada8 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -169,6 +169,7 @@ static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->row); taosMemoryFreeClear(pResInfo->pCol); taosMemoryFreeClear(pResInfo->fields); + taosMemoryFreeClear(pResInfo->userFields); if (pResInfo->convertBuf != NULL) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8cd53f18b0..80559c025f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -56,7 +56,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } char localDb[TSDB_DB_NAME_LEN] = {0}; - if (db != NULL) { + if (db != NULL && strlen(db) > 0) { if (!validateDbName(db)) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; return NULL; @@ -164,6 +164,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { if ((*pQuery)->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); } + TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*); TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*); } @@ -228,12 +229,24 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t assert(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; - pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0])); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); + pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; - pResInfo->fields[i].type = pSchema[i].type; + pResInfo->fields[i].type = pSchema[i].type; + + pResInfo->userFields[i].bytes = pSchema[i].bytes; + pResInfo->userFields[i].type = pSchema[i].type; + + if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) { + pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + tstrncpy(pResInfo->userFields[i].name, pSchema[i].name, tListLen(pResInfo->userFields[i].name)); } } @@ -740,6 +753,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pStart += colLength[i]; } + // convert UCS4-LE encoded character to native multi-bytes character in current data block. for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; int32_t bytes = pResultInfo->fields[i].bytes; @@ -766,6 +780,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 } pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; } } @@ -791,6 +806,16 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { taosThreadMutexUnlock(&pTscObj->mutex); } +void resetConnectDB(STscObj* pTscObj) { + if (pTscObj == NULL) { + return; + } + + taosThreadMutexLock(&pTscObj->mutex); + pTscObj->db[0] = 0; + taosThreadMutexUnlock(&pTscObj->mutex); +} + int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { assert(pResultInfo != NULL && pRsp != NULL); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 42d4b5b2e5..ee4fc7fcb5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { } SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo); - return pResInfo->fields; + return pResInfo->userFields; } TAOS_RES *taos_query(TAOS *taos, const char *sql) { @@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - // TODO - return NULL; + SRequestObj* pRequest = (SRequestObj*) res; + if (pRequest == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + if (taos_is_update_query(res)) { + return NULL; + } + + return &pRequest->body.resInfo.row; } // todo intergrate with tDataTypes @@ -313,7 +322,9 @@ int taos_affected_rows(TAOS_RES *res) { return pResInfo->numOfRows; } -int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } +int taos_result_precision(TAOS_RES *res) { + return TSDB_TIME_PRECISION_MILLI; +} int taos_select_db(TAOS *taos, const char *db) { STscObj *pObj = (STscObj *)taos; @@ -365,8 +376,7 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { } bool taos_is_update_query(TAOS_RES *res) { - // TODO - return true; + return taos_num_fields(res) == 0; } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { @@ -393,8 +403,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { int taos_validate_sql(TAOS *taos, const char *sql) { return true; } void taos_reset_current_db(TAOS *taos) { - // TODO - return; + if (taos == NULL) { + return; + } + + resetConnectDB(taos); } const char *taos_get_server_info(TAOS *taos) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7f25fd1e80..fb6640e59e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1139,7 +1139,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { } } -int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { if (0 == numOfRows) { return TSDB_CODE_SUCCESS; } @@ -1181,7 +1181,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = blockDataEnsureColumnCapacity(p, numOfRows); + code = colInfoDataEnsureCapacity(p, numOfRows); if (code) { return code; } @@ -1233,6 +1233,63 @@ void colDataDestroy(SColumnInfoData* pColData) { taosMemoryFree(pColData->pData); } + +static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { + int32_t len = BitmapLen(total); + + int32_t newLen = BitmapLen(total - n); + if (n%8 == 0) { + memmove(nullBitmap, nullBitmap + n/8, newLen); + } else { + int32_t tail = n % 8; + int32_t i = 0; + + uint8_t* p = (uint8_t*) nullBitmap; + while(i < len) { + uint8_t v = p[i]; + + p[i] = 0; + p[i] = (v << tail); + + if (i < len - 1) { + uint8_t next = p[i + 1]; + p[i] |= (next >> (8 - tail)); + } + + i += 1; + } + } +} + +static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n)); + memset(&pColInfoData->varmeta.offset[total - n - 1], 0, n); + } else { + int32_t bytes = pColInfoData->info.bytes; + memmove(pColInfoData->pData, ((char*)pColInfoData->pData + n * bytes), (total - n) * bytes); + doShiftBitmap(pColInfoData->nullbitmap, n, total); + } +} + +int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { + if (n == 0) { + return TSDB_CODE_SUCCESS; + } + + if (pBlock->info.rows <= n) { + blockDataCleanup(pBlock); + } else { + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); + } + + pBlock->info.rows -= n; + } + return TSDB_CODE_SUCCESS; +} + int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.uid; int16_t numOfCols = pBlock->info.numOfCols; @@ -1372,6 +1429,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } + void blockDebugShowData(const SArray* dataBlocks) { char pBuf[128]; int32_t sz = taosArrayGetSize(dataBlocks); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7fd66e95ad..1d7a4aeb8f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -85,6 +85,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); tlen += taosEncodeFixedI8(buf, colType(pCol)); + tlen += taosEncodeFixedI8(buf, colSma(pCol)); tlen += taosEncodeFixedI16(buf, colColId(pCol)); tlen += taosEncodeFixedI16(buf, colBytes(pCol)); } @@ -107,12 +108,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; + int8_t sma = TSDB_BSMA_TYPE_NONE; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); + buf = taosDecodeFixedI8(buf, &sma); buf = taosDecodeFixedI16(buf, &colId); buf = taosDecodeFixedI32(buf, &bytes); - if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) { + if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; } @@ -148,7 +151,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->version = version; } -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) { +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { @@ -161,6 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); colSetType(pCol, type); colSetColId(pCol, colId); + colSetSma(pCol, sma); if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { @@ -168,9 +172,6 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]); } - // TODO: set sma value by user input - pCol->sma = 1; - if (IS_VAR_DATA_TYPE(type)) { colSetBytes(pCol, bytes); pBuilder->tlen += (TYPE_BYTES[type] + bytes); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 06dd2906be..2a16b58565 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -296,33 +296,30 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { switch (pReq->type) { case TD_SUPER_TABLE: tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid); - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma); tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); } - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols); + for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); } - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); - } if (pReq->rollup && pReq->stbCfg.pRSmaParam) { SRSmaParam *param = pReq->stbCfg.pRSmaParam; - tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); - tlen += taosEncodeFixedI8(buf, param->delayUnit); + tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); + tlen += taosEncodeFixedI32(buf, param->delay); tlen += taosEncodeFixedI8(buf, param->nFuncIds); for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } - tlen += taosEncodeFixedI64(buf, param->delay); } break; case TD_CHILD_TABLE: @@ -330,26 +327,23 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); break; case TD_NORMAL_TABLE: - tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); + tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma); tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } - tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { - tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]); - } if (pReq->rollup && pReq->ntbCfg.pRSmaParam) { SRSmaParam *param = pReq->ntbCfg.pRSmaParam; - tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); - tlen += taosEncodeFixedI8(buf, param->delayUnit); + tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); + tlen += taosEncodeFixedI32(buf, param->delay); tlen += taosEncodeFixedI8(buf, param->nFuncIds); for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } - tlen += taosEncodeFixedI64(buf, param->delay); } break; default: @@ -370,45 +364,38 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { switch (pReq->type) { case TD_SUPER_TABLE: buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); - buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); + pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx)); + for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } - buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); + buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols); pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - if (pReq->stbCfg.nBSmaCols > 0) { - pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); - } - } else { - pReq->stbCfg.pBSmaCols = NULL; - } if (pReq->rollup) { pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; - buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); - buf = taosDecodeFixedI8(buf, ¶m->delayUnit); + buf = taosDecodeBinaryTo(buf, (void*)¶m->xFilesFactor, sizeof(param->xFilesFactor)); + buf = taosDecodeFixedI32(buf, ¶m->delay); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); if (param->nFuncIds > 0) { + param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t)); for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { param->pFuncIds = NULL; } - buf = taosDecodeFixedI64(buf, ¶m->delay); } else { pReq->stbCfg.pRSmaParam = NULL; } @@ -418,37 +405,30 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); break; case TD_NORMAL_TABLE: - buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); + buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); + pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx)); + for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); + buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma); buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } - buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); - if (pReq->ntbCfg.nBSmaCols > 0) { - pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); - for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { - buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i); - } - } else { - pReq->ntbCfg.pBSmaCols = NULL; - } if (pReq->rollup) { pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->ntbCfg.pRSmaParam; - buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); - buf = taosDecodeFixedI8(buf, ¶m->delayUnit); + buf = taosDecodeBinaryTo(buf, (void*)¶m->xFilesFactor, sizeof(param->xFilesFactor)); + buf = taosDecodeFixedI32(buf, ¶m->delay); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); if (param->nFuncIds > 0) { + param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t)); for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { param->pFuncIds = NULL; } - buf = taosDecodeFixedI64(buf, ¶m->delay); } else { pReq->ntbCfg.pRSmaParam = NULL; } diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index e90634a604..cb5e2b07c1 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -32,10 +32,6 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); } -int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) { - return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq); -} - void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 0c1bafbeb8..ef75adeb5d 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -431,7 +431,7 @@ FORCE_INLINE void *getDataMax(int32_t type) { } } -bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } +bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; } void setVardataNull(void *val, int32_t type) { if (type == TSDB_DATA_TYPE_BINARY) { diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 81c434eb27..0c9a3733a2 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,10 +1,10 @@ -aux_source_directory(dm/src DNODE_SRC) -aux_source_directory(qm/src DNODE_SRC) -aux_source_directory(bm/src DNODE_SRC) -aux_source_directory(sm/src DNODE_SRC) -aux_source_directory(vm/src DNODE_SRC) -aux_source_directory(mm/src DNODE_SRC) -aux_source_directory(main/src DNODE_SRC) +aux_source_directory(dm DNODE_SRC) +aux_source_directory(qm DNODE_SRC) +aux_source_directory(bm DNODE_SRC) +aux_source_directory(sm DNODE_SRC) +aux_source_directory(vm DNODE_SRC) +aux_source_directory(mm DNODE_SRC) +aux_source_directory(main DNODE_SRC) add_library(dnode STATIC ${DNODE_SRC}) target_link_libraries( dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor @@ -12,20 +12,14 @@ target_link_libraries( target_include_directories( dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -aux_source_directory(main/exe EXEC_SRC) +aux_source_directory(exe EXEC_SRC) add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/bm/src/bmMsg.c b/source/dnode/mgmt/bm/bmHandle.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmMsg.c rename to source/dnode/mgmt/bm/bmHandle.c diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/bmInt.c similarity index 91% rename from source/dnode/mgmt/bm/src/bmInt.c rename to source/dnode/mgmt/bm/bmInt.c index 4b87f4463c..2cb0f50dfe 100644 --- a/source/dnode/mgmt/bm/src/bmInt.c +++ b/source/dnode/mgmt/bm/bmInt.c @@ -19,12 +19,7 @@ static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { - SMsgCb msgCb = {0}; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); pOption->msgCb = msgCb; } @@ -114,7 +109,7 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { return code; } -void bmGetMgmtFp(SMgmtWrapper *pWrapper) { +void bmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = bmOpen; mgmtFp.closeFp = bmClose; diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/bmWorker.c similarity index 98% rename from source/dnode/mgmt/bm/src/bmWorker.c rename to source/dnode/mgmt/bm/bmWorker.c index 932c008e34..a5a97f6af0 100644 --- a/source/dnode/mgmt/bm/src/bmWorker.c +++ b/source/dnode/mgmt/bm/bmWorker.c @@ -18,7 +18,7 @@ static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rpcRsp); + tmsgSendRsp(&rpcRsp); dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); diff --git a/source/dnode/mgmt/bm/inc/bm.h b/source/dnode/mgmt/bm/inc/bm.h deleted file mode 100644 index 79cf76d113..0000000000 --- a/source/dnode/mgmt/bm/inc/bm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_H_ -#define _TD_DND_BNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void bmGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dm/src/dmFile.c b/source/dnode/mgmt/dm/dmFile.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmFile.c rename to source/dnode/mgmt/dm/dmFile.c diff --git a/source/dnode/mgmt/dm/dmHandle.c b/source/dnode/mgmt/dm/dmHandle.c new file mode 100644 index 0000000000..6492995c5c --- /dev/null +++ b/source/dnode/mgmt/dm/dmHandle.c @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +void dmSendStatusReq(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + SStatusReq req = {0}; + + taosRLockLatch(&pMgmt->latch); + req.sver = tsVersion; + req.dver = pMgmt->dver; + req.dnodeId = pDnode->dnodeId; + req.clusterId = pDnode->clusterId; + req.rebootTime = pDnode->rebootTime; + req.updateTime = pMgmt->updateTime; + req.numOfCores = tsNumOfCores; + req.numOfSupportVnodes = pDnode->numOfSupportVnodes; + tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN); + + req.clusterCfg.statusInterval = tsStatusInterval; + req.clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); + taosRUnLockLatch(&pMgmt->latch); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(pWrapper, req.pVloads); + dndReleaseWrapper(pWrapper); + } + + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSStatusReq(pHead, contLen, &req); + taosArrayDestroy(req.pVloads); + + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527}; + pMgmt->statusSent = 1; + + dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + tmsgSendReq(&pMgmt->msgCb, &epSet, &rpcMsg); +} + +static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { + SDnode *pDnode = pMgmt->pDnode; + + if (pDnode->dnodeId == 0) { + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + taosWLockLatch(&pMgmt->latch); + pDnode->dnodeId = pCfg->dnodeId; + pDnode->clusterId = pCfg->clusterId; + dmWriteFile(pMgmt); + taosWUnLockLatch(&pMgmt->latch); + } +} + +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRsp = &pMsg->rpcMsg; + + if (pRsp->code != TSDB_CODE_SUCCESS) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) { + dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId); + pDnode->dropped = 1; + dmWriteFile(pMgmt); + } + } else { + SStatusRsp statusRsp = {0}; + if (pRsp->pCont != NULL && pRsp->contLen != 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + pMgmt->dver = statusRsp.dver; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps); + } + tFreeSStatusRsp(&statusRsp); + } + + pMgmt->statusSent = 0; +} + +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("auth rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("grant rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDCfgDnodeReq *pCfg = pReq->pCont; + dError("config req is received, but not supported yet"); + return TSDB_CODE_OPS_NOT_SUPPORT; +} + + +static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dmProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dmProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dmProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dmProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dmProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dmProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dmProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dmProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} + +static void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} + +void dmInitMsgHandle(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmPutMsgToStatusWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); +} diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/dmInt.c similarity index 89% rename from source/dnode/mgmt/dm/src/dmInt.c rename to source/dnode/mgmt/dm/dmInt.c index 3c5f394d5e..c710af9006 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/dmInt.c @@ -78,7 +78,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) { return dmStartThread(pWrapper->pMgmt); } -int32_t dmInit(SMgmtWrapper *pWrapper) { +static int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); dInfo("dnode-mgmt start to init"); @@ -112,22 +112,19 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return -1; } - if (dndInitServer(pDnode) != 0) { - dError("failed to init trans server since %s", terrstr()); - return -1; - } - - if (dndInitClient(pDnode) != 0) { - dError("failed to init trans client since %s", terrstr()); + if (dndInitTrans(pDnode) != 0) { + dError("failed to init transport since %s", terrstr()); return -1; } pWrapper->pMgmt = pMgmt; + pMgmt->msgCb = dndCreateMsgcb(pWrapper); + dInfo("dnode-mgmt is initialized"); return 0; } -void dmCleanup(SMgmtWrapper *pWrapper) { +static void dmCleanup(SMgmtWrapper *pWrapper) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; @@ -151,25 +148,24 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); + dndCleanupTrans(pDnode); dInfo("dnode-mgmt is cleaned up"); } -int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { +static int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { *required = true; return 0; } -void dmGetMgmtFp(SMgmtWrapper *pWrapper) { +void dmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = dmInit; mgmtFp.closeFp = dmCleanup; mgmtFp.startFp = dmStart; mgmtFp.requiredFp = dmRequire; - dmInitMsgHandles(pWrapper); + dmInitMsgHandle(pWrapper); pWrapper->name = "dnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/dmWorker.c similarity index 73% rename from source/dnode/mgmt/dm/src/dmWorker.c rename to source/dnode/mgmt/dm/dmWorker.c index 63b9704b78..40ca858183 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/dmWorker.c @@ -14,16 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "bm.h" #include "dmInt.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; int64_t lastStatusTime = taosGetTimestampMs(); int64_t lastMonitorTime = lastStatusTime; @@ -37,8 +32,7 @@ static void *dmThreadRoutine(void *param) { } int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; + float statusInterval = (curTime - lastStatusTime) / 1000.0f; if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { dmSendStatusReq(pMgmt); lastStatusTime = curTime; @@ -52,25 +46,26 @@ static void *dmThreadRoutine(void *param) { } } +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode queue", pMsg); switch (pRpc->msgType) { - case TDMT_DND_CREATE_MNODE: - case TDMT_DND_CREATE_QNODE: - case TDMT_DND_CREATE_SNODE: - case TDMT_DND_CREATE_BNODE: - case TDMT_DND_DROP_MNODE: - case TDMT_DND_DROP_QNODE: - case TDMT_DND_DROP_SNODE: - case TDMT_DND_DROP_BNODE: - code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); - break; case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -84,8 +79,8 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { code = dmProcessGrantRsp(pMgmt, pMsg); break; default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + code = dmProcessCDnodeMsg(pMgmt->pDnode, pMsg); + break; } if (pRpc->msgType & 1u) { @@ -100,16 +95,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { } int32_t dmStartWorker(SDnodeMgmt *pMgmt) { - SSingleWorkerCfg mgmtCfg = { - .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { + SSingleWorkerCfg mcfg = {.min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mcfg) != 0) { dError("failed to start dnode mgmt worker since %s", terrstr()); return -1; } - SSingleWorkerCfg statusCfg = { - .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { + SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) { dError("failed to start dnode status worker since %s", terrstr()); return -1; } @@ -118,17 +111,6 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return 0; } -int32_t dmStartThread(SDnodeMgmt *pMgmt) { - pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - void dmStopWorker(SDnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->statusWorker); @@ -140,12 +122,18 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { dDebug("dnode workers are closed"); } -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } + + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->statusWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); diff --git a/source/dnode/mgmt/dm/inc/dm.h b/source/dnode/mgmt/dm/inc/dm.h deleted file mode 100644 index 3984e6dbd4..0000000000 --- a/source/dnode/mgmt/dm/inc/dm.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDnodeMgmt SDnodeMgmt; - -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void dmInitMsgHandles(SMgmtWrapper *pWrapper); - -void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c deleted file mode 100644 index 3795cffbfa..0000000000 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmInt.h" -#include "vm.h" - -void dmSendStatusReq(SDnodeMgmt *pMgmt) { - SDnode *pDnode = pMgmt->pDnode; - SStatusReq req = {0}; - - taosRLockLatch(&pMgmt->latch); - req.sver = tsVersion; - req.dver = pMgmt->dver; - req.dnodeId = pDnode->dnodeId; - req.clusterId = pDnode->clusterId; - req.rebootTime = pDnode->rebootTime; - req.updateTime = pMgmt->updateTime; - req.numOfCores = tsNumOfCores; - req.numOfSupportVnodes = pDnode->numOfSupportVnodes; - tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN); - - req.clusterCfg.statusInterval = tsStatusInterval; - req.clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); - memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pMgmt->latch); - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); - if (pWrapper != NULL) { - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - vmMonitorVnodeLoads(pWrapper, req.pVloads); - dndReleaseWrapper(pWrapper); - } - - int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); - tSerializeSStatusReq(pHead, contLen, &req); - taosArrayDestroy(req.pVloads); - - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527}; - pMgmt->statusSent = 1; - - dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); - dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); -} - -static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { - SDnode *pDnode = pMgmt->pDnode; - - if (pDnode->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pMgmt->latch); - pDnode->dnodeId = pCfg->dnodeId; - pDnode->clusterId = pCfg->clusterId; - dmWriteFile(pMgmt); - taosWUnLockLatch(&pMgmt->latch); - } -} - -int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SDnode *pDnode = pMgmt->pDnode; - SRpcMsg *pRsp = &pMsg->rpcMsg; - - if (pRsp->code != TSDB_CODE_SUCCESS) { - if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId); - pDnode->dropped = 1; - dmWriteFile(pMgmt); - } - } else { - SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen != 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pMgmt->dver = statusRsp.dver; - dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); - dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps); - } - tFreeSStatusRsp(&statusRsp); - } - - pMgmt->statusSent = 0; -} - -int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pRsp = &pMsg->rpcMsg; - dError("auth rsp is received, but not supported yet"); - return 0; -} - -int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pRsp = &pMsg->rpcMsg; - dError("grant rsp is received, but not supported yet"); - return 0; -} - -int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; - SDCfgDnodeReq *pCfg = pReq->pCont; - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; -} - -void dmInitMsgHandles(SMgmtWrapper *pWrapper) { - // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID); - - // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID); -} diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c similarity index 97% rename from source/dnode/mgmt/main/exe/dndMain.c rename to source/dnode/mgmt/exe/dndMain.c index 0d2ddbfbbc..c6a109d62a 100644 --- a/source/dnode/mgmt/main/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -30,7 +30,6 @@ static struct { } global = {0}; static void dndStopDnode(int signum, void *info, void *ctx) { - dInfo("system signal:%d received", signum); SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); if (pDnode != NULL) { dndHandleEvent(pDnode, DND_EVENT_STOP); @@ -41,8 +40,10 @@ static void dndSetSignalHandle() { taosSetSignal(SIGTERM, dndStopDnode); taosSetSignal(SIGHUP, dndStopDnode); taosSetSignal(SIGINT, dndStopDnode); + taosSetSignal(SIGTSTP, dndStopDnode); taosSetSignal(SIGABRT, dndStopDnode); taosSetSignal(SIGBREAK, dndStopDnode); + taosSetSignal(SIGQUIT, dndStopDnode); if (!tsMultiProcess) { } else if (global.ntype == DNODE || global.ntype == NODE_MAX) { @@ -72,7 +73,7 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) { } else if (strcmp(argv[i], "-n") == 0) { global.ntype = atoi(argv[++i]); if (global.ntype <= DNODE || global.ntype > NODE_MAX) { - printf("'-n' range is [1-5], default is 0\n"); + printf("'-n' range is [1 - %d], default is 0\n", NODE_MAX - 1); return -1; } } else if (strcmp(argv[i], "-k") == 0) { diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h similarity index 98% rename from source/dnode/mgmt/bm/inc/bmInt.h rename to source/dnode/mgmt/inc/bmInt.h index f19ba4e034..7a719633fa 100644 --- a/source/dnode/mgmt/bm/inc/bmInt.h +++ b/source/dnode/mgmt/inc/bmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_BNODE_INT_H_ #define _TD_DND_BNODE_INT_H_ -#include "bm.h" +#include "dnd.h" #include "bnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/inc/dmInt.h similarity index 76% rename from source/dnode/mgmt/dm/inc/dmInt.h rename to source/dnode/mgmt/inc/dmInt.h index 3036d1f5ad..e68f7ddb65 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/inc/dmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_DNODE_INT_H_ #define _TD_DND_DNODE_INT_H_ -#include "dm.h" +#include "dnd.h" #ifdef __cplusplus extern "C" { @@ -33,28 +33,38 @@ typedef struct SDnodeMgmt { SRWLatch latch; SSingleWorker mgmtWorker; SSingleWorker statusWorker; + SMsgCb msgCb; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; } SDnodeMgmt; +// dmInt.c +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + // dmFile.c int32_t dmReadFile(SDnodeMgmt *pMgmt); int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); -// dmMsg.c +// dmHandle.c +void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg); // dmWorker.c +int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); -int32_t dmStartThread(SDnodeMgmt *pMgmt); -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/inc/dnd.h similarity index 52% rename from source/dnode/mgmt/main/inc/dnd.h rename to source/dnode/mgmt/inc/dnd.h index b9c760c980..449c4399db 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/inc/dnd.h @@ -27,13 +27,13 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "tmsgcb.h" #include "tprocess.h" #include "tqueue.h" #include "trpc.h" #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tmsgcb.h" #include "dnode.h" #include "monitor.h" @@ -42,12 +42,42 @@ extern "C" { #endif -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} +#define dFatal(...) \ + { \ + if (dDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ + } +#define dError(...) \ + { \ + if (dDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ + } +#define dWarn(...) \ + { \ + if (dDebugFlag & DEBUG_WARN) { \ + taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ + } +#define dInfo(...) \ + { \ + if (dDebugFlag & DEBUG_INFO) { \ + taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ + } +#define dDebug(...) \ + { \ + if (dDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); \ + } \ + } +#define dTrace(...) \ + { \ + if (dDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); \ + } \ + } typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; @@ -73,7 +103,7 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef struct SMsgHandle { - SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pQndWrapper; SMgmtWrapper *pMndWrapper; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -92,6 +122,7 @@ typedef struct SMgmtWrapper { char *path; int32_t refCount; SRWLatch latch; + ENodeType ntype; bool deployed; bool required; EProcType procType; @@ -134,32 +165,63 @@ typedef struct SDnode { SMgmtWrapper wrappers[NODE_MAX]; } SDnode; -const char *dndNodeLogStr(ENodeType ntype); -const char *dndNodeProcStr(ENodeType ntype); -const char *dndEventStr(EDndEvent ev); -EDndStatus dndGetStatus(SDnode *pDnode); -void dndSetStatus(SDnode *pDnode, EDndStatus stat); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dndSendMonitorReport(SDnode *pDnode); - -int32_t dndInitServer(SDnode *pDnode); -void dndCleanupServer(SDnode *pDnode); -int32_t dndInitClient(SDnode *pDnode); -void dndCleanupClient(SDnode *pDnode); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); - +// dndFile.h int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); +// dndInt.h +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); void dndReleaseWrapper(SMgmtWrapper *pWrapper); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); + +// dndMonitor.h +void dndSendMonitorReport(SDnode *pDnode); + +// dndStr.h +const char *dndStatStr(EDndStatus stat); +const char *dndNodeLogStr(ENodeType ntype); +const char *dndNodeProcStr(ENodeType ntype); +const char *dndEventStr(EDndEvent ev); + +// dndTransport.h +int32_t dndInitTrans(SDnode *pDnode); +void dndCleanupTrans(SDnode *pDnode); +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); +int32_t dndInitMsgHandle(SDnode *pDnode); + +// mgmt +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void bmSetMgmtFp(SMgmtWrapper *pWrapper); +void qmSetMgmtFp(SMgmtWrapper *pMgmt); +void smSetMgmtFp(SMgmtWrapper *pWrapper); +void vmSetMgmtFp(SMgmtWrapper *pWrapper); +void mmSetMgmtFp(SMgmtWrapper *pMgmt); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/inc/dndInt.h similarity index 68% rename from source/dnode/mgmt/main/inc/dndInt.h rename to source/dnode/mgmt/inc/dndInt.h index 612d35d513..9477bd06b9 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/inc/dndInt.h @@ -18,40 +18,26 @@ #include "dnd.h" -#include "bm.h" -#include "dm.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" - #ifdef __cplusplus extern "C" { #endif -// dndInt.c -int32_t dndInit(); -void dndCleanup(); -const char *dndStatStr(EDndStatus stat); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); +// dndEnv.h +int32_t dndInit(); +void dndCleanup(); -// dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); - -// dndExec.c +// dndExec.h int32_t dndOpenNode(SMgmtWrapper *pWrapper); void dndCloseNode(SMgmtWrapper *pWrapper); int32_t dndRun(SDnode *pDnode); -// dndObj.c +// dndInt.c SDnode *dndCreate(const SDnodeOpt *pOption); void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); -// dndTransport.c -int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); +// dndMsg.c +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndFile.c TdFilePtr dndCheckRunning(const char *dataDir); diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/inc/mmInt.h similarity index 99% rename from source/dnode/mgmt/mm/inc/mmInt.h rename to source/dnode/mgmt/inc/mmInt.h index 86cba97a33..48c11ae8c4 100644 --- a/source/dnode/mgmt/mm/inc/mmInt.h +++ b/source/dnode/mgmt/inc/mmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_MNODE_INT_H_ #define _TD_DND_MNODE_INT_H_ -#include "mm.h" +#include "dnd.h" #include "mnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/inc/qmInt.h similarity index 99% rename from source/dnode/mgmt/qm/inc/qmInt.h rename to source/dnode/mgmt/inc/qmInt.h index 3e975663d3..b4d57488ef 100644 --- a/source/dnode/mgmt/qm/inc/qmInt.h +++ b/source/dnode/mgmt/inc/qmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_QNODE_INT_H_ #define _TD_DND_QNODE_INT_H_ -#include "qm.h" +#include "dnd.h" #include "qnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/inc/smInt.h similarity index 99% rename from source/dnode/mgmt/sm/inc/smInt.h rename to source/dnode/mgmt/inc/smInt.h index 9290384cab..2c9a43cd98 100644 --- a/source/dnode/mgmt/sm/inc/smInt.h +++ b/source/dnode/mgmt/inc/smInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_SNODE_INT_H_ #define _TD_DND_SNODE_INT_H_ -#include "sm.h" +#include "dnd.h" #include "snode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/inc/vmInt.h similarity index 99% rename from source/dnode/mgmt/vm/inc/vmInt.h rename to source/dnode/mgmt/inc/vmInt.h index 6722fe1d65..9f8eac9e97 100644 --- a/source/dnode/mgmt/vm/inc/vmInt.h +++ b/source/dnode/mgmt/inc/vmInt.h @@ -17,7 +17,7 @@ #define _TD_DND_VNODES_INT_H_ #include "sync.h" -#include "vm.h" +#include "dnd.h" #include "vnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/dndEnv.c similarity index 62% rename from source/dnode/mgmt/main/src/dndStr.c rename to source/dnode/mgmt/main/dndEnv.c index 8a5af68b4f..8bdc2867d2 100644 --- a/source/dnode/mgmt/main/src/dndStr.c +++ b/source/dnode/mgmt/main/dndEnv.c @@ -15,6 +15,48 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#include "wal.h" + +static int8_t once = DND_ENV_INIT; + +int32_t dndInit() { + dDebug("start to init dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + SMonCfg monCfg = {0}; + monCfg.maxLogs = tsMonitorMaxLogs; + monCfg.port = tsMonitorPort; + monCfg.server = tsMonitorFqdn; + monCfg.comp = tsMonitorComp; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + return -1; + } + + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + dDebug("start to cleanup dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + monCleanup(); + walCleanUp(); + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} const char *dndStatStr(EDndStatus status) { switch (status) { @@ -25,7 +67,7 @@ const char *dndStatStr(EDndStatus status) { case DND_STAT_STOPPED: return "stopped"; default: - return "unknown"; + return "UNKNOWN"; } } diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/dndExec.c similarity index 74% rename from source/dnode/mgmt/main/src/dndExec.c rename to source/dnode/mgmt/main/dndExec.c index b37893aa6f..a98c0954fe 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/dndExec.c @@ -45,7 +45,7 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) { } void dndCloseNode(SMgmtWrapper *pWrapper) { - dDebug("node:%s, start to close", pWrapper->name); + dDebug("node:%s, mgmt start to close", pWrapper->name); pWrapper->required = false; taosWLockLatch(&pWrapper->latch); if (pWrapper->deployed) { @@ -62,53 +62,9 @@ void dndCloseNode(SMgmtWrapper *pWrapper) { taosProcCleanup(pWrapper->pProc); pWrapper->pProc = NULL; } - dDebug("node:%s, has been closed", pWrapper->name); + dDebug("node:%s, mgmt has been closed", pWrapper->name); } -static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - SRpcMsg *pRpc = &pMsg->rpcMsg; - pRpc->pCont = pCont; - dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); - - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - int32_t code = (*msgFp)(pWrapper, pMsg); - - if (code != 0) { - dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); - } - - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pCont); - } -} - -static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - pMsg->pCont = pCont; - dTrace("msg:%p, get from parent queue, handle:%p app:%p", pMsg, pMsg->handle, pMsg->ahandle); - - switch (ftype) { - case PROC_REG: - rpcRegisterBrokenLinkArg(pMsg); - break; - case PROC_RELEASE: - rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); - rpcFreeCont(pCont); - break; - case PROC_REQ: - // todo send to dnode - dndSendReqToMnode(pWrapper, pMsg); - default: - dndSendRpcRsp(pWrapper, pMsg); - break; - } - taosMemoryFree(pMsg); -} static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { char tstr[8] = {0}; @@ -132,22 +88,6 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { return 0; } -static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .shm = pWrapper->shm, - .pParent = pWrapper, - .name = pWrapper->name}; - return cfg; -} static int32_t dndRunInSingleProcess(SDnode *pDnode) { dInfo("dnode run in single process"); @@ -180,6 +120,7 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) { while (1) { if (pDnode->event == DND_EVENT_STOP) { dInfo("dnode is about to stop"); + dndSetStatus(pDnode, DND_STAT_STOPPED); break; } taosMsleep(100); @@ -202,7 +143,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (!pWrapper->required) continue; int32_t shmsize = 1024 * 1024 * 2; // size will be a configuration item - if (taosCreateShm(&pWrapper->shm, shmsize) != 0) { + if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) { terrno = TAOS_SYSTEM_ERROR(terrno); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); return -1; @@ -255,6 +196,8 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { while (1) { if (pDnode->event == DND_EVENT_STOP) { dInfo("dnode is about to stop"); + dndSetStatus(pDnode, DND_STAT_STOPPED); + for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; if (!pWrapper->required) continue; @@ -263,15 +206,6 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) { dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId); taosKillProc(pWrapper->procId); - } - } - - for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - if (!pWrapper->required) continue; - if (pDnode->ntype == NODE_MAX) continue; - - if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) { dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId); taosWaitProc(pWrapper->procId); dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId); @@ -331,6 +265,8 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) { } } + dndSetStatus(pDnode, DND_STAT_RUNNING); + if (taosProcRun(pWrapper->pProc) != 0) { dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); return -1; @@ -340,11 +276,14 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) { dndReportStartup(pDnode, "TDengine", "initialized successfully"); while (1) { if (pDnode->event == DND_EVENT_STOP) { - dInfo("dnode is about to stop"); + dInfo("%s is about to stop", pWrapper->name); + dndSetStatus(pDnode, DND_STAT_STOPPED); break; } taosMsleep(100); } + + return 0; } int32_t dndRun(SDnode *pDnode) { diff --git a/source/dnode/mgmt/main/src/dndFile.c b/source/dnode/mgmt/main/dndFile.c similarity index 100% rename from source/dnode/mgmt/main/src/dndFile.c rename to source/dnode/mgmt/main/dndFile.c diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/dndInt.c similarity index 84% rename from source/dnode/mgmt/main/src/dndInt.c rename to source/dnode/mgmt/main/dndInt.c index 602ebc6b3c..80fefdb3ef 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/dndInt.c @@ -82,12 +82,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { } dndSetStatus(pDnode, DND_STAT_INIT); - dmGetMgmtFp(&pDnode->wrappers[DNODE]); - mmGetMgmtFp(&pDnode->wrappers[MNODE]); - vmGetMgmtFp(&pDnode->wrappers[VNODES]); - qmGetMgmtFp(&pDnode->wrappers[QNODE]); - smGetMgmtFp(&pDnode->wrappers[SNODE]); - bmGetMgmtFp(&pDnode->wrappers[BNODE]); + dmSetMgmtFp(&pDnode->wrappers[DNODE]); + mmSetMgmtFp(&pDnode->wrappers[MNODE]); + vmSetMgmtFp(&pDnode->wrappers[VNODES]); + qmSetMgmtFp(&pDnode->wrappers[QNODE]); + smSetMgmtFp(&pDnode->wrappers[SNODE]); + bmSetMgmtFp(&pDnode->wrappers[BNODE]); for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; @@ -95,6 +95,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { pWrapper->path = strdup(path); pWrapper->shm.id = -1; pWrapper->pDnode = pDnode; + pWrapper->ntype = n; if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; @@ -133,14 +134,6 @@ _OVER: void dndClose(SDnode *pDnode) { if (pDnode == NULL) return; - if (dndGetStatus(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down, data:%p", pDnode); - return; - } - - dInfo("start to close dnode, data:%p", pDnode); - dndSetStatus(pDnode, DND_STAT_STOPPED); - for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; dndCloseNode(pWrapper); @@ -151,7 +144,6 @@ void dndClose(SDnode *pDnode) { } void dndHandleEvent(SDnode *pDnode, EDndEvent event) { - dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode); if (event == DND_EVENT_STOP) { pDnode->event = event; } @@ -197,4 +189,25 @@ void dndReleaseWrapper(SMgmtWrapper *pWrapper) { int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); taosRUnLockLatch(&pWrapper->latch); dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); -} \ No newline at end of file +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; + pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} diff --git a/source/dnode/mgmt/main/src/dndMonitor.c b/source/dnode/mgmt/main/dndMonitor.c similarity index 100% rename from source/dnode/mgmt/main/src/dndMonitor.c rename to source/dnode/mgmt/main/dndMonitor.c diff --git a/source/dnode/mgmt/main/dndTransport.c b/source/dnode/mgmt/main/dndTransport.c new file mode 100644 index 0000000000..babc6d1209 --- /dev/null +++ b/source/dnode/mgmt/main/dndTransport.c @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" +#define INTERNAL_SECRET "_pwd" + +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); +} + +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } + + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + pMsg->clientIp = connInfo.clientIp; + pMsg->clientPort = connInfo.clientPort; + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + return 0; +} + +static void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = (*msgFp)(pWrapper, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and put into child queue, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); + } else { + dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + ASSERT(1); + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed in parent process", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + tmsgSendRsp(&rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pMsg->msgType; + bool isReq = msgType & 1u; + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + SMgmtWrapper *pWrapper = pHandle->pWrapper; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dndProcessStartupReq(pDnode, pMsg); + return; + } + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + return; + } + + if (isReq && pMsg->pCont == NULL) { + dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + if (pWrapper == NULL) { + dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + } + + if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = ntohl(pHead->vgId); + if (vgId == QNODE_HANDLE) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MNODE_HANDLE) { + pWrapper = pHandle->pMndWrapper; + } else { + } + } + + dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +} + +static int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcCfp)dndProcessMsg; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = INTERNAL_USER; + rpcInit.ckey = INTERNAL_CKEY; + rpcInit.spi = 1; + rpcInit.parent = pDnode; + + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init dnode rpc client"); + return -1; + } + + dDebug("dnode rpc client is initialized"); + return 0; +} + +static void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dDebug("dnode rpc client is closed"); + } +} + +static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { + SEpSet epSet = {0}; + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); +} + +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, + char *ckey) { + int32_t code = 0; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + if (strcmp(user, INTERNAL_USER) == 0) { + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + } else { + code = -1; + } + + if (code == 0) { + memcpy(secret, pass, TSDB_PASSWORD_LEN); + *spi = 1; + *encrypt = 0; + *ckey = 0; + } + + return code; +} + +static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (dndGetHideUserAuth(pDnode, user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); + + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); + } else { + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +static int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->serverPort; + rpcInit.label = "DND"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = (RpcCfp)dndProcessMsg; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = (RpcAfp)dndRetrieveUserAuthInfo; + rpcInit.parent = pDnode; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init dnode rpc server"); + return -1; + } + + dDebug("dnode rpc server is initialized"); + return 0; +} + +static void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + dDebug("dnode rpc server is closed"); + } +} + +int32_t dndInitTrans(SDnode *pDnode) { + if (dndInitServer(pDnode) != 0) return -1; + if (dndInitClient(pDnode) != 0) return -1; + return 0; +} + +void dndCleanupTrans(SDnode *pDnode) { + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); +} + +int32_t dndInitMsgHandle(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + + for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { + NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + int8_t vgId = pWrapper->msgVgIds[msgIndex]; + if (msgFp == NULL) continue; + + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; + if (vgId == QNODE_HANDLE) { + if (pHandle->pQndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pQndWrapper = pWrapper; + } else if (vgId == MNODE_HANDLE) { + if (pHandle->pMndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pMndWrapper = pWrapper; + } else { + if (pHandle->pWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pWrapper = pWrapper; + } + } + } + + return 0; +} + +static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (pMgmt->clientRpc == NULL) { + terrno = TSDB_CODE_DND_OFFLINE; + return -1; + } + + rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); + return 0; +} + +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { + if (pRsp->code == TSDB_CODE_APP_NOT_READY) { + if (pWrapper->ntype == MNODE) { + dmSendRedirectRsp(pWrapper->pMgmt, pRsp); + return; + } + } + + rpcSendResponse(pRsp); +} + +static int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (dndGetStatus(pWrapper->pDnode) != DND_STAT_RUNNING) { + terrno = TSDB_CODE_DND_OFFLINE; + dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); + return -1; + } + + if (pWrapper->procType != PROC_CHILD) { + return dndSendRpcReq(&pWrapper->pDnode->trans, pEpSet, pReq); + } else { + char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet)); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memcpy(pHead, pReq, sizeof(SRpcMsg)); + memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); + taosProcPutToParentQ(pWrapper->pProc, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen, + PROC_REQ); + taosMemoryFree(pHead); + return 0; + } +} + +static void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { + if (pWrapper->procType != PROC_CHILD) { + dndSendRpcRsp(pWrapper, pRsp); + } else { + taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP); + } +} + +static void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { + if (pWrapper->procType != PROC_CHILD) { + rpcRegisterBrokenLinkArg(pMsg); + } else { + taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGIST); + } +} + +static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type) { + if (pWrapper->procType != PROC_CHILD) { + rpcReleaseHandle(handle, type); + } else { + SRpcMsg msg = {.handle = handle, .code = type}; + taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE); + } +} + +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { + SMsgCb msgCb = { + .pWrapper = pWrapper, + .sendReqFp = dndSendReq, + .sendRspFp = dndSendRsp, + .registerBrokenLinkArgFp = dndRegisterBrokenLinkArg, + .releaseHandleFp = dndReleaseHandle, + }; + return msgCb; +} + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + pMsg->pCont = pCont; + dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); + + switch (ftype) { + case PROC_REGIST: + rpcRegisterBrokenLinkArg(pMsg); + break; + case PROC_RELEASE: + rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); + rpcFreeCont(pCont); + break; + case PROC_REQ: + dndSendRpcReq(&pWrapper->pDnode->trans, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + break; + case PROC_RSP: + dndSendRpcRsp(pWrapper, pMsg); + break; + default: + break; + } + taosMemoryFree(pMsg); +} + +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { + SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->shm, + .pParent = pWrapper, + .name = pWrapper->name}; + return cfg; +} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c deleted file mode 100644 index 8792147822..0000000000 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" -#include "wal.h" - -static int8_t once = DND_ENV_INIT; - -int32_t dndInit() { - dDebug("start to init dnode env"); - if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { - terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); - return -1; - } - - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - - SMonCfg monCfg = {0}; - monCfg.maxLogs = tsMonitorMaxLogs; - monCfg.port = tsMonitorPort; - monCfg.server = tsMonitorFqdn; - monCfg.comp = tsMonitorComp; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - return -1; - } - - dInfo("dnode env is initialized"); - return 0; -} - -void dndCleanup() { - dDebug("start to cleanup dnode env"); - if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); - return; - } - - monCleanup(); - walCleanUp(); - taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); -} - -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { - pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; - pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; -} - -EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } - -void dndSetStatus(SDnode *pDnode, EDndStatus status) { - if (pDnode->status != status) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); - pDnode->status = status; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c deleted file mode 100644 index 2ab1e401c7..0000000000 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); - dndReleaseWrapper(pWrapper); - } -} - -static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - if (msgFp == NULL) { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - return msgFp; -} - -static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { - SRpcConnInfo connInfo = {0}; - if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); - return -1; - } - - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - pMsg->clientIp = connInfo.clientIp; - pMsg->clientPort = connInfo.clientPort; - memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; -} - -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - int32_t code = -1; - SNodeMsg *pMsg = NULL; - NodeMsgFp msgFp = NULL; - - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - - if (dndMarkWrapper(pWrapper) != 0) goto _OVER; - if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; - if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - - if (pWrapper->procType == PROC_SINGLE) { - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); - code = (*msgFp)(pWrapper, pMsg); - } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, - pRpc->ahandle, pMsg->user); - code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); - } else { - dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - ASSERT(1); - } - -_OVER: - if (code == 0) { - if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed in parent process", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { - dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); - } - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - - dndReleaseWrapper(pWrapper); -} - -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper != NULL) { - dndReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - dError("failed to create node since %s", terrstr()); - return -1; - } - - pWrapper = &pDnode->wrappers[ntype]; - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been opened", pWrapper->name); - pWrapper->deployed = true; - } - - return code; -} - -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - dError("failed to drop node since %s", terrstr()); - return -1; - } - - taosWLockLatch(&pWrapper->latch); - pWrapper->deployed = false; - - int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); - if (code != 0) { - pWrapper->deployed = true; - dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); - } else { - pWrapper->deployed = false; - dDebug("node:%s, has been dropped", pWrapper->name); - } - - taosWUnLockLatch(&pWrapper->latch); - dndReleaseWrapper(pWrapper); - return code; -} - -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { - switch (pMsg->rpcMsg.msgType) { - case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); - case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - return -1; - } -} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c deleted file mode 100644 index 07ea0309a8..0000000000 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" - -static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMsgHead *pHead = pMsg->pCont; - int32_t vgId = htonl(pHead->vgId); - - SMgmtWrapper *pWrapper = pHandle->pWrapper; - if (vgId == QND_VGID) { - pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { - pWrapper = pHandle->pMndWrapper; - } - - dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, - pMsg->handle, pMsg->ahandle, vgId); - dndProcessRpcMsg(pWrapper, pMsg, pEpSet); -} - -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pRsp->msgType; - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), - pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); - dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); - } - } else { - dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - } -} - -int32_t dndInitClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; - rpcInit.ckey = INTERNAL_CKEY; - rpcInit.spi = 1; - rpcInit.parent = pDnode; - - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - rpcInit.secret = pass; - - pMgmt->clientRpc = rpcOpen(&rpcInit); - if (pMgmt->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; - } - - dDebug("dnode rpc client is initialized"); - return 0; -} - -void dndCleanupClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - if (pMgmt->clientRpc) { - rpcClose(pMgmt->clientRpc); - pMgmt->clientRpc = NULL; - dDebug("dnode rpc client is closed"); - } -} - -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pReq->msgType; - - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, - pReq->handle, pReq->ahandle); - dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); - } - } else { - dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); - dndReleaseWrapper(pWrapper); - } - - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); -} - -static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = 0; - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - - if (strcmp(user, INTERNAL_USER) == 0) { - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - } else { - code = -1; - } - - if (code == 0) { - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - } - - return code; -} - -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { - dndReleaseWrapper(pWrapper); - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - dndReleaseWrapper(pWrapper); - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -int32_t dndInitServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->serverPort; - rpcInit.label = "DND"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; - rpcInit.parent = pDnode; - - pMgmt->serverRpc = rpcOpen(&rpcInit); - if (pMgmt->serverRpc == NULL) { - dError("failed to init dnode rpc server"); - return -1; - } - - dDebug("dnode rpc server is initialized"); - return 0; -} - -void dndCleanupServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - if (pMgmt->serverRpc) { - rpcClose(pMgmt->serverRpc); - pMgmt->serverRpc = NULL; - dDebug("dnode rpc server is closed"); - } -} - -int32_t dndInitMsgHandle(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->trans; - - for (ENodeType n = 0; n < NODE_MAX; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - - for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { - NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; - int32_t vgId = pWrapper->msgVgIds[msgIndex]; - if (msgFp == NULL) continue; - - // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); - - SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (vgId == QND_VGID) { - if (pHandle->pQndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pQndWrapper = pWrapper; - } else if (vgId == MND_VGID) { - if (pHandle->pMndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pMndWrapper = pWrapper; - } else { - if (pHandle->pWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pWrapper = pWrapper; - } - } - } - - return 0; -} - -static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) { - if (pMgmt->clientRpc == NULL) { - terrno = TSDB_CODE_DND_OFFLINE; - return -1; - } - - rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); - return 0; -} - -int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { - if (pWrapper->procType != PROC_CHILD) { - SDnode *pDnode = pWrapper->pDnode; - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - terrno = TSDB_CODE_DND_OFFLINE; - dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); - return -1; - } - return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); - } else { - while (taosProcPutToParentQ(pWrapper->pProc, pReq, sizeof(SRpcMsg), pReq->pCont, pReq->contLen, PROC_REQ) != 0) { - taosMsleep(1); - } - } -} - -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - STransMgmt *pTrans = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper2 = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper2 != NULL) { - dmGetMnodeEpSet(pWrapper2->pMgmt, &epSet); - dndReleaseWrapper(pWrapper2); - } - return dndSendRpcReq(pTrans, &epSet, pReq); -} - -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { - if (pRsp->code == TSDB_CODE_APP_NOT_READY) { - SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - if (pDnodeWrapper != NULL) { - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); - dndReleaseWrapper(pDnodeWrapper); - } else { - rpcSendResponse(pRsp); - } - } else { - rpcSendResponse(pRsp); - } -} - -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { - if (pWrapper->procType != PROC_CHILD) { - dndSendRpcRsp(pWrapper, pRsp); - } else { - while (taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP) != 0) { - taosMsleep(1); - } - } -} - -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { - if (pWrapper->procType != PROC_CHILD) { - rpcRegisterBrokenLinkArg(pMsg); - } else { - while (taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REG) != 0) { - taosMsleep(1); - } - } -} - -static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type) { - if (pWrapper->procType != PROC_CHILD) { - rpcReleaseHandle(handle, type); - } else { - SRpcMsg msg = {.handle = handle, .code = type}; - while (taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE) != 0) { - taosMsleep(1); - } - } -} - -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { - SMsgCb msgCb = { - .pWrapper = pWrapper, - .registerBrokenLinkArgFp = dndRegisterBrokenLinkArg, - .releaseHandleFp = dndReleaseHandle, - .sendMnodeReqFp = dndSendReqToMnode, - .sendReqFp = dndSendReqToDnode, - .sendRspFp = dndSendRsp, - }; - return msgCb; -} \ No newline at end of file diff --git a/source/dnode/mgmt/mm/inc/mm.h b/source/dnode/mgmt/mm/inc/mm.h deleted file mode 100644 index 6ed6c42d93..0000000000 --- a/source/dnode/mgmt/mm/inc/mm.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void mmGetMgmtFp(SMgmtWrapper *pMgmt); - -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mm/src/mmFile.c b/source/dnode/mgmt/mm/mmFile.c similarity index 98% rename from source/dnode/mgmt/mm/src/mmFile.c rename to source/dnode/mgmt/mm/mmFile.c index e5cc0ce087..76aba771cb 100644 --- a/source/dnode/mgmt/mm/src/mmFile.c +++ b/source/dnode/mgmt/mm/mmFile.c @@ -111,7 +111,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno);; + terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; } @@ -145,7 +145,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); if (taosRenameFile(file, realfile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno);; + terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to rename %s since %s", file, terrstr()); return -1; } diff --git a/source/dnode/mgmt/mm/src/mmMsg.c b/source/dnode/mgmt/mm/mmHandle.c similarity index 78% rename from source/dnode/mgmt/mm/src/mmMsg.c rename to source/dnode/mgmt/mm/mmHandle.c index 6afcd249b3..666f9daa91 100644 --- a/source/dnode/mgmt/mm/src/mmMsg.c +++ b/source/dnode/mgmt/mm/mmHandle.c @@ -75,91 +75,91 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); } diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/mmInt.c similarity index 87% rename from source/dnode/mgmt/mm/src/mmInt.c rename to source/dnode/mgmt/mm/mmInt.c index 3d15c9b9ae..97fdafe139 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/mmInt.c @@ -39,20 +39,11 @@ static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { } static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - SDnode *pDnode = pMgmt->pDnode; - pOption->dnodeId = pDnode->dnodeId; - pOption->clusterId = pDnode->clusterId; - - SMsgCb msgCb = {0}; - msgCb.pWrapper = pMgmt->pWrapper; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue; msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue; msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; msgCb.queueFps[SYNC_QUEUE] = mmPutMsgToWriteQueue; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; pOption->msgCb = msgCb; } @@ -66,6 +57,7 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pReplica->id = 1; pReplica->port = pDnode->serverPort; tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN); + pOption->deploy = true; pMgmt->selfIndex = pOption->selfIndex; pMgmt->replica = pOption->replica; @@ -77,6 +69,7 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pOption->selfIndex = pMgmt->selfIndex; pOption->replica = pMgmt->replica; memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); + pOption->deploy = false; } static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { @@ -89,7 +82,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre pReplica->id = pCreate->replicas[i].id; pReplica->port = pCreate->replicas[i].port; memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - if (pReplica->id == pOption->dnodeId) { + if (pReplica->id == pMgmt->pDnode->dnodeId) { pOption->selfIndex = i; } } @@ -98,6 +91,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre dError("failed to build mnode options since %s", terrstr()); return -1; } + pOption->deploy = true; pMgmt->selfIndex = pOption->selfIndex; pMgmt->replica = pOption->replica; @@ -225,9 +219,7 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { return code; } -static int32_t mmOpen(SMgmtWrapper *pWrapper) { - return mmOpenFromMsg(pWrapper, NULL); -} +static int32_t mmOpen(SMgmtWrapper *pWrapper) { return mmOpenFromMsg(pWrapper, NULL); } static int32_t mmStart(SMgmtWrapper *pWrapper) { dDebug("mnode-mgmt start to run"); @@ -235,7 +227,7 @@ static int32_t mmStart(SMgmtWrapper *pWrapper) { return mndStart(pMgmt->pMnode); } -void mmGetMgmtFp(SMgmtWrapper *pWrapper) { +void mmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = mmOpen; mgmtFp.closeFp = mmClose; @@ -249,16 +241,8 @@ void mmGetMgmtFp(SMgmtWrapper *pWrapper) { pWrapper->fp = mgmtFp; } -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - - int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { + SMonGrantInfo *pGrantInfo) { SMnodeMgmt *pMgmt = pWrapper->pMgmt; return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); } diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/mmWorker.c similarity index 98% rename from source/dnode/mgmt/mm/src/mmWorker.c rename to source/dnode/mgmt/mm/mmWorker.c index c4aafe05e4..cde4eb853a 100644 --- a/source/dnode/mgmt/mm/src/mmWorker.c +++ b/source/dnode/mgmt/mm/mmWorker.c @@ -37,7 +37,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); } SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } @@ -60,7 +60,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { if (pRpc->handle != NULL && code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } diff --git a/source/dnode/mgmt/qm/inc/qm.h b/source/dnode/mgmt/qm/inc/qm.h deleted file mode 100644 index e28ea3e948..0000000000 --- a/source/dnode/mgmt/qm/inc/qm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void qmGetMgmtFp(SMgmtWrapper *pMgmt); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qm/src/qmMsg.c b/source/dnode/mgmt/qm/qmHandle.c similarity index 93% rename from source/dnode/mgmt/qm/src/qmMsg.c rename to source/dnode/mgmt/qm/qmHandle.c index da5ba6472a..14ac08af05 100644 --- a/source/dnode/mgmt/qm/src/qmMsg.c +++ b/source/dnode/mgmt/qm/qmHandle.c @@ -56,14 +56,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void qmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QNODE_HANDLE); } diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/qmInt.c similarity index 92% rename from source/dnode/mgmt/qm/src/qmInt.c rename to source/dnode/mgmt/qm/qmInt.c index 11c80a2904..53c43bf841 100644 --- a/source/dnode/mgmt/qm/src/qmInt.c +++ b/source/dnode/mgmt/qm/qmInt.c @@ -19,15 +19,10 @@ static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { - SMsgCb msgCb = {0}; - msgCb.pWrapper = pMgmt->pWrapper; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue; msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue; msgCb.qsizeFp = qmGetQueueSize; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; pOption->msgCb = msgCb; } @@ -117,7 +112,7 @@ int32_t qmOpen(SMgmtWrapper *pWrapper) { return code; } -void qmGetMgmtFp(SMgmtWrapper *pWrapper) { +void qmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = qmOpen; mgmtFp.closeFp = qmClose; diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c similarity index 99% rename from source/dnode/mgmt/qm/src/qmWorker.c rename to source/dnode/mgmt/qm/qmWorker.c index 14efb311b1..5ceb9dbf88 100644 --- a/source/dnode/mgmt/qm/src/qmWorker.c +++ b/source/dnode/mgmt/qm/qmWorker.c @@ -18,7 +18,7 @@ static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/sm/inc/sm.h b/source/dnode/mgmt/sm/inc/sm.h deleted file mode 100644 index 82a52e5d1f..0000000000 --- a/source/dnode/mgmt/sm/inc/sm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void smGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/sm/src/smMsg.c b/source/dnode/mgmt/sm/smHandle.c similarity index 98% rename from source/dnode/mgmt/sm/src/smMsg.c rename to source/dnode/mgmt/sm/smHandle.c index c522ef7fc3..dc5ee1155a 100644 --- a/source/dnode/mgmt/sm/src/smMsg.c +++ b/source/dnode/mgmt/sm/smHandle.c @@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void smInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/smInt.c similarity index 91% rename from source/dnode/mgmt/sm/src/smInt.c rename to source/dnode/mgmt/sm/smInt.c index a639fc76bb..a279655609 100644 --- a/source/dnode/mgmt/sm/src/smInt.c +++ b/source/dnode/mgmt/sm/smInt.c @@ -19,12 +19,7 @@ static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { - SMsgCb msgCb = {0}; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); pOption->msgCb = msgCb; } @@ -114,7 +109,7 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { return code; } -void smGetMgmtFp(SMgmtWrapper *pWrapper) { +void smSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = smOpen; mgmtFp.closeFp = smClose; diff --git a/source/dnode/mgmt/sm/src/smWorker.c b/source/dnode/mgmt/sm/smWorker.c similarity index 100% rename from source/dnode/mgmt/sm/src/smWorker.c rename to source/dnode/mgmt/sm/smWorker.c diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index e2de3fc33a..40e7472d42 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -70,46 +70,6 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); } } - - { - SCreateVnodeReq createReq = {0}; - createReq.vgId = 2; - createReq.dnodeId = 3; - strcpy(createReq.db, "1.d1"); - createReq.dbUid = 9527; - createReq.vgVersion = 1; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; - createReq.daysPerFile = 10; - createReq.daysToKeep0 = 3650; - createReq.daysToKeep1 = 3650; - createReq.daysToKeep2 = 3650; - createReq.minRows = 100; - createReq.minRows = 4096; - createReq.commitTime = 3600; - createReq.fsyncPeriod = 3000; - createReq.walLevel = 1; - createReq.precision = 0; - createReq.compression = 2; - createReq.replica = 1; - createReq.quorum = 1; - createReq.update = 0; - createReq.cacheLastRow = 0; - createReq.selfIndex = 0; - for (int r = 0; r < createReq.replica; ++r) { - SReplica* pReplica = &createReq.replicas[r]; - pReplica->id = 1; - pReplica->port = 9527; - } - - int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateVnodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION); - } } TEST_F(DndTestVnode, 02_Alter_Vnode) { @@ -164,37 +124,37 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.keep = 0; req.type = TD_SUPER_TABLE; - SSchema schemas[5] = {0}; + SSchemaEx schemas[2] = {0}; { - SSchema* pSchema = &schemas[0]; + SSchemaEx* pSchema = &schemas[0]; pSchema->bytes = htonl(8); pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema->name, "ts"); } { - SSchema* pSchema = &schemas[1]; + SSchemaEx* pSchema = &schemas[1]; pSchema->bytes = htonl(4); pSchema->type = TSDB_DATA_TYPE_INT; strcpy(pSchema->name, "col1"); } - + SSchema tagSchemas[3] = {0}; { - SSchema* pSchema = &schemas[2]; + SSchema* pSchema = &tagSchemas[0]; pSchema->bytes = htonl(2); pSchema->type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema->name, "tag1"); } { - SSchema* pSchema = &schemas[3]; + SSchema* pSchema = &tagSchemas[1]; pSchema->bytes = htonl(8); pSchema->type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema->name, "tag2"); } { - SSchema* pSchema = &schemas[4]; + SSchema* pSchema = &tagSchemas[2]; pSchema->bytes = htonl(16); pSchema->type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema->name, "tag3"); @@ -204,7 +164,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.stbCfg.nCols = 2; req.stbCfg.pSchema = &schemas[0]; req.stbCfg.nTagCols = 3; - req.stbCfg.pTagSchema = &schemas[2]; + req.stbCfg.pTagSchema = &tagSchemas[0]; int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); @@ -236,37 +196,37 @@ TEST_F(DndTestVnode, 04_Alter_Stb) { req.keep = 0; req.type = TD_SUPER_TABLE; - SSchema schemas[5] = {0}; + SSchemaEx schemas[2] = {0}; { - SSchema* pSchema = &schemas[0]; + SSchemaEx* pSchema = &schemas[0]; pSchema->bytes = htonl(8); pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema->name, "ts"); } { - SSchema* pSchema = &schemas[1]; + SSchemaEx* pSchema = &schemas[1]; pSchema->bytes = htonl(4); pSchema->type = TSDB_DATA_TYPE_INT; strcpy(pSchema->name, "col1"); } - + SSchema tagSchemas[3] = {0}; { - SSchema* pSchema = &schemas[2]; + SSchema* pSchema = &tagSchemas[0]; pSchema->bytes = htonl(2); pSchema->type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema->name, "_tag1"); } { - SSchema* pSchema = &schemas[3]; + SSchema* pSchema = &tagSchemas[1]; pSchema->bytes = htonl(8); pSchema->type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema->name, "_tag2"); } { - SSchema* pSchema = &schemas[4]; + SSchema* pSchema = &tagSchemas[2]; pSchema->bytes = htonl(16); pSchema->type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema->name, "_tag3"); @@ -276,7 +236,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) { req.stbCfg.nCols = 2; req.stbCfg.pSchema = &schemas[0]; req.stbCfg.nTagCols = 3; - req.stbCfg.pTagSchema = &schemas[2]; + req.stbCfg.pTagSchema = &tagSchemas[0]; int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); diff --git a/source/dnode/mgmt/vm/inc/vm.h b/source/dnode/mgmt/vm/inc/vm.h deleted file mode 100644 index 60d9cfc3a1..0000000000 --- a/source/dnode/mgmt/vm/inc/vm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -void vmGetMgmtFp(SMgmtWrapper *pWrapper); - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vm/src/vmFile.c b/source/dnode/mgmt/vm/vmFile.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmFile.c rename to source/dnode/mgmt/vm/vmFile.c diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/vmHandle.c similarity index 86% rename from source/dnode/mgmt/vm/src/vmMsg.c rename to source/dnode/mgmt/vm/vmHandle.c index f00bb89354..6a51c9a77f 100644 --- a/source/dnode/mgmt/vm/src/vmMsg.c +++ b/source/dnode/mgmt/vm/vmHandle.c @@ -68,12 +68,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SWrapperCfg wrapperCfg = {0}; vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); - if (createReq.dnodeId != pMgmt->pDnode->dnodeId) { - terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; - dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); if (pVnode != NULL) { dDebug("vgId:%d, already exist", createReq.vgId); @@ -82,16 +76,12 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return -1; } - SMsgCb msgCb = {0}; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); msgCb.pWrapper = pMgmt->pWrapper; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.qsizeFp = vmGetQueueSize; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; vnodeCfg.msgCb = msgCb; vnodeCfg.pTfs = pMgmt->pTfs; @@ -245,49 +235,49 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/vmInt.c similarity index 98% rename from source/dnode/mgmt/vm/src/vmInt.c rename to source/dnode/mgmt/vm/vmInt.c index b52c6253dc..d78a567b69 100644 --- a/source/dnode/mgmt/vm/src/vmInt.c +++ b/source/dnode/mgmt/vm/vmInt.c @@ -128,16 +128,12 @@ static void *vmOpenVnodeFunc(void *param) { pMgmt->state.openVnodes, pMgmt->state.totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SMsgCb msgCb = {0}; + SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper); msgCb.pWrapper = pMgmt->pWrapper; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.qsizeFp = vmGetQueueSize; - msgCb.sendReqFp = dndSendReqToDnode; - msgCb.sendMnodeReqFp = dndSendReqToMnode; - msgCb.sendRspFp = dndSendRsp; - msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg; SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { @@ -337,7 +333,7 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { return 0; } -void vmGetMgmtFp(SMgmtWrapper *pWrapper) { +void vmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = vmInit; mgmtFp.closeFp = vmCleanup; diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/vmWorker.c similarity index 99% rename from source/dnode/mgmt/vm/src/vmWorker.c rename to source/dnode/mgmt/vm/vmWorker.c index 4be6311cf8..c228e6e7dd 100644 --- a/source/dnode/mgmt/vm/src/vmWorker.c +++ b/source/dnode/mgmt/vm/vmWorker.c @@ -18,7 +18,7 @@ static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { @@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; - dndSendRsp(pVnode->pWrapper, pRsp); + tmsgSendRsp(pRsp); taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 0206695b88..5b7bac4486 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -25,6 +25,7 @@ extern "C" { int32_t mndInitCluster(SMnode *pMnode); void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); +int64_t mndGetClusterId(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index f89e9d8fe0..1cb0c78aa5 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -100,7 +100,6 @@ typedef struct { } SGrantInfo; typedef struct SMnode { - int32_t dnodeId; int64_t clusterId; int8_t replica; int8_t selfIndex; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index dde7e1fe8f..94e1efde61 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -17,7 +17,7 @@ #include "mndCluster.h" #include "mndShow.h" -#define TSDB_CLUSTER_VER_NUMBE 1 +#define TSDB_CLUSTER_VER_NUMBE 1 #define TSDB_CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -61,6 +61,23 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { return 0; } +int64_t mndGetClusterId(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int64_t clusterId = -1; + + while (1) { + SClusterObj *pCluster = NULL; + pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); + if (pIter == NULL) break; + + clusterId = pCluster->id; + sdbRelease(pSdb, pCluster); + } + + return clusterId; +} + static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 6c38d8626c..cad89399a3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -433,12 +433,6 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { pHeartbeat->connId = htonl(pHeartbeat->connId); pHeartbeat->pid = htonl(pHeartbeat->pid); - SRpcConnInfo info = {0}; - if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) { - mError("user:%s, connId:%d failed to process hb since %s", pReq->user, pHeartbeat->connId, terrstr()); - return -1; - } - SConnObj *pConn = mndAcquireConn(pMnode, pHeartbeat->connId); if (pConn == NULL) { pConn = mndCreateConn(pMnode, &info, pHeartbeat->pid, pHeartbeat->app, 0); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 4b19a26bc4..85718e2037 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -423,7 +423,7 @@ static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) { DROP_QNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } mndReleaseQnode(pMnode, pObj); diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 40b4f60bd4..d184e354c4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -52,7 +52,7 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) { } int32_t mndInitQuery(SMnode *pMnode) { - if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { + if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 5e0d9fae9a..4f24c6f7eb 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -433,7 +433,7 @@ static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) { DROP_SNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } mndReleaseSnode(pMnode, pObj); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 196767462e..85e7ade462 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -333,6 +333,15 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { return mndAcquireDb(pMnode, db); } +static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) { + if (*(col_id_t *)colId < ((SSchemaEx *)pSchema)->colId) { + return -1; + } else if (*(col_id_t *)colId > ((SSchemaEx *)pSchema)->colId) { + return 1; + } + return 0; +} + static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -345,16 +354,91 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; + req.rollup = pStb->aggregationMethod > -1 ? 1 : 0; req.type = TD_SUPER_TABLE; req.stbCfg.suid = pStb->uid; req.stbCfg.nCols = pStb->numOfColumns; - req.stbCfg.pSchema = pStb->pColumns; req.stbCfg.nTagCols = pStb->numOfTags; req.stbCfg.pTagSchema = pStb->pTags; + req.stbCfg.nBSmaCols = pStb->numOfSmas; + req.stbCfg.pSchema = (SSchemaEx *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaEx)); + if (req.stbCfg.pSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + int bSmaStat = 0; // no column has bsma + if (pStb->numOfSmas == pStb->numOfColumns) { // assume pColumns > 0 + bSmaStat = 1; // all columns have bsma + } else if (pStb->numOfSmas != 0) { + bSmaStat = 2; // partial columns have bsma + TASSERT(pStb->pSmas != NULL); // TODO: remove the assert + } + + for (int32_t i = 0; i < req.stbCfg.nCols; ++i) { + SSchemaEx *pSchemaEx = req.stbCfg.pSchema + i; + SSchema *pSchema = pStb->pColumns + i; + pSchemaEx->type = pSchema->type; + pSchemaEx->sma = (bSmaStat == 1) ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE; + pSchemaEx->colId = pSchema->colId; + pSchemaEx->bytes = pSchema->bytes; + memcpy(pSchemaEx->name, pSchema->name, TSDB_COL_NAME_LEN); + } + + if (bSmaStat == 2) { + if (pStb->pSmas == NULL) { + mError("stb:%s, sma options is empty", pStb->name); + taosMemoryFreeClear(req.stbCfg.pSchema); + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return NULL; + } + for (int32_t i = 0; i < pStb->numOfSmas; ++i) { + SSchema *pSmaSchema = pStb->pSmas + i; + SSchemaEx *pColSchema = taosbsearch(&pSmaSchema->colId, req.stbCfg.pSchema, req.stbCfg.nCols, sizeof(SSchemaEx), + schemaExColIdCompare, TD_EQ); + if (pColSchema == NULL) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + taosMemoryFreeClear(req.stbCfg.pSchema); + mError("stb:%s, sma col:%s not found in columns", pStb->name, pSmaSchema->name); + return NULL; + } + pColSchema->sma = TSDB_BSMA_TYPE_LATEST; + } + } + + SRSmaParam *pRSmaParam = NULL; + if (req.rollup) { + pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); + if (pRSmaParam == NULL) { + taosMemoryFreeClear(req.stbCfg.pSchema); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pRSmaParam->xFilesFactor = pStb->xFilesFactor; + pRSmaParam->delay = pStb->delay; + pRSmaParam->nFuncIds = 1; // only 1 aggregation method supported currently + pRSmaParam->pFuncIds = (func_id_t *)taosMemoryCalloc(pRSmaParam->nFuncIds, sizeof(func_id_t)); + if (pRSmaParam->pFuncIds == NULL) { + taosMemoryFreeClear(req.stbCfg.pRSmaParam); + taosMemoryFreeClear(req.stbCfg.pSchema); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) { + *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod; + } + req.stbCfg.pRSmaParam = pRSmaParam; + } + + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { + if (pRSmaParam) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam); + } + taosMemoryFreeClear(req.stbCfg.pSchema); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -366,6 +450,11 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt tSerializeSVCreateTbReq(&pBuf, &req); *pContLen = contLen; + if (pRSmaParam) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam); + } + taosMemoryFreeClear(req.stbCfg.pSchema); return pHead; } @@ -498,7 +587,6 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -559,9 +647,9 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj } static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { - for (int32_t col = 0; col < pStb->numOfColumns; col++) { + for (int32_t col = 0; col < pStb->numOfColumns; ++col) { SSchema *pSchema = &pStb->pColumns[col]; - if (strcasecmp(pStb->pColumns[col].name, colName) == 0) { + if (strncasecmp(pSchema->name, colName, TSDB_COL_NAME_LEN) == 0) { return pSchema; } } @@ -578,6 +666,9 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.dbUid = pDb->uid; stbObj.version = 1; stbObj.nextColId = 1; + stbObj.xFilesFactor = pCreate->xFilesFactor; + stbObj.aggregationMethod = pCreate->aggregationMethod; + stbObj.delay = pCreate->delay; stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; @@ -625,7 +716,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre SSchema *pSchema = &stbObj.pSmas[i]; SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name); if (pColSchema == NULL) { - mError("stb:%s, sma:%s not found in columns", stbObj.name, pSchema->name); + mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name); terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -1061,7 +1152,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 67b7d6dd45..5c3dd778e1 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -187,7 +187,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle return 0; } -static int32_t mndInitSteps(SMnode *pMnode) { +static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; @@ -210,7 +210,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; - if (pMnode->clusterId <= 0) { + if (deploy) { if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1; } else { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; @@ -263,23 +263,15 @@ static int32_t mndExecSteps(SMnode *pMnode) { } } + pMnode->clusterId = mndGetClusterId(pMnode); return 0; } -static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { - pMnode->dnodeId = pOption->dnodeId; - pMnode->clusterId = pOption->clusterId; +static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->replica = pOption->replica; pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); pMnode->msgCb = pOption->msgCb; - - if (pMnode->dnodeId < 0 || pMnode->clusterId < 0) { - terrno = TSDB_CODE_MND_INVALID_OPTIONS; - return -1; - } - - return 0; } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -294,6 +286,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { char timestr[24] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + mndSetOptions(pMnode, pOption); pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { @@ -312,16 +305,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } - code = mndSetOptions(pMnode, pOption); - if (code != 0) { - code = terrno; - mError("failed to open mnode since %s", terrstr()); - mndClose(pMnode); - terrno = code; - return NULL; - } - - code = mndInitSteps(pMnode); + code = mndInitSteps(pMnode, pOption->deploy); if (code != 0) { code = terrno; mError("failed to open mnode since %s", terrstr()); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 0d3fcffe7d..1e4a7e5a93 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -171,6 +171,8 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo); + bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); /** diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index b91c6cd9e3..9485c291ae 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -70,9 +70,12 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW); +static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx); static void metaDBWLock(SMetaDB *pDB); static void metaDBRLock(SMetaDB *pDB); static void metaDBULock(SMetaDB *pDB); +static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -155,13 +158,13 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - tb_uid_t uid; - char buf[512]; - char buf1[512]; - void *pBuf; - DBT key1, value1; - DBT key2, value2; - SSchema *pSchema = NULL; + tb_uid_t uid; + char buf[512]; + char buf1[512]; + void *pBuf; + DBT key1, value1; + DBT key2, value2; + SSchemaEx *pSchema = NULL; if (pTbCfg->type == META_SUPER_TABLE) { uid = pTbCfg->stbCfg.suid; @@ -204,8 +207,8 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { key2.data = &schemaKey; key2.size = sizeof(schemaKey); - SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema}; - metaEncodeSchema(&pBuf, &sw); + SSchemaWrapper sw = {.nCols = ncols, .pSchemaEx = pSchema}; + metaEncodeSchemaEx(&pBuf, &sw); value2.data = buf1; value2.size = POINTER_DISTANCE(pBuf, buf1); @@ -298,6 +301,8 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + + int8_t dummy; for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); @@ -309,6 +314,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { return buf; } +static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) { + int tlen = 0; + SSchemaEx *pSchema; + + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchemaEx + i; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI8(buf, pSchema->sma); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeString(buf, pSchema->name); + } + + return tlen; +} + +static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) { + buf = taosDecodeFixedU32(buf, &pSW->nCols); + if (isGetEx) { + pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + SSchemaEx *pSchema = pSW->pSchemaEx + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI8(buf, &pSchema->sma); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + } else { + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + SSchema *pSchema = pSW->pSchema + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosSkipFixedLen(buf, sizeof(int8_t)); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + } + + return buf; +} + static SMetaDB *metaNewDB() { SMetaDB *pDB = NULL; pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB)); @@ -652,12 +701,16 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { + return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false); +} + +static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) { uint32_t nCols; SSchemaWrapper *pSW = NULL; SMetaDB *pDB = pMeta->pDB; int ret; void *pBuf; - SSchema *pSchema; + // SSchema *pSchema; SSchemaKey schemaKey = {uid, sver, 0}; DBT key = {0}; DBT value = {0}; @@ -678,7 +731,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // Decode the schema pBuf = value.data; pSW = taosMemoryMalloc(sizeof(*pSW)); - metaDecodeSchema(pBuf, pSW); + metaDecodeSchemaEx(pBuf, pSW, isGetEx); return pSW; } @@ -755,7 +808,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchemaBuilder sb; STSchema *pTSchema = NULL; - SSchema *pSchema; + SSchemaEx *pSchema; SSchemaWrapper *pSW; STbCfg *pTbCfg; tb_uid_t quid; @@ -767,16 +820,16 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { quid = uid; } - pSW = metaGetTableSchema(pMeta, quid, sver, true); + pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true); if (pSW == NULL) { return NULL; } // Rebuild a schema tdInitTSchemaBuilder(&sb, 0); - for (int32_t i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes); + for (int32_t i = 0; i < pSW->nCols; ++i) { + pSchema = pSW->pSchemaEx + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes); } pTSchema = tdGetSchemaFromBuilder(&sb); tdDestroyTSchemaBuilder(&sb); diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index ee928ef039..8e82cf1abc 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -46,6 +46,10 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW); +static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx); + +static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx); static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { tb_uid_t uid1, uid2; @@ -228,7 +232,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema; } pVal = pBuf = buf; - metaEncodeSchema(&pBuf, &schemaWrapper); + metaEncodeSchemaEx(&pBuf, &schemaWrapper); vLen = POINTER_DISTANCE(pBuf, buf); ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); if (ret < 0) { @@ -345,6 +349,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { + return *metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false); +} + +static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) { void *pKey; void *pVal; int kLen; @@ -368,7 +376,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // decode pBuf = pVal; pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper)); - metaDecodeSchema(pBuf, pSchemaWrapper); + metaDecodeSchemaEx(pBuf, pSchemaWrapper, isGetEx); TDB_FREE(pVal); @@ -379,7 +387,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tb_uid_t quid; SSchemaWrapper *pSW; STSchemaBuilder sb; - SSchema *pSchema; + SSchemaEx *pSchema; STSchema *pTSchema; STbCfg *pTbCfg; @@ -390,15 +398,15 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { quid = uid; } - pSW = metaGetTableSchema(pMeta, quid, sver, true); + pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true); if (pSW == NULL) { return NULL; } tdInitTSchemaBuilder(&sb, 0); for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes); + pSchema = pSW->pSchemaEx + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes); } pTSchema = tdGetSchemaFromBuilder(&sb); tdDestroyTSchemaBuilder(&sb); @@ -605,6 +613,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { return buf; } +static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) { + int tlen = 0; + SSchemaEx *pSchema; + + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int i = 0; i < pSW->nCols; ++i) { + pSchema = pSW->pSchemaEx + i; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI8(buf, pSchema->sma); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeString(buf, pSchema->name); + } + + return tlen; +} + +static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) { + buf = taosDecodeFixedU32(buf, &pSW->nCols); + if (isGetEx) { + pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + SSchemaEx *pSchema = pSW->pSchemaEx + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI8(buf, &pSchema->sma); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + } else { + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + SSchema *pSchema = pSW->pSchema + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosSkipFixedLen(buf, sizeof(int8_t)); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + } + + return buf; +} + static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { int tsize = 0; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 61538055f2..a18374015b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -142,7 +142,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = pColSchema->colId; colInfo.info.type = pColSchema->type; - if (blockDataEnsureColumnCapacity(&colInfo, numOfRows) < 0) { + if (colInfoDataEnsureCapacity(&colInfo, numOfRows) < 0) { taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cae6d5ec1a..4a3e764b13 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -14,7 +14,7 @@ */ #include "tsdbDef.h" -#include +#include "tdatablock.h" #include "os.h" #include "talgo.h" #include "tcompare.h" @@ -31,6 +31,7 @@ #include "tlosertree.h" #include "tsdbDef.h" #include "tmsg.h" +#include "tsdbCommit.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -209,34 +210,34 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { -// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; -// -// int64_t rows = 0; -// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; -// if (pMemTable == NULL) { return rows; } -// -//// STableData* pMem = NULL; -//// STableData* pIMem = NULL; -// -//// SMemTable* pMemT = pMemRef->snapshot.mem; -//// SMemTable* pIMemT = pMemRef->snapshot.imem; -// -// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); -// for (int32_t i = 0; i < size; ++i) { -// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); -// -//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { -//// pMem = pMemT->tData[pCheckInfo->tableId]; -//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; -//// } -//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { -//// pIMem = pIMemT->tData[pCheckInfo->tableId]; -//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; -//// } -// } -// return rows; -//} +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + + int64_t rows = 0; + STsdbMemTable* pMemTable = NULL;//pTsdbReadHandle->pMemTable; + if (pMemTable == NULL) { return rows; } + +// STableData* pMem = NULL; +// STableData* pIMem = NULL; + +// SMemTable* pMemT = pMemRef->snapshot.mem; +// SMemTable* pIMemT = pMemRef->snapshot.imem; + + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + +// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +// pMem = pMemT->tData[pCheckInfo->tableId]; +// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +// } +// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +// pIMem = pIMemT->tData[pCheckInfo->tableId]; +// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +// } + } + return rows; +} static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) { size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); @@ -403,7 +404,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - int32_t code = blockDataEnsureColumnCapacity(&colInfo, pReadHandle->outputCapacity); + int32_t code = colInfoDataEnsureCapacity(&colInfo, pReadHandle->outputCapacity); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -2261,12 +2262,13 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { cur->mixBlock = false; cur->blockCompleted = false; } -#if 0 -int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) { + +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file @@ -2284,9 +2286,11 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); + int defaultRows = 4096;//TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; + bool ascTraverse = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + while (true) { numOfBlocks = 0; tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); @@ -2299,8 +2303,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + if ((ascTraverse && win.skey > pTsdbReadHandle->window.ekey) || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle, pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); @@ -2342,19 +2345,26 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa int32_t numOfRows = pBlock[j].numOfRows; pTableBlockInfo->totalRows += numOfRows; - if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; - if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; - if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1; - int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; - SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); - blockInfo->numBlocksOfStep++; + if (numOfRows > pTableBlockInfo->maxRows) { + pTableBlockInfo->maxRows = numOfRows; + } + + if (numOfRows < pTableBlockInfo->minRows) { + pTableBlockInfo->minRows = numOfRows; + } + + if (numOfRows < defaultRows) { + pTableBlockInfo->numOfSmallBlocks += 1; + } +// int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; +// SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); +// blockInfo->numBlocksOfStep++; } } } return code; } -#endif static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 38eed46c9d..7ef0b50402 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -78,11 +78,13 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } - // TODO: maybe need to clear the request struct + // TODO: to encapsule a free API taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); - taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols); - taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); + if(vCreateTbReq.stbCfg.pRSmaParam) { + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); + } taosMemoryFree(vCreateTbReq.dbFName); taosMemoryFree(vCreateTbReq.name); break; @@ -111,19 +113,24 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } + // TODO: to encapsule a free API taosMemoryFree(pCreateTbReq->name); taosMemoryFree(pCreateTbReq->dbFName); if (pCreateTbReq->type == TD_SUPER_TABLE) { taosMemoryFree(pCreateTbReq->stbCfg.pSchema); taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); - taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols); - taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); + if (pCreateTbReq->stbCfg.pRSmaParam) { + taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); + } } else if (pCreateTbReq->type == TD_CHILD_TABLE) { taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); - taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols); - taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); + if (pCreateTbReq->ntbCfg.pRSmaParam) { + taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); + } } } @@ -148,10 +155,13 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); + // TODO: to encapsule a free API taosMemoryFree(vAlterTbReq.stbCfg.pSchema); taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); - taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols); - taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); + if (vAlterTbReq.stbCfg.pRSmaParam) { + taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); + } taosMemoryFree(vAlterTbReq.dbFName); taosMemoryFree(vAlterTbReq.name); break; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 7ffbd4f80a..4d4ac6c1e4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -16,7 +16,16 @@ #include "command.h" #include "tdatablock.h" -// #define SET_VARSTR(pData, val, pOffset) +static int32_t getSchemaBytes(const SSchema* pSchema) { + switch (pSchema->type) { + case TSDB_DATA_TYPE_BINARY: + return (pSchema->bytes - VARSTR_HEADER_SIZE); + case TSDB_DATA_TYPE_NCHAR: + return (pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + default: + return pSchema->bytes; + } +} static void buildRspData(const STableMeta* pMeta, char* pData) { int32_t* pColSizes = (int32_t*)pData; @@ -50,7 +59,7 @@ static void buildRspData(const STableMeta* pMeta, char* pData) { // Length pData += BitmapLen(numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { - *(int32_t*)pData = pMeta->schema[i].bytes; + *(int32_t*)pData = getSchemaBytes(pMeta->schema + i); pData += sizeof(int32_t); } pColSizes[2] = sizeof(int32_t) * numOfRows; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 287cb094a6..5a3e25bd68 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -128,6 +128,11 @@ typedef struct { int64_t sumRunTimes; } SOperatorProfResult; +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + typedef struct STaskCostInfo { int64_t created; int64_t start; @@ -163,6 +168,11 @@ typedef struct SOperatorCostInfo { uint64_t execCost; } SOperatorCostInfo; +typedef struct SOrder { + uint32_t order; + SColumn col; +} SOrder; + // The basic query information extracted from the SQueryInfo tree to support the // execution of query in a data node. typedef struct STaskAttr { @@ -196,7 +206,6 @@ typedef struct STaskAttr { STimeWindow window; SInterval interval; - SSessionWindow sw; int16_t precision; int16_t numOfOutput; int16_t fillType; @@ -206,13 +215,8 @@ typedef struct STaskAttr { int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. int32_t maxTableColumnWidth; int32_t tagLen; // tag value length of current query - SGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; - SExprInfo* pExpr2; - int32_t numOfExpr2; - SExprInfo* pExpr3; - int32_t numOfExpr3; SColumnInfo* tableCols; SColumnInfo* tagColList; @@ -220,8 +224,6 @@ typedef struct STaskAttr { int64_t* fillVal; SSingleColumnFilterInfo* pFilterInfo; - // SFilterInfo *pFilters; - void* tsdb; STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; @@ -384,7 +386,7 @@ typedef struct SExchangeInfo { } SExchangeInfo; typedef struct STableScanInfo { - void* pTsdbReadHandle; + void* dataReader; int32_t numOfBlocks; // extract basic running information. int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -393,6 +395,7 @@ typedef struct STableScanInfo { int32_t times; // repeat counts int32_t current; int32_t reverseTimes; // 0 by default + SNode* pFilterNode; // filter operator info SqlFunctionCtx* pCtx; // next operator query context SResultRowInfo* pResultRowInfo; int32_t* rowCellInfoOffset; @@ -496,14 +499,11 @@ typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; SSDataBlock *existDataBlock; int32_t threshold; + SLimit limit; + int64_t curOffset; + int64_t curOutput; } SProjectOperatorInfo; -typedef struct SLimitOperatorInfo { - SLimit limit; - int64_t currentOffset; - int64_t currentRows; -} SLimitOperatorInfo; - typedef struct SSLimitOperatorInfo { int64_t groupTotal; int64_t currentGroupOffset; @@ -522,11 +522,6 @@ typedef struct SSLimitOperatorInfo { int64_t threshold; } SSLimitOperatorInfo; -typedef struct SFilterOperatorInfo { - SSingleColumnFilterInfo* pFilterInfo; - int32_t numOfFilterCols; -} SFilterOperatorInfo; - typedef struct SFillOperatorInfo { struct SFillInfo* pFillInfo; SSDataBlock* pRes; @@ -557,15 +552,16 @@ typedef struct SGroupbyOperatorInfo { } SGroupbyOperatorInfo; typedef struct SSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SGroupResInfo groupResInfo; - STimeWindow curWindow; // current time window - TSKEY prevTs; // previous timestamp - int32_t numOfRows; // number of rows - int32_t start; // start row index - bool reptScan; // next round scan - int64_t gap; // session window gap + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SGroupResInfo groupResInfo; + STimeWindow curWindow; // current time window + TSKEY prevTs; // previous timestamp + int32_t numOfRows; // number of rows + int32_t start; // start row index + bool reptScan; // next round scan + int64_t gap; // session window gap + SColumnInfoData timeWindowData; // query time window info for scalar function execution. } SSessionAggOperatorInfo; typedef struct SStateWindowOperatorInfo { @@ -636,27 +632,27 @@ typedef struct SDistinctOperatorInfo { SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime, - int32_t reverseTime, SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo); + int32_t reverseTime, SArray* pColMatchInfo, SNode* pCondition, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); - SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, @@ -673,30 +669,19 @@ SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExp SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); + SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); - void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); - -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); -int32_t createQueryFilter(char* data, uint16_t len, SFilterInfo** pFilters); - -int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId); -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); - STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 54a75d9c7e..2ca06cb15f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -194,9 +194,6 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, - int32_t numOfCols, int32_t* rowCellInfoOffset); - void setResultRowOutputBufInitCtx(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); static bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -214,8 +211,6 @@ static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); // static STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); -static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); - static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); @@ -254,7 +249,6 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); -static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); @@ -264,10 +258,6 @@ static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExpr); -static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo); @@ -296,44 +286,6 @@ static int compareRowData(const void* a, const void* b, const void* userData) { return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } -static void sortGroupResByOrderList(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, - SSDataBlock* pDataBlock) { - SArray* columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - size_t size = taosArrayGetSize(columnOrderList); - taosArrayDestroy(columnOrderList); - - if (size <= 0) { - return; - } - - int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.colId; - if (orderId <= 0) { - return; - } - - bool found = false; - int16_t dataOffset = 0; - - for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, j); - if (orderId == j) { - found = true; - break; - } - - dataOffset += pColInfoData->info.bytes; - } - - if (found == false) { - return; - } - - int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type; - - SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; - taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); -} - // setup the output buffer for each operator SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); @@ -384,17 +336,6 @@ static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput // return (numOfSelectivity > 0 && hasTags); } -static bool isProjQuery(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { - return false; - } - } - - return true; -} - static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -1023,7 +964,7 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - blockDataEnsureColumnCapacity(pColData, 5); + colInfoDataEnsureCapacity(pColData, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -1033,29 +974,25 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin) { +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { int64_t* ts = (int64_t*)pColData->pData; + int32_t delta = includeEndpoint? 1:0; - int64_t duration = pWin->ekey - pWin->skey + 1; + int64_t duration = pWin->ekey - pWin->skey + delta; ts[2] = duration; // set the duration ts[3] = pWin->skey; // window start key - ts[4] = pWin->ekey + 1; // window end key + ts[4] = pWin->ekey + delta; // window end key } static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { - SScalarParam intervalParam = {.numOfRows = 5, .columnData = pTimeWindowData}; //TODO move out of this function - if (pTimeWindowData != NULL) { - updateTimeWindowInfo(pTimeWindowData, pWin); - } - for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pWin->skey; // keep it temporarialy - bool hasAgg = pCtx[k].input.colDataAggIsSet; + bool hasAgg = pCtx[k].input.colDataAggIsSet; + int32_t numOfRows = pCtx[k].input.numOfRows; int32_t startOffset = pCtx[k].input.startRowIndex; - int32_t numOfRows = pCtx[k].input.numOfRows; int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); pCtx[k].input.startRowIndex = pos; @@ -1075,12 +1012,14 @@ static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInf SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); char* p = GET_ROWCELL_INTERBUF(pEntryInfo); - SScalarParam out = {.columnData = NULL}; - out.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - out.columnData->info.type = TSDB_DATA_TYPE_BIGINT; - out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - out.columnData->pData = p; - pCtx[k].sfp.process(&intervalParam, 1, &out); + SColumnInfoData idata = {0}; + idata.info.type = TSDB_DATA_TYPE_BIGINT; + idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + idata.pData = p; + + SScalarParam out = {.columnData = &idata}; + SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; + pCtx[k].sfp.process(&tw, 1, &out); pEntryInfo->numOfRes = 1; pEntryInfo->hasResult = ','; continue; @@ -1422,15 +1361,10 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + bool ascQuery = true; TSKEY curTs = tsCols[pos]; - TSKEY lastTs = *(TSKEY*)pRuntimeEnv->prevRow[0]; + TSKEY lastTs = 0;//*(TSKEY*)pRuntimeEnv->prevRow[0]; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation @@ -1445,27 +1379,24 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlF return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); return true; } static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperatorInfo->numOfOutput; + int32_t order = TSDB_ORDER_ASC; + int32_t numOfOutput = pOperatorInfo->numOfOutput; TSKEY actualEndKey = tsCols[endRowIndex]; - - TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr) ? win->ekey : win->skey; + TSKEY key = order ? win->ekey : win->skey; // not ended in current data block, do not invoke interpolation - if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); return false; } @@ -1476,7 +1407,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); int32_t nextRowIndex = endRowIndex + step; assert(nextRowIndex >= 0); @@ -1626,8 +1557,9 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe #endif // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); + + updateTimeWindowInfo(&pInfo->timeWindowData, &win, true); doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; @@ -1656,8 +1588,9 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); + + updateTimeWindowInfo(&pInfo->timeWindowData, &win, true); doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } @@ -1676,10 +1609,9 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = true; TSKEY* tsCols = NULL; if (pSDataBlock->pDataBlock != NULL) { @@ -1692,7 +1624,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe int32_t startPos = ascQuery ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); - STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + STimeWindow win = {0};//getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; @@ -1708,7 +1640,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + TSKEY ekey = 0;//reviseWindowEkey(pQueryAttr, &win); // forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, // binarySearchForKey, true); @@ -1722,31 +1654,31 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, // prevEndPos); if (startPos < 0) { - if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { - int32_t code = - setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - startPos = pSDataBlock->info.rows - 1; +// if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { +// int32_t code = +// setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, +// tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); +// if (code != TSDB_CODE_SUCCESS || pResult == NULL) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); +// } +// +// startPos = pSDataBlock->info.rows - 1; // window start(end) key interpolation // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, // forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, // forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } +// } break; } setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0; +// if (pQueryAttr->timeWindowInterpo) { +// int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0; // saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } +// } // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } @@ -1924,73 +1856,80 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } +static void doKeepTuple(SSessionAggOperatorInfo* pInfo, int64_t ts) { + pInfo->curWindow.ekey = ts; + pInfo->prevTs = ts; + pInfo->numOfRows += 1; +} + +static void doKeepSessionStartInfo(SSessionAggOperatorInfo* pInfo, const int64_t* tsList, int32_t rowIndex) { + pInfo->start = rowIndex; + pInfo->numOfRows = 0; + pInfo->curWindow.skey = tsList[rowIndex]; +} + // todo handle multiple tables cases. static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - // primary timestamp column SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - bool masterScan = true; - STimeWindow window = {0}; - int32_t numOfOutput = pOperator->numOfOutput; - int64_t gid = pBlock->info.groupId; + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfOutput; + int64_t gid = pBlock->info.groupId; int64_t gap = pInfo->gap; pInfo->numOfRows = 0; - if (/*IS_REPEAT_SCAN(pRuntimeEnv) && */ !pInfo->reptScan) { + if (!pInfo->reptScan) { pInfo->reptScan = true; - pInfo->prevTs = INT64_MIN; + pInfo->prevTs = INT64_MIN; } + // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. TSKEY* tsList = (TSKEY*)pColInfoData->pData; for (int32_t j = 0; j < pBlock->info.rows; ++j) { if (pInfo->prevTs == INT64_MIN) { - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; + doKeepSessionStartInfo(pInfo, tsList, j); + doKeepTuple(pInfo, tsList[j]); } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) { - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows += 1; + // The gap is less than the threshold, so it belongs to current session window that has been opened already. + doKeepTuple(pInfo, tsList[j]); if (j == 0 && pInfo->start != 0) { - pInfo->numOfRows = 1; pInfo->start = 0; } } else { // start a new session window SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pInfo->curWindow; + pInfo->curWindow.ekey = pInfo->curWindow.skey; int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + &pResult, gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } // pInfo->numOfRows data belong to the current session window - doApplyFunctions(pInfo->binfo.pCtx, &window, NULL, pInfo->start, pInfo->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->timeWindowData, pInfo->start, pInfo->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; + // here we start a new session window + doKeepSessionStartInfo(pInfo, tsList, j); + doKeepTuple(pInfo, tsList[j]); } } SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, &pResult, - gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); + pInfo->curWindow.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pInfo->curWindow, masterScan, &pResult, + gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pInfo->binfo.pCtx, &window, NULL, pInfo->start, pInfo->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->timeWindowData, &pInfo->curWindow, false); + doApplyFunctions(pInfo->binfo.pCtx, &pInfo->curWindow, &pInfo->timeWindowData, pInfo->start, pInfo->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { @@ -2025,28 +1964,6 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo return TSDB_CODE_SUCCESS; } -static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock) { - size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo); - for (int32_t k = 0; k < num; ++k) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - continue; - } - - int32_t colId = pColIndex->colId; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - if (pColInfo->info.colId == colId) { - return i; - } - } - } - - assert(0); - return -1; -} - static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -2266,64 +2183,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv* pRuntimeEnv, int32_t numOfTables, SArray* pOperator, - void* merger) { - // qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pQueryAttr = pQueryAttr; - - pRuntimeEnv->pResultRowHashTable = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pRuntimeEnv->pResultRowListSet = - taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - // pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); - pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - - pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen); - - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pRuntimeEnv->pTableRetrieveTsMap = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - // pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); - - if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || - pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { - goto _clean; - } - - if (pQueryAttr->numOfCols) { - char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*)pRuntimeEnv->prevRow; - pRuntimeEnv->prevRow[0] = start; - for (int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { - pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i - 1].bytes; - } - - if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { - *(int64_t*)pRuntimeEnv->prevRow[0] = INT64_MIN; - } - } - - // qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); - - // group by normal column, sliding window query, interval query are handled by interval query processor - // interval (down sampling operation) - return TSDB_CODE_SUCCESS; - -_clean: - // destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); - taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable); - taosMemoryFreeClear(pRuntimeEnv->keyBuf); - taosMemoryFreeClear(pRuntimeEnv->prevRow); - taosMemoryFreeClear(pRuntimeEnv->tagVal); - - return TSDB_CODE_QRY_OUT_OF_MEMORY; -} - static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -2402,17 +2261,6 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q // return false; //} -static bool isFirstLastRowQuery(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionID == FUNCTION_LAST_ROW) { - return true; - } - } - - return false; -} - static bool isCachedLastQuery(STaskAttr* pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -2495,18 +2343,6 @@ static bool onlyOneQueryType(STaskAttr* pQueryAttr, int32_t functId, int32_t fun return true; } -static bool onlyFirstQuery(STaskAttr* pQueryAttr) { - return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); -} - -static bool onlyLastQuery(STaskAttr* pQueryAttr) { - return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); -} - -static bool notContainSessionOrStateWindow(STaskAttr* pQueryAttr) { - return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); -} - static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { bool hasFirstLastFunc = false; bool hasOtherFunc = false; @@ -3008,7 +2844,7 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, *status = BLK_DATA_ALL_NEEDED; - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); + SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (pCols == NULL) { return terrno; } @@ -3025,6 +2861,39 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); } + if (pTableScanInfo->pFilterNode != NULL) { + SFilterInfo* filter = NULL; + int32_t code = filterInitFromNode((SNode*)pTableScanInfo->pFilterNode, &filter, 0); + + SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m1); + + int8_t* rowRes = NULL; + bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); + + SSDataBlock* px = createOneDataBlock(pBlock); + blockDataEnsureCapacity(px, pBlock->info.rows); + + int32_t numOfRow = 0; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + + numOfRow = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } + + colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); + numOfRow += 1; + } + *pSrc = *pDst; + } + + pBlock->info.rows = numOfRow; + } + return TSDB_CODE_SUCCESS; } @@ -3344,11 +3213,6 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCt offset += pLocalExprInfo->base.resSchema.bytes; } - - // todo : use index to avoid iterator all possible output columns - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); - } } // set the tsBuf start position before check each data block @@ -3544,19 +3408,6 @@ void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) } } -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity) { - SSDataBlock* pDataBlock = pBInfo->pRes; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId < 0) { - memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity)); - } - } -} - void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); @@ -3714,23 +3565,6 @@ STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow return pTableQueryInfo; } -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(1, sizeof(STableQueryInfo)); - - // pTableQueryInfo->win = win; - pTableQueryInfo->lastKey = win.skey; - - // set more initial size of interval/groupby query - int32_t initialSize = 16; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableQueryInfo); - return NULL; - } - - return pTableQueryInfo; -} - void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; @@ -3834,30 +3668,6 @@ void setExecutionContext(int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKe pAggInfo->groupId = tableGroupId; } -void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, - int32_t* rowCellInfoOffset) { - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - SFilePage* page = getBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - - int16_t offset = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); - offset += pCtx[i].resDataInfo.bytes; - - int32_t functionId = pCtx[i].functionId; - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || - functionId == FUNCTION_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; - } - - /* - * set the output buffer information and intermediate buffer, - * not all queries require the interResultBuf, such as COUNT - */ - pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - } -} - void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -3927,76 +3737,6 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S return 0; } -// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExprInfo) { -#if 0 - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo1 = &(pExprInfo[i]); - if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) { - continue; - } - - SExprBasicInfo* pExpr = &pExprInfo1->base; - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->info.colId == pExpr->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -#endif -} - -void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExpr, char* val, int16_t bytes) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -#if 0 - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprBasicInfo* pExpr1 = &pExpr[i].base; - if (pExpr1->functionId != FUNCTION_STDDEV_DST) { - continue; - } - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->info.colId == pExpr1->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -#endif -} - /* * There are two cases to handle: * @@ -4626,73 +4366,6 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ return terrno; } -int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, - void* param) { - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - - STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - pQueryAttr->tsdb = tsdb; - - if (tsdb != NULL) { - int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); - - pRuntimeEnv->groupResInfo.totalGroup = (int32_t)(pQueryAttr->stableQuery ? GET_NUM_OF_TABLEGROUP(pRuntimeEnv) : 0); - pRuntimeEnv->enableGroupData = false; - - pRuntimeEnv->pQueryAttr = pQueryAttr; - pRuntimeEnv->pTsBuf = pTsBuf; - pRuntimeEnv->cur.vgroupIndex = -1; - setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); - - if (sourceOptr != NULL) { - assert(pRuntimeEnv->proot == NULL); - pRuntimeEnv->proot = sourceOptr; - } - - if (pTsBuf != NULL) { - int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); - } - - int32_t ps = 4096; - getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); - - int32_t TENMB = 1024 * 1024 * 10; - int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp"); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // create runtime environment - int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; - pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); - pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); - if (pQInfo->summary.queryProfEvents == NULL) { - // qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); - } - - pQInfo->summary.operatorProfResults = - taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); - - if (pQInfo->summary.operatorProfResults == NULL) { - // qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); - } - - code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t)pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // setTaskStatus(pOperator->pTaskInfo, QUERY_NOT_COMPLETED); - return TSDB_CODE_SUCCESS; -} - static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { #if 0 if (order == TSDB_ORDER_ASC) { @@ -4774,13 +4447,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { *newgroup = false; - while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { if (isTaskKilled(pOperator->pTaskInfo)) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4790,7 +4463,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { // break; // } // - // pRuntimeEnv->current = *pTableQueryInfo; // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); // } @@ -4818,7 +4490,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->pTsdbReadHandle == NULL) { + if (pTableScanInfo->dataReader == NULL) { return NULL; } @@ -4846,11 +4518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - // if (pTaskInfo->pTsBuf) { - // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - // assert(ret); - // } - // if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = 0; } @@ -4886,44 +4553,44 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { STableScanInfo* pTableScanInfo = pOperator->info; *newgroup = false; -#if 0 - STableBlockDist tableBlockDist = {0}; - tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + + STableBlockDistInfo tableBlockDist = {0}; + tableBlockDist.numOfTables = 1; // TODO set the correct number of tables int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++numRowSteps; } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; pBlock->info.numOfCols = 1; - SBufferWriter bw = tbufInitWriter(NULL, false); - blockDistInfoToBinary(&tableBlockDist, &bw); +// SBufferWriter bw = tbufInitWriter(NULL, false); +// blockDistInfoToBinary(&tableBlockDist, &bw); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); +// int32_t len = (int32_t) tbufTell(&bw); +// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); +// *(int32_t*) pColInfo->pData = len; +// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); +// +// tbufCloseWriter(&bw); - *(int32_t*) pColInfo->pData = len; - memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - - tbufCloseWriter(&bw); - - SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); - pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); +// SArray* g = GET_TABLEGROUP(pOperator->, 0); +// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); pOperator->status = OP_EXEC_DONE; return pBlock; -#endif } static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { @@ -5548,7 +5215,7 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, - SExecTaskInfo* pTaskInfo) { + SNode* pCondition, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); @@ -5567,21 +5234,22 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, taosArrayPush(pInfo->block.pDataBlock, &idata); } - pInfo->pTsdbReadHandle = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pTsdbReadHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -5589,7 +5257,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbReadHandle; + pInfo->dataReader = pTsdbReadHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5610,32 +5278,42 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim return pOperator; } -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } - pInfo->pTsdbReadHandle = pTsdbReadHandle; + pInfo->dataReader = dataReader; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {{0}}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; infoData.info.bytes = 1024; infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableBlockInfoScanOperator"; + pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - // pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->getNextFn = doBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doBlockInfoScan; + + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5674,16 +5352,17 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pInfo->readerHandle = streamReadHandle; pInfo->pRes = pResBlock; - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doStreamBlockScan; + pOperator->closeFn = operatorDummyCloseFn; + pOperator->pTaskInfo = pTaskInfo; + return pOperator; } @@ -5799,7 +5478,7 @@ void getDBNameFromCondition(SNode* pCondition, char* dbName) { return; } - nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); } static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { @@ -5838,7 +5517,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); int64_t tmp = 0; char t[10] = {0}; - STR_TO_VARSTR(t, "_"); + STR_TO_VARSTR(t, "_"); //TODO if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { colDataAppend(pColInfoData, numOfRows, t, false); } else { @@ -6017,80 +5696,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return pOperator; } -SArray* getOrderCheckColumns(STaskAttr* pQuery) { - int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQuery->interval.interval > 0) { - if (pOrderColumns == NULL) { - pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); - } - - SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; - taosArrayPush(pOrderColumns, &colIndex); - } - - { - numOfCols = (int32_t)taosArrayGetSize(pOrderColumns); - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); - - if (index->colId == pExpr->pParam[0].pCol->colId && - (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) { - index->colIndex = j; - index->colId = pExpr->resSchema.colId; - } - } - } - } - - return pOrderColumns; -} - -SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { - int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - - bool found = false; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); - - // FUNCTION_TAG_DUMMY function needs to be ignored - // if (index->colId == pExpr->pColumns->info.colId && - // ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) || - // (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) { - // index->colIndex = j; - // index->colId = pExpr->resSchema.colId; - // found = true; - // break; - // } - } - - assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); - } - - return pOrderColumns; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey); static void cleanupAggSup(SAggSupporter* pAggSup); @@ -6802,7 +6407,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - +#if 0 if (pProjectInfo->existDataBlock) { // TODO refactor SSDataBlock* pBlock = pProjectInfo->existDataBlock; pProjectInfo->existDataBlock = NULL; @@ -6824,6 +6429,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) return pRes; } } +#endif SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -6863,67 +6469,32 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput); + + if (pProjectInfo->curOffset < pInfo->pRes->info.rows && pProjectInfo->curOffset > 0) { + blockDataTrimFirstNRows(pInfo->pRes, pProjectInfo->curOffset); + pProjectInfo->curOffset = 0; + break; + } else if (pProjectInfo->curOffset >= pInfo->pRes->info.rows) { + pProjectInfo->curOffset -= pInfo->pRes->info.rows; + blockDataCleanup(pInfo->pRes); + continue; + } + if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; } } + + if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pInfo->pRes->info.rows >= pProjectInfo->limit.limit) { + pInfo->pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); + } + + pProjectInfo->curOutput += pInfo->pRes->info.rows; // copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; } -static SSDataBlock* doLimit(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SLimitOperatorInfo* pInfo = pOperator->info; - - SSDataBlock* pBlock = NULL; - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pDownstream->getNextFn(pDownstream, newgroup); - publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - if (pInfo->currentOffset == 0) { - break; - } else if (pInfo->currentOffset >= pBlock->info.rows) { - pInfo->currentOffset -= pBlock->info.rows; - } else { // TODO handle the data movement - int32_t remain = (int32_t)(pBlock->info.rows - pInfo->currentOffset); - pBlock->info.rows = remain; - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColInfoData->info.bytes; - memmove(pColInfoData->pData, pColInfoData->pData + bytes * pInfo->currentOffset, remain * bytes); - } - - pInfo->currentOffset = 0; - break; - } - } - - if (pInfo->currentRows + pBlock->info.rows >= pInfo->limit.limit) { - pBlock->info.rows = (int32_t)(pInfo->limit.limit - pInfo->currentRows); - pInfo->currentRows = pInfo->limit.limit; - - doSetOperatorCompleted(pOperator); - } else { - pInfo->currentRows += pBlock->info.rows; - } - - return pBlock; -} - static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -7173,9 +6744,6 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr return pIntervalInfo->binfo.pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -7191,14 +6759,14 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); +// setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); } pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order +// pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); @@ -7294,22 +6862,22 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { } SStateWindowOperatorInfo* pWindowInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } +// if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { +// pOperator->status = OP_EXEC_DONE; +// } return pBInfo->pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; + int32_t order = TSDB_ORDER_ASC; + STimeWindow win = pTaskInfo->window; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); @@ -7319,28 +6887,29 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { if (pBlock == NULL) { break; } - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); - if (pWindowInfo->colIndex == -1) { - pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } + +// setInputDataBlock(pOperator, pBInfo->pCtx, pDataBlock, TSDB_ORDER_ASC); +// if (pWindowInfo->colIndex == -1) { +// pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); +// } doStateWindowAggImpl(pOperator, pWindowInfo, pBlock); } // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; +// pQueryAttr->order.order = order; +// pQueryAttr->window = win; pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pBInfo->pCtx, pOperator->numOfOutput); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); +// initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } +// if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { +// pOperator->status = OP_EXEC_DONE; +// } return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -7354,8 +6923,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pBInfo->pRes, pBInfo->capacity, - pBInfo->rowCellInfoOffset); + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pBInfo->pRes, pBInfo->capacity, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -7383,14 +6951,11 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) // restore the value pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); - blockDataEnsureCapacity(pBInfo->pRes, pBInfo->capacity); - toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pBInfo->pRes, pBInfo->capacity, - pBInfo->rowCellInfoOffset); + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pBInfo->pRes, pBInfo->capacity, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -7772,11 +7337,6 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pSortInfo); } -static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { - SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*)param; - doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); -} - static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*)param; taosHashCleanup(pInfo->pSet); @@ -7846,19 +7406,20 @@ _error: } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, - SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } + pInfo->limit = *pLimit; + pInfo->curOffset = pLimit->offset; pInfo->binfo.pRes = pResBlock; pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, num, &pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pCtx == NULL) { goto _error; } - // initResultRowInfo(&pBInfo->resultRowInfo, 8); // setFunctionResultOutput(pBInfo, MAIN_SCAN); @@ -7886,35 +7447,6 @@ _error: return NULL; } -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { - SLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SLimitOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->limit = *pLimit; - pInfo->currentOffset = pLimit->offset; - - pOperator->name = "LimitOperator"; - // pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LIMIT; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doLimit; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { @@ -7996,9 +7528,9 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S return pOperator; } -SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput) { +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); + pInfo->colIndex = -1; pInfo->reptScan = false; // pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); @@ -8009,13 +7541,14 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->name = "StateWindowOperator"; // pOperator->operatorType = OP_StateWindow; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = doStateWindowAgg; - pOperator->closeFn = destroyStateWindowOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->getNextFn = doStateWindowAgg; + pOperator->closeFn = destroyStateWindowOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -8030,28 +7563,28 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo } int32_t numOfRows = 4096; - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + initExecTimeWindowInfo(&pInfo->timeWindowData, &pTaskInfo->window); - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->getNextFn = doSessionWindowAgg; - pOperator->closeFn = destroySWindowOperatorInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->getNextFn = doSessionWindowAgg; + pOperator->closeFn = destroySWindowOperatorInfo; + pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -8317,7 +7850,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pQueryAttr->numOfOutput == 1); SExprInfo* pExprInfo = &pOperator->pExpr[0]; @@ -8818,7 +8350,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, - pScanPhyNode->reverse, pColList, pTaskInfo); + pScanPhyNode->reverse, pColList, pScanPhyNode->node.pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); @@ -8861,9 +8393,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, NULL, &num); + SProjectPhysiNode* pProjPhyNode = (SProjectPhysiNode*) pPhyNode; + SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &num); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo); + SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; + + return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, &limit, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == nodeType(pPhyNode)) { size_t size = LIST_LENGTH(pPhyNode->pChildren); assert(size == 1); @@ -9096,8 +8632,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod return pList; } -int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, - uint64_t queryId, uint64_t taskId) { +int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); @@ -9179,41 +8714,6 @@ _complete: return code; } -int32_t cloneExprFilterInfo(SColumnFilterInfo** dst, SColumnFilterInfo* src, int32_t filterNum) { - if (filterNum <= 0) { - return TSDB_CODE_SUCCESS; - } - - *dst = taosMemoryCalloc(filterNum, sizeof(*src)); - if (*dst == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(*dst, src, sizeof(*src) * filterNum); - - for (int32_t i = 0; i < filterNum; i++) { - if ((*dst)[i].filterstr && dst[i]->len > 0) { - void* pz = taosMemoryCalloc(1, (size_t)(*dst)[i].len + 1); - - if (pz == NULL) { - if (i == 0) { - taosMemoryFree(*dst); - } else { - freeColumnFilterInfo(*dst, i); - } - - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(pz, (void*)src->pz, (size_t)src->len + 1); - - (*dst)[i].pz = (int64_t)pz; - } - } - - return TSDB_CODE_SUCCESS; -} - static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { for (int32_t i = 0; i < numOfOutput; ++i) { @@ -9236,150 +8736,6 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol return TSDB_CODE_SUCCESS; } -// TODO tag length should be passed from client, refactor -int32_t createQueryFilter(char* data, uint16_t len, SFilterInfo** pFilters) { - tExprNode* expr = NULL; - - TRY(TSDB_MAX_TAG_CONDITIONS) { expr = exprTreeFromBinary(data, len); } - CATCH(code) { - CLEANUP_EXECUTE(); - return code; - } - END_TRY - - if (expr == NULL) { - // qError("failed to create expr tree"); - return TSDB_CODE_QRY_APP_ERROR; - } - - // int32_t ret = filterInitFromTree(expr, pFilters, 0); - // tExprTreeDestroy(expr, NULL); - - // return ret; -} - -// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** -// pFilterInfo, uint64_t qId) { -// *pFilterInfo = taosMemoryCalloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); -// if (*pFilterInfo == NULL) { -// return TSDB_CODE_QRY_OUT_OF_MEMORY; -// } -// -// for (int32_t i = 0, j = 0; i < numOfCols; ++i) { -// if (pCols[i].flist.numOfFilters > 0) { -// SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]); -// -// memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo)); -// pFilter->info = pCols[i]; -// -// pFilter->numOfFilters = pCols[i].flist.numOfFilters; -// pFilter->pFilters = taosMemoryCalloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); -// if (pFilter->pFilters == NULL) { -// return TSDB_CODE_QRY_OUT_OF_MEMORY; -// } -// -// for (int32_t f = 0; f < pFilter->numOfFilters; ++f) { -// SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f]; -// pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f]; -// -// int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; -// int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; -// if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { -// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); -// return TSDB_CODE_QRY_INVALID_MSG; -// } -// -// pSingleColFilter->fp = getFilterOperator(lower, upper); -// if (pSingleColFilter->fp == NULL) { -// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); -// return TSDB_CODE_QRY_INVALID_MSG; -// } -// -// pSingleColFilter->bytes = pCols[i].bytes; -// -// if (lower == TSDB_RELATION_IN) { -//// buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), -///(int32_t)(pSingleColFilter->filterInfo.len)); -// } -// } -// -// j++; -// } -// } -// -// return TSDB_CODE_SUCCESS; -//} - -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { - // for (int32_t i = 0; i < numOfFilterCols; ++i) { - // if (pFilterInfo[i].numOfFilters > 0) { - // if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - // taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); - // } - // taosMemoryFreeClear(pFilterInfo[i].pFilters); - // } - // } - // - // taosMemoryFreeClear(pFilterInfo); - return NULL; -} - -int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { - for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { - // if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { - // pQueryAttr->numOfFilterCols++; - // } - } - - if (pQueryAttr->numOfFilterCols == 0) { - return TSDB_CODE_SUCCESS; - } - - // doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - // &pQueryAttr->pFilterInfo, qId); - - pQueryAttr->createFilterOperator = true; - - return TSDB_CODE_SUCCESS; -} - -static void doUpdateExprColumnIndex(STaskAttr* pQueryAttr) { - assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); - - for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { - SExprBasicInfo* pSqlExprMsg = &pQueryAttr->pExpr1[k].base; - // if (pSqlExprMsg->functionId == FUNCTION_ARITHM) { - // continue; - // } - - // todo opt performance - SColIndex* pColIndex = NULL; /*&pSqlExprMsg->colInfo;*/ - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfCols; ++f) { - if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfCols); - } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { - // do nothing for user-defined constant value result columns - } else { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfTags; ++f) { - if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); - } - } -} - void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); @@ -9389,16 +8745,16 @@ void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const float THRESHOLD_RATIO = 0.85f; - if (isProjQuery(pQueryAttr)) { - int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - } - - pResultInfo->capacity = numOfRes; - } else { // in case of non-prj query, a smaller output buffer will be used. - pResultInfo->capacity = DEFAULT_MIN_ROWS; - } +// if (isProjQuery(pQueryAttr)) { +// int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; +// if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { +// numOfRes = MIN_ROWS_FOR_PRJ_QUERY; +// } +// +// pResultInfo->capacity = numOfRes; +// } else { // in case of non-prj query, a smaller output buffer will be used. +// pResultInfo->capacity = DEFAULT_MIN_ROWS; +// } pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); pResultInfo->totalRows = 0; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f7fccb29f7..ab7bfc7767 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -38,9 +38,14 @@ void minFunction(SqlFunctionCtx* pCtx); void maxFunction(SqlFunctionCtx *pCtx); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); void stddevFunction(SqlFunctionCtx* pCtx); void stddevFinalize(SqlFunctionCtx* pCtx); +bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +void percentileFunction(SqlFunctionCtx *pCtx); + bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); void firstFunction(SqlFunctionCtx *pCtx); void lastFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 024da4e04b..0048cc6f29 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -68,9 +68,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .classification = FUNC_MGT_AGG_FUNC, .checkFunc = stubCheckAndGetResultType, .getEnvFunc = getStddevFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize }, { .name = "percentile", @@ -282,6 +282,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = atanFunction, .finalizeFunc = NULL }, + { + .name = "length", + .type = FUNCTION_TYPE_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lengthFunction, + .finalizeFunc = NULL + }, + { + .name = "char_length", + .type = FUNCTION_TYPE_CHAR_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = charLengthFunction, + .finalizeFunc = NULL + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, @@ -289,7 +309,67 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = concatFunction, + .finalizeFunc = NULL + }, + { + .name = "concat_ws", + .type = FUNCTION_TYPE_CONCAT_WS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = concatWsFunction, + .finalizeFunc = NULL + }, + { + .name = "lower", + .type = FUNCTION_TYPE_LOWER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lowerFunction, + .finalizeFunc = NULL + }, + { + .name = "upper", + .type = FUNCTION_TYPE_UPPER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = upperFunction, + .finalizeFunc = NULL + }, + { + .name = "ltrim", + .type = FUNCTION_TYPE_LTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ltrimFunction, + .finalizeFunc = NULL + }, + { + .name = "rtrim", + .type = FUNCTION_TYPE_RTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = rtrimFunction, + .finalizeFunc = NULL + }, + { + .name = "substr", + .type = FUNCTION_TYPE_SUBSTR, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = substrFunction, .finalizeFunc = NULL }, { @@ -361,6 +441,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = NULL, .sprocessFunc = winDurFunction, .finalizeFunc = NULL + }, + { + .name = "now", + .type = FUNCTION_TYPE_NOW, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = winDurFunction, + .finalizeFunc = NULL } }; @@ -399,12 +489,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; break; } - case FUNCTION_TYPE_CONCAT: - case FUNCTION_TYPE_ROWTS: - case FUNCTION_TYPE_TBNAME: { - // todo - break; - } case FUNCTION_TYPE_QENDTS: case FUNCTION_TYPE_QSTARTTS: @@ -424,6 +508,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_STDDEV: case FUNCTION_TYPE_SIN: case FUNCTION_TYPE_COS: case FUNCTION_TYPE_TAN: @@ -437,6 +522,67 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_LENGTH: + case FUNCTION_TYPE_CHAR_LENGTH: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT }; + break; + } + + case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT_WS: { + int32_t paraType, paraBytes = 0; + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + paraBytes += pParam->node.resType.bytes; + paraType = pParam->node.resType.type; + } + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + break; + //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; + //int32_t firstParamIndex = 0; + //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { + // firstParamIndex = 1; + // SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); + // sepBytes = pSep->node.resType.type; + //} + //for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { + // SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + // int32_t paraType = pParam->node.resType.type; + // if (i == firstParamIndex) { + // paraTypeFirst = paraType; + // } + // if (paraType != paraTypeFirst) { + // return TSDB_CODE_FAILED; + // } + // //TODO: for constants also needs numOfRows + // totalBytes += pParam->node.resType.bytes; + //} + ////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. + //totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; + //pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; + //break; + } + case FUNCTION_TYPE_LOWER: + case FUNCTION_TYPE_UPPER: + case FUNCTION_TYPE_LTRIM: + case FUNCTION_TYPE_RTRIM: + case FUNCTION_TYPE_SUBSTR: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + int32_t paraBytes = pParam->node.resType.bytes; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + break; + } + + case FUNCTION_TYPE_ROWTS: + case FUNCTION_TYPE_TBNAME: { + // todo + break; + } + + case FUNCTION_TYPE_NOW: + // todo + break; default: ASSERT(0); // to found the fault ASAP. } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 610e5a0bb2..0dc2989f77 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include "tpercentile.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -453,6 +454,7 @@ bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { } typedef struct SStddevRes { + double result; int64_t count; union {double quadraticDSum; int64_t quadraticISum;}; union {double dsum; int64_t isum;}; @@ -463,38 +465,129 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } +bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SStddevRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); + memset(pRes, 0, sizeof(SStddevRes)); + return true; +} + void stddevFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; - int32_t type = pInput->pData[0]->info.type; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); -// } else { // computing based on the true data block - SColumnInfoData* pCol = pInput->pData[0]; + // computing based on the true data block + SColumnInfoData* pCol = pInput->pData[0]; - int32_t start = pInput->startRowIndex; - int32_t numOfRows = pInput->numOfRows; + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; - switch(type) { - case TSDB_DATA_TYPE_INT: { - int32_t* plist = (int32_t*)pCol->pData; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; } + numOfElem += 1; pStddevRes->count += 1; - pStddevRes->isum += plist[i]; + pStddevRes->isum += plist[i]; pStddevRes->quadraticISum += plist[i] * plist[i]; } + + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; } break; } + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + default: + break; + } + // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); } @@ -503,11 +596,122 @@ void stddevFinalize(SqlFunctionCtx* pCtx) { functionFinalize(pCtx); SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - double res = pStddevRes->quadraticISum/pStddevRes->count - (pStddevRes->isum / pStddevRes->count) * (pStddevRes->isum / pStddevRes->count); + double avg = pStddevRes->isum / ((double) pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); } +typedef struct SPercentileInfo { + tMemBucket *pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; +} SPercentileInfo; +bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SPercentileInfo); + return true; +} +bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + // in the first round, get the min-max value of all involved data + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); + SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); + pInfo->numOfElems = 0; + + return true; +} + +void percentileFunction(SqlFunctionCtx *pCtx) { + int32_t notNullElems = 0; +#if 0 + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + pInfo->stage += 1; + + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + return; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + } + + // the first stage, only acquire the min/max value + if (pInfo->stage == 0) { + if (pCtx->preAggVals.isSet) { + double tmin = 0.0, tmax = 0.0; + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); + tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); + } else { + assert(true); + } + + if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { + SET_DOUBLE_VAL(&pInfo->minval, tmin); + } + + if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { + SET_DOUBLE_VAL(&pInfo->maxval, tmax); + } + + pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); + } else { + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + double v = 0; + GET_TYPED_DATA(v, double, pCtx->inputType, data); + + if (v < GET_DOUBLE_VAL(&pInfo->minval)) { + SET_DOUBLE_VAL(&pInfo->minval, v); + } + + if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { + SET_DOUBLE_VAL(&pInfo->maxval, v); + } + + pInfo->numOfElems += 1; + } + } + + return; + } + + // the second stage, calculate the true percentile value + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + notNullElems += 1; + tMemBucketPut(pInfo->pMemBucket, data, 1); + } + + SET_VAL(pCtx, notNullElems, 1); + pResInfo->hasResult = DATA_SET_FLAG; +#endif + +} bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 4034a0eb0f..5bd7f22987 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsPseudoColumnFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); +} + bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index af24906c9e..9174420ff4 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -176,26 +176,6 @@ typedef struct SResPair { double avg; } SResPair; -#define TSDB_BLOCK_DIST_STEP_ROWS 16 - -typedef struct STableBlockDist { - uint16_t rowSize; - uint16_t numOfFiles; - uint32_t numOfTables; - uint64_t totalSize; - uint64_t totalRows; - int32_t maxRows; - int32_t minRows; - int32_t firstSeekTimeUs; - uint32_t numOfRowsInMemTable; - uint32_t numOfSmallBlocks; - SArray *dataBlockInfos; -} STableBlockDist; - -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } @@ -3984,7 +3964,7 @@ static void irate_function(SqlFunctionCtx *pCtx) { } } -static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { +static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) { SBufferReader br = tbufInitReader(data, len, false); pDist->numOfTables = tbufReadUint32(&br); @@ -4024,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi static void blockInfo_func(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); @@ -4036,8 +4016,8 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) { //pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) { - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); +static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) { + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); pDist->numOfTables += pSrc->numOfTables; @@ -4071,7 +4051,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock } void block_func_merge(SqlFunctionCtx* pCtx) { - STableBlockDist info = {0}; + STableBlockDistInfo info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4082,7 +4062,7 @@ void block_func_merge(SqlFunctionCtx* pCtx) { //pResInfo->hasResult = DATA_SET_FLAG; } -void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, +void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, double* percents, int32_t* percentiles) { if (totalBlocks == 0) { for (int32_t i = 0; i < numOfPercents; ++i) { @@ -4117,7 +4097,7 @@ void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32 } } -void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { +void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) { if (pTableBlockDist == NULL) { return; } @@ -4178,7 +4158,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); pDist->rowSize = (uint16_t)pCtx->param[0].i; generateBlockDistResult(pDist, pCtx->pOutput); diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 24bc7a93a2..bc3ecea7a5 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -642,6 +642,9 @@ static const char* fstNodeState(FstNode* node) { } void fstNodeDestroy(FstNode* node) { + if (node == NULL) { + return; + } fstSliceDestroy(&node->data); taosMemoryFree(node); } @@ -1247,11 +1250,13 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); + StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; + node = NULL; + taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); - fstNodeDestroy(node); } else { // This is a little tricky. We're in this case if the // given bound is not a prefix of any key in the FST. @@ -1272,6 +1277,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { return true; } } + + fstNodeDestroy(node); + uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { StreamState* s = taosArrayGet(sws->stack, sz - 1); @@ -1349,7 +1357,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } - FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); + FstSlice slice = fstSliceCreate(buf, isz); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index eff53108cd..bfe1bb21bf 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -480,6 +480,15 @@ void checkFstCheckIteratorRange2() { assert(result.size() == 4); automCtxDestroy(ctx); } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "bb", GE, "ed", LT, result); + assert(result.size() == 3); + automCtxDestroy(ctx); + } { // range search std::vector result; @@ -510,6 +519,68 @@ void checkFstCheckIteratorRange2() { } delete m; } +void checkFstCheckIteratorRange3() { + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + // Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("ab", 1); + fw->Put("b", 2); + fw->Put("cdd", 3); + fw->Put("cde", 3); + fw->Put("ddd", 4); + fw->Put("ed", 5); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "b", GE, "", (RangeType)10, result); + assert(result.size() == 5); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result); + assert(result.size() == 1); + automCtxDestroy(ctx); + // taosMemoryFree(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + delete m; +} void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { @@ -578,6 +649,7 @@ int main(int argc, char* argv[]) { checkFstCheckIteratorPrefix(); checkFstCheckIteratorRange1(); checkFstCheckIteratorRange2(); + checkFstCheckIteratorRange3(); // checkFstLongTerm(); // checkFstPrefixSearch(); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index e728078a38..ac78988900 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); - COPY_SCALAR_FIELD(isDuration); - COPY_SCALAR_FIELD(translate); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - COPY_SCALAR_FIELD(datum.b); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - COPY_SCALAR_FIELD(datum.i); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - COPY_SCALAR_FIELD(datum.u); - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - COPY_SCALAR_FIELD(datum.d); - break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -206,6 +183,12 @@ static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode return (SNode*)pDst; } +static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + CLONE_NODE_LIST_FIELD(pNodeList); + return (SNode*)pDst; +} + static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); @@ -360,6 +343,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return orderByExprNodeCopy((const SOrderByExprNode*)pNode, (SOrderByExprNode*)pDst); case QUERY_NODE_LIMIT: break; + case QUERY_NODE_NODE_LIST: + return nodeListNodeCopy((const SNodeListNode*)pNode, (SNodeListNode*)pDst); case QUERY_NODE_FILL: return fillNodeCopy((const SFillNode*)pNode, (SFillNode*)pDst); case QUERY_NODE_DATABLOCK_DESC: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 32040a2e3f..1b16ba9206 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1491,6 +1491,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkValueGenByCalc = "GenByCalc"; static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; @@ -1544,6 +1545,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -1614,6 +1618,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -2015,6 +2022,31 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkFillMode = "Mode"; +static const char* jkFillValues = "Values"; + +static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { + const SFillNode* pNode = (const SFillNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkFillMode, pNode->mode); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues); + } + + return code; +} + +static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { + SFillNode* pNode = (SFillNode*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkFillMode, pNode->mode); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); + } + + return code; +} + static const char* jkTargetDataBlockId = "DataBlockId"; static const char* jkTargetSlotId = "SlotId"; static const char* jkTargetExpr = "Expr"; @@ -2328,6 +2360,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_NODE_LIST: return nodeListNodeToJson(pObj, pJson); case QUERY_NODE_FILL: + return fillNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: break; case QUERY_NODE_TARGET: @@ -2431,7 +2464,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToIntervalWindowNode(pJson, pObj); case QUERY_NODE_NODE_LIST: return jsonToNodeListNode(pJson, pObj); - // case QUERY_NODE_FILL: + case QUERY_NODE_FILL: + return jsonToFillNode(pJson, pObj); case QUERY_NODE_TARGET: return jsonToTargetNode(pJson, pObj); // case QUERY_NODE_RAW_EXPR: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index bbd0473edd..15c078d8d2 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { +void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } @@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe return DEAL_RES_CONTINUE; } -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); } -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); } @@ -290,20 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa switch (clause) { case SQL_CLAUSE_FROM: - nodesWalkNode(pSelect->pFromTable, walker, pContext); - nodesWalkNode(pSelect->pWhere, walker, pContext); + nodesWalkExpr(pSelect->pFromTable, walker, pContext); + nodesWalkExpr(pSelect->pWhere, walker, pContext); case SQL_CLAUSE_WHERE: - nodesWalkList(pSelect->pPartitionByList, walker, pContext); + nodesWalkExprs(pSelect->pPartitionByList, walker, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesWalkNode(pSelect->pWindow, walker, pContext); + nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: - nodesWalkList(pSelect->pGroupByList, walker, pContext); + nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: - nodesWalkNode(pSelect->pHaving, walker, pContext); + nodesWalkExpr(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pOrderByList, walker, pContext); + nodesWalkExprs(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: - nodesWalkList(pSelect->pProjectionList, walker, pContext); + nodesWalkExprs(pSelect->pProjectionList, walker, pContext); case SQL_CLAUSE_SELECT: default: break; @@ -319,20 +319,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit switch (clause) { case SQL_CLAUSE_FROM: - nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext); - nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext); case SQL_CLAUSE_WHERE: - nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: - nodesRewriteList(pSelect->pGroupByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: - nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); case SQL_CLAUSE_HAVING: - nodesRewriteList(pSelect->pProjectionList, rewriter, pContext); + nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); case SQL_CLAUSE_SELECT: - nodesRewriteList(pSelect->pOrderByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 732d9acfbe..90f06c6c9c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -860,6 +860,24 @@ bool nodesIsExprNode(const SNode* pNode) { return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); } +bool nodesIsUnaryOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_MINUS: + case OP_TYPE_IS_NULL: + case OP_TYPE_IS_NOT_NULL: + case OP_TYPE_IS_TRUE: + case OP_TYPE_IS_FALSE: + case OP_TYPE_IS_UNKNOWN: + case OP_TYPE_IS_NOT_TRUE: + case OP_TYPE_IS_NOT_FALSE: + case OP_TYPE_IS_NOT_UNKNOWN: + return true; + default: + break; + } + return false; +} + bool nodesIsArithmeticOp(const SOperatorNode* pOp) { switch (pOp->opType) { case OP_TYPE_ADD: diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp index fd4a9e8c20..fdfb9522db 100644 --- a/source/libs/nodes/test/nodesTest.cpp +++ b/source/libs/nodes/test/nodesTest.cpp @@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) { EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); EDealRes res = DEAL_RES_CONTINUE; - nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res); EXPECT_EQ(res, DEAL_RES_CONTINUE); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5d02868657..c3157480a9 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function transport qcom + PRIVATE os util nodes catalog function scalar transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index dbf7d379ec..ee17de50e0 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -77,8 +77,7 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; bool cloned; - STagData tagData; - + int32_t createTbReqLen; SParsedDataColInfo boundColumnInfo; SRowBuilder rowBuilder; } STableDataBlocks; @@ -119,6 +118,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; + pBlocks->schemaLen = dataBuf->createTbReqLen; if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -137,7 +137,7 @@ void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 9bad3e9eb9..ea01caf9d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -23,9 +23,10 @@ extern "C" { #include "parser.h" int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 29b1d8857a..4ae33e5c5c 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -513,7 +513,7 @@ expression(A) ::= NK_PLUS(B) expression(C). } expression(A) ::= NK_MINUS(B) expression(C). { SToken t = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL)); + A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, C), NULL)); } expression(A) ::= expression(B) NK_PLUS expression(C). { SToken s = getTokenFromRawExprNode(pCxt, B); @@ -549,38 +549,14 @@ expression_list(A) ::= expression_list(B) NK_COMMA expression(C). column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } -//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } -pseudo_column(A) ::= NK_UNDERLINE(B) ROWTS(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } +pseudo_column(A) ::= NOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= ROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } -pseudo_column(A) ::= NK_UNDERLINE(B) QSTARTTS(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } -pseudo_column(A) ::= NK_UNDERLINE(B) QENDTS(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } -pseudo_column(A) ::= NK_UNDERLINE(B) WSTARTTS(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } -pseudo_column(A) ::= NK_UNDERLINE(B) WENDTS(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } -pseudo_column(A) ::= NK_UNDERLINE(B) WDURATION(C). { - SToken t = B; - t.n = (C.z + C.n) - B.z; - A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } +pseudo_column(A) ::= QSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= QENDTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= WSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= WENDTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } /************************************************ predicate ***********************************************************/ predicate(A) ::= expression(B) compare_op(C) expression(D). { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 18a0ae8cfa..07316ddd39 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -243,27 +243,27 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } - char val[20] = {0}; - int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val)); - char* pStart = val; - char* pEnd = val + len; - int32_t sepOrder = 1; - while (1) { - char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':'); - SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { - pCxt->valid = false; - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); - return pOptions; + if (pVal->n > 2) { + char* pStart = pVal->z + 1; + char* pEnd = pVal->z + pVal->n - 1; + int32_t sepOrder = 1; + while (1) { + char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':'); + SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { + pCxt->valid = false; + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); + return pOptions; + } + if (NULL == pPos) { + break; + } + pStart = pPos + 1; } - if (NULL == pPos) { - break; - } - pStart = pPos + 1; } - if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val); + if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z); pCxt->valid = false; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5c38ccaff8..76af6b7ac7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*); extern void ParseFree(void*, FFree); extern void ParseTrace(FILE*, char*); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c new file mode 100644 index 0000000000..46cdcc4243 --- /dev/null +++ b/source/libs/parser/src/parCalcConst.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "functionMgt.h" +#include "parInt.h" +#include "scalar.h" + +typedef struct SCalcConstContext { + int32_t code; +} SCalcConstContext; + +static int32_t calcConstQuery(SNode* pStmt); + +static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { + SNode* pNew = NULL; + pCxt->code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS != pCxt->code) { + return DEAL_RES_ERROR; + } + ((SValueNode*)pNew)->genByCalc = true; + ((SValueNode*)pNew)->translate = true; + *pNode = pNew; + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { + SOperatorNode* pOp = *pNode; + if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); + } + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { + SFunctionNode* pFunc = *pNode; + if (fmIsPseudoColumnFunc(pFunc->funcId)) { + return DEAL_RES_CONTINUE; + } + SNode* pParam = NULL; + FOREACH(pParam, pFunc->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) { + SLogicConditionNode* pCond = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pCond->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { + SCalcConstContext* pCxt = pContext; + pCxt->code = calcConstQuery((*pNode)->pSubquery); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes calcConst(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + return calcConstOperator((SOperatorNode**)pNode, pContext); + case QUERY_NODE_FUNCTION: + return calcConstFunction((SFunctionNode**)pNode, pContext); + case QUERY_NODE_LOGIC_CONDITION: + return calcConstLogicCond((SLogicConditionNode**)pNode, pContext); + case QUERY_NODE_TEMP_TABLE: + return calcConstSubquery((STempTableNode**)pNode, pContext); + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t calcConstSelect(SSelectStmt* pSelect) { + SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; + nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); + } + return cxt.code; +} + +static int32_t calcConstQuery(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return calcConstSelect((SSelectStmt*)pStmt); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { + return calcConstQuery(pQuery->pRoot); +} diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 11b552fb94..d9cf0b39b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -55,8 +55,10 @@ typedef struct SInsertParseContext { STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global + SHashObj* pSubTableHashObj; // global SArray* pTableDataBlocks; // global SArray* pVgDataBlocks; // global int32_t totalNum; @@ -738,8 +740,20 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, SKVRow row) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + pCxt->createTblReq.type = TD_CHILD_TABLE; + pCxt->createTblReq.dbFName = strdup(dbFName); + pCxt->createTblReq.name = strdup(pName->tname); + pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid; + pCxt->createTblReq.ctbCfg.pTag = row; + + return TSDB_CODE_SUCCESS; +} + // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -749,9 +763,9 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; - param.schema = pSchema; - CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); @@ -760,23 +774,47 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, } tdSortKVRowByColIdx(row); - // todo construct payload + return buildCreateTbReq(pCxt, pName, row); +} - taosMemoryFreeClear(row); +static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { + *pDst = taosMemoryMalloc(TABLE_META_SIZE(pSrc)); + if (NULL == *pDst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(*pDst, pSrc, TABLE_META_SIZE(pSrc)); + return TSDB_CODE_SUCCESS; +} - return 0; +static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) { + STableMeta* pBackup = NULL; + if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBackup->uid = tGenIdPI64(); + return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { - SToken sToken; + SName name; + createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg); + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&name, tbFName); + int32_t len = strlen(tbFName); + STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len); + if (NULL != pMeta) { + return cloneTableMeta(*pMeta, &pCxt->pTableMeta); + } + SToken sToken; // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); CHECK_CODE(getTableMeta(pCxt, &sToken)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } + CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -796,7 +834,11 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_NK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); + } return TSDB_CODE_SUCCESS; } @@ -904,10 +946,17 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } +static void destroyCreateSubTbReq(SVCreateTbReq* pReq) { + taosMemoryFreeClear(pReq->dbFName); + taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->ctbCfg.pTag); +} + static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroyKVRowBuilder(&pCxt->tagsBuilder); + destroyCreateSubTbReq(&pCxt->createTblReq); } static void destroyDataBlock(STableDataBlocks* pDataBlock) { @@ -971,8 +1020,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); - + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); + if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); @@ -1002,7 +1051,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); } @@ -1021,11 +1070,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pTableMeta = NULL, .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), .totalNum = 0, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT) }; - if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) { + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || + NULL == context.pSubTableHashObj || NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index f70e514b5a..088b25d544 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -149,8 +149,28 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { + int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); + if (pBlocks->nAllocSize - pBlocks->size < len) { + pBlocks->nAllocSize += len + pBlocks->rowSize; + char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); + if (pTmp != NULL) { + pBlocks->pData = pTmp; + memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + } else { + pBlocks->nAllocSize -= len + pBlocks->rowSize; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + char* pBuf = pBlocks->pData + pBlocks->size; + tSerializeSVCreateTbReq((void**)&pBuf, pCreateTbReq); + pBlocks->size += len; + pBlocks->createTbReqLen = len; + return TSDB_CODE_SUCCESS; +} + int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -163,6 +183,13 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 return ret; } + if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctbCfg.pTag) { + ret = buildCreateTbMsg(*dataBlocks, pCreateTbReq); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + } + taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); @@ -294,7 +321,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey int32_t extendedRowSize = getExtendedRowSize(dataBuf); SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; + char * pBlockData = pBlocks->data + pBlocks->schemaLen; int n = 0; while (n < nRows) { pBlkKeyTuple->skey = TD_ROW_KEY((STSRow *)pBlockData); @@ -340,44 +367,26 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey } // Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, bool isRawPayload) { // TODO: optimize this function, handle the case while binary is not presented STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = getTableInfo(pTableMeta); SSchema* pSchema = getTableColumnSchema(pTableMeta); + int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); - pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); + memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); + pDataBlock = (char*)pDataBlock + nonDataLen; int32_t flen = 0; // original total length of row - - // schema needs to be included into the submit data block - if (schemaAttached) { - int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta); - for(int32_t j = 0; j < numOfCols; ++j) { - STColumn* pCol = (STColumn*) pDataBlock; - pCol->colId = htons(pSchema[j].colId); - pCol->type = pSchema[j].type; - pCol->bytes = htons(pSchema[j].bytes); - pCol->offset = 0; - - pDataBlock = (char*)pDataBlock + sizeof(STColumn); + if (isRawPayload) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { flen += TYPE_BYTES[pSchema[j].type]; } - - int32_t schemaSize = sizeof(STColumn) * numOfCols; - pBlock->schemaLen = schemaSize; - } else { - if (isRawPayload) { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; - } - } - pBlock->schemaLen = 0; } + pBlock->schemaLen = pTableDataBlock->createTbReqLen; - char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + char* p = pTableDataBlock->pData + nonDataLen; pBlock->dataLen = 0; int32_t numOfRows = pBlock->numOfRows; @@ -414,7 +423,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB return pBlock->dataLen + pBlock->schemaLen; } -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -429,7 +438,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); @@ -474,7 +483,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index be2bd0d0b6..21191563e1 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -125,10 +125,10 @@ static SKeyword keywordTable[] = { {"PRECISION", TK_PRECISION}, {"PRIVILEGE", TK_PRIVILEGE}, {"PREV", TK_PREV}, - {"QENDTS", TK_QENDTS}, + {"_QENDTS", TK_QENDTS}, {"QNODE", TK_QNODE}, {"QNODES", TK_QNODES}, - {"QSTARTTS", TK_QSTARTTS}, + {"_QSTARTTS", TK_QSTARTTS}, {"QTIME", TK_QTIME}, {"QUERIES", TK_QUERIES}, {"QUERY", TK_QUERY}, @@ -138,7 +138,7 @@ static SKeyword keywordTable[] = { {"RESET", TK_RESET}, {"RETENTIONS", TK_RETENTIONS}, {"ROLLUP", TK_ROLLUP}, - {"ROWTS", TK_ROWTS}, + {"_ROWTS", TK_ROWTS}, {"SCORES", TK_SCORES}, {"SELECT", TK_SELECT}, {"SESSION", TK_SESSION}, @@ -176,6 +176,7 @@ static SKeyword keywordTable[] = { {"USER", TK_USER}, {"USERS", TK_USERS}, {"USING", TK_USING}, + {"VALUE", TK_VALUE}, {"VALUES", TK_VALUES}, {"VARCHAR", TK_VARCHAR}, {"VARIABLES", TK_VARIABLES}, @@ -183,10 +184,10 @@ static SKeyword keywordTable[] = { {"VGROUPS", TK_VGROUPS}, {"VNODES", TK_VNODES}, {"WAL", TK_WAL}, - {"WDURATION", TK_WDURATION}, - {"WENDTS", TK_WENDTS}, + {"_WDURATION", TK_WDURATION}, + {"_WENDTS", TK_WENDTS}, {"WHERE", TK_WHERE}, - {"WSTARTTS", TK_WSTARTTS}, + {"_WSTARTTS", TK_WSTARTTS}, // {"ID", TK_ID}, // {"STRING", TK_STRING}, // {"EQ", TK_EQ}, @@ -439,10 +440,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { *tokenId = TK_NK_QUESTION; return 1; } - case '_': { - *tokenId = TK_NK_UNDERLINE; - return 1; - } case '`': case '\'': case '"': { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2947d25e1f..7018c20eb3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { - va_list vArgList; - va_start(vArgList, errCode); - generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList); - va_end(vArgList); - pCxt->errCode = errCode; - return DEAL_RES_ERROR; -} +#define generateDealNodeErrMsg(pCxt, code, ...) \ + ({ \ + generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ + pCxt->errCode = code; \ + DEAL_RES_ERROR; \ + }) static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -439,6 +437,12 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { } static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + if (nodesIsUnaryOp(pOp)) { + if (OP_TYPE_MINUS == pOp->opType && !IS_NUMERIC_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } + return DEAL_RES_CONTINUE; + } SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; if (nodesIsArithmeticOp(pOp)) { @@ -507,12 +511,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt); return pCxt->errCode; } static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { - nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt); return pCxt->errCode; } @@ -567,7 +571,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -575,7 +579,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -602,9 +606,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; - nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (cxt.existAggFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); @@ -2152,10 +2156,11 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { +static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) { pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = calcTypeBytes(pCol->dataType); + pSchema->sma = TSDB_BSMA_TYPE_LATEST; // TODO: use default value currently, and use the real value later. strcpy(pSchema->name, pCol->colName); } @@ -2177,7 +2182,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ntbCfg.nCols = LIST_LENGTH(pColumns); - req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); + req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -2188,6 +2193,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index); ++index; } + // TODO: use the real sma for normal table. pBatch->info = *pVgroupInfo; strcpy(pBatch->dbName, pDbName); @@ -2630,7 +2636,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = { .pParseCxt = pParseCxt, .errCode = TSDB_CODE_SUCCESS, diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d410aa2e17..ebe76cc129 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) { } static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { - int32_t code = doParse(pCxt, pQuery); + int32_t code = parse(pCxt, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = doTranslate(pCxt, *pQuery); + code = translate(pCxt, *pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = calculateConstant(pCxt, *pQuery); } return code; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 618e80cbfe..86e564adab 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -132,17 +132,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 512 -#define YYNRULE 389 +#define YYNSTATE 511 +#define YYNRULE 390 #define YYNTOKEN 201 -#define YY_MAX_SHIFT 511 +#define YY_MAX_SHIFT 510 #define YY_MIN_SHIFTREDUCE 762 -#define YY_MAX_SHIFTREDUCE 1150 -#define YY_ERROR_ACTION 1151 -#define YY_ACCEPT_ACTION 1152 -#define YY_NO_ACTION 1153 -#define YY_MIN_REDUCE 1154 -#define YY_MAX_REDUCE 1542 +#define YY_MAX_SHIFTREDUCE 1151 +#define YY_ERROR_ACTION 1152 +#define YY_ACCEPT_ACTION 1153 +#define YY_NO_ACTION 1154 +#define YY_MIN_REDUCE 1155 +#define YY_MAX_REDUCE 1544 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -209,297 +209,300 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1412) +#define YY_ACTTAB_COUNT (1448) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1398, 24, 191, 442, 442, 504, 503, 253, 1307, 72, - /* 10 */ 72, 270, 1394, 1401, 252, 307, 349, 355, 233, 1305, - /* 20 */ 1010, 1197, 1155, 442, 1398, 1260, 1260, 319, 1398, 305, - /* 30 */ 331, 31, 29, 27, 26, 25, 1394, 1400, 429, 332, - /* 40 */ 1394, 1400, 234, 84, 1348, 1260, 83, 82, 81, 80, - /* 50 */ 79, 78, 77, 76, 75, 273, 31, 29, 27, 26, - /* 60 */ 25, 429, 1338, 1340, 265, 213, 236, 1347, 1290, 441, - /* 70 */ 496, 495, 494, 493, 492, 491, 490, 489, 207, 486, - /* 80 */ 485, 484, 483, 482, 481, 480, 479, 478, 998, 1026, - /* 90 */ 31, 29, 27, 26, 25, 84, 441, 1056, 83, 82, - /* 100 */ 81, 80, 79, 78, 77, 76, 75, 330, 1154, 442, - /* 110 */ 325, 324, 323, 322, 321, 306, 318, 317, 316, 315, - /* 120 */ 311, 310, 309, 308, 442, 236, 27, 26, 25, 340, - /* 130 */ 312, 1260, 93, 92, 91, 90, 89, 88, 87, 86, - /* 140 */ 85, 1428, 362, 1057, 357, 12, 1260, 361, 426, 1012, - /* 150 */ 160, 442, 358, 356, 1001, 359, 1056, 313, 404, 399, - /* 160 */ 410, 1061, 874, 465, 464, 463, 878, 462, 880, 881, - /* 170 */ 461, 883, 458, 1260, 889, 455, 891, 892, 452, 449, - /* 180 */ 400, 126, 98, 1010, 23, 260, 1051, 1052, 1053, 1054, - /* 190 */ 1055, 1059, 1060, 20, 112, 1412, 300, 1013, 299, 219, - /* 200 */ 385, 266, 1057, 31, 29, 27, 26, 25, 442, 111, - /* 210 */ 111, 217, 405, 401, 339, 96, 1428, 1262, 1263, 292, - /* 220 */ 1061, 136, 1251, 426, 412, 121, 1467, 1468, 1177, 1472, - /* 230 */ 1260, 477, 301, 428, 294, 126, 113, 1385, 1166, 1412, - /* 240 */ 1011, 386, 414, 23, 260, 1051, 1052, 1053, 1054, 1055, - /* 250 */ 1059, 1060, 126, 68, 1413, 1414, 1417, 1460, 30, 28, - /* 260 */ 1428, 235, 1456, 1521, 1385, 442, 262, 426, 990, 468, - /* 270 */ 1385, 1257, 1521, 1521, 1249, 1412, 125, 428, 990, 159, - /* 280 */ 1519, 1385, 67, 352, 988, 125, 125, 1260, 1117, 1519, - /* 290 */ 1519, 1374, 49, 11, 988, 6, 1428, 70, 1413, 1414, - /* 300 */ 1417, 1460, 95, 413, 354, 1459, 1456, 45, 44, 304, - /* 310 */ 1255, 130, 1245, 428, 1200, 1, 298, 1385, 396, 1115, - /* 320 */ 1116, 1118, 1119, 410, 242, 442, 290, 284, 286, 282, - /* 330 */ 127, 439, 441, 69, 1413, 1414, 1417, 1460, 508, 126, - /* 340 */ 326, 255, 1456, 120, 1428, 98, 1247, 1260, 508, 49, - /* 350 */ 989, 426, 126, 442, 1014, 187, 30, 28, 1093, 440, - /* 360 */ 989, 392, 1487, 362, 262, 357, 990, 1256, 361, 30, - /* 370 */ 28, 160, 1412, 358, 356, 1260, 359, 262, 96, 990, - /* 380 */ 1243, 12, 988, 403, 991, 135, 134, 1176, 122, 1467, - /* 390 */ 1468, 11, 1472, 1428, 991, 988, 9, 8, 117, 1238, - /* 400 */ 413, 994, 995, 224, 11, 1039, 114, 272, 1224, 1300, - /* 410 */ 428, 994, 995, 1, 1385, 111, 1412, 31, 29, 27, - /* 420 */ 26, 25, 126, 1262, 65, 1412, 1, 1088, 1152, 1385, - /* 430 */ 69, 1413, 1414, 1417, 1460, 99, 508, 1428, 255, 1456, - /* 440 */ 120, 1307, 1252, 1521, 426, 340, 1428, 267, 989, 508, - /* 450 */ 165, 1307, 1305, 426, 428, 1058, 1520, 274, 1385, 1488, - /* 460 */ 1519, 989, 1305, 428, 186, 1069, 1038, 1385, 1040, 1041, - /* 470 */ 1042, 1043, 1044, 1062, 227, 1413, 1414, 1417, 277, 467, - /* 480 */ 30, 28, 991, 69, 1413, 1414, 1417, 1460, 262, 1412, - /* 490 */ 990, 255, 1456, 1533, 131, 991, 21, 1175, 1236, 994, - /* 500 */ 995, 224, 1494, 1039, 1412, 384, 988, 275, 417, 1521, - /* 510 */ 1428, 1174, 994, 995, 224, 111, 1039, 426, 47, 410, - /* 520 */ 1015, 46, 125, 1262, 1307, 1428, 1519, 428, 1173, 798, - /* 530 */ 1092, 1385, 426, 1412, 1474, 1339, 1147, 7, 799, 1385, - /* 540 */ 798, 98, 428, 1167, 477, 347, 1385, 69, 1413, 1414, - /* 550 */ 1417, 1460, 1471, 1385, 1428, 255, 1456, 1533, 800, 188, - /* 560 */ 508, 426, 230, 1413, 1414, 1417, 1517, 1100, 1225, 1172, - /* 570 */ 1385, 428, 989, 1012, 96, 1385, 442, 1171, 1474, 30, - /* 580 */ 28, 427, 205, 1335, 123, 1467, 1468, 262, 1472, 990, - /* 590 */ 133, 69, 1413, 1414, 1417, 1460, 1470, 1170, 1260, 255, - /* 600 */ 1456, 1533, 1146, 1474, 22, 988, 991, 1412, 1169, 376, - /* 610 */ 1478, 1385, 30, 28, 31, 29, 27, 26, 25, 1385, - /* 620 */ 262, 1469, 990, 994, 995, 224, 442, 1039, 1428, 58, - /* 630 */ 172, 1168, 276, 1165, 1164, 426, 7, 1163, 988, 1385, - /* 640 */ 1521, 1162, 1161, 487, 126, 428, 295, 1253, 1260, 1385, - /* 650 */ 1385, 269, 268, 125, 414, 1479, 1088, 1519, 1160, 508, - /* 660 */ 418, 1003, 30, 28, 511, 225, 1413, 1414, 1417, 7, - /* 670 */ 262, 989, 990, 1385, 421, 1385, 1385, 996, 210, 1385, - /* 680 */ 397, 94, 1307, 1385, 1385, 1521, 1159, 500, 988, 209, - /* 690 */ 1158, 1157, 508, 1306, 9, 8, 152, 416, 125, 150, - /* 700 */ 1385, 1301, 1519, 1091, 989, 991, 1412, 154, 156, 158, - /* 710 */ 153, 155, 157, 66, 1193, 1188, 203, 1186, 104, 1, - /* 720 */ 425, 388, 994, 995, 224, 64, 1039, 1428, 1385, 1149, - /* 730 */ 1150, 443, 1385, 1385, 426, 60, 363, 365, 991, 368, - /* 740 */ 181, 1405, 508, 999, 428, 374, 438, 43, 1385, 175, - /* 750 */ 1114, 997, 177, 1403, 989, 994, 995, 224, 372, 1039, - /* 760 */ 410, 346, 1412, 411, 70, 1413, 1414, 1417, 1460, 1490, - /* 770 */ 1429, 391, 424, 1456, 168, 32, 32, 1004, 1063, 1023, - /* 780 */ 190, 1010, 98, 1428, 2, 279, 283, 419, 991, 974, - /* 790 */ 426, 164, 241, 841, 1007, 995, 32, 243, 966, 957, - /* 800 */ 428, 414, 314, 422, 1385, 994, 995, 224, 211, 1039, - /* 810 */ 1412, 194, 132, 101, 196, 96, 434, 1000, 1337, 320, - /* 820 */ 115, 1413, 1414, 1417, 328, 184, 1467, 409, 1048, 408, - /* 830 */ 333, 1428, 1521, 31, 29, 27, 26, 25, 426, 102, - /* 840 */ 1019, 246, 202, 104, 147, 125, 867, 119, 428, 1519, - /* 850 */ 327, 43, 1385, 345, 862, 146, 329, 447, 415, 1534, - /* 860 */ 895, 334, 102, 159, 1412, 899, 335, 352, 70, 1413, - /* 870 */ 1414, 1417, 1460, 1412, 1018, 336, 139, 1457, 247, 50, - /* 880 */ 245, 244, 144, 351, 103, 1428, 337, 905, 354, 1017, - /* 890 */ 104, 1026, 426, 904, 1428, 102, 338, 142, 105, 48, - /* 900 */ 341, 426, 428, 145, 1016, 348, 1385, 1412, 350, 261, - /* 910 */ 377, 428, 353, 1250, 149, 1385, 74, 1246, 393, 151, - /* 920 */ 106, 107, 229, 1413, 1414, 1417, 1248, 1244, 1428, 108, - /* 930 */ 109, 229, 1413, 1414, 1417, 426, 143, 360, 138, 251, - /* 940 */ 140, 379, 378, 1412, 167, 428, 170, 1015, 389, 1385, - /* 950 */ 398, 1412, 383, 1501, 367, 390, 995, 137, 432, 1500, - /* 960 */ 5, 387, 380, 1481, 1428, 228, 1413, 1414, 1417, 375, - /* 970 */ 407, 426, 1428, 173, 395, 4, 254, 1412, 176, 426, - /* 980 */ 1491, 428, 402, 162, 394, 1385, 370, 1088, 97, 428, - /* 990 */ 1014, 364, 1237, 1385, 161, 33, 259, 406, 1428, 1235, - /* 1000 */ 1475, 115, 1413, 1414, 1417, 426, 1412, 118, 180, 229, - /* 1010 */ 1413, 1414, 1417, 256, 183, 428, 1518, 182, 41, 1385, - /* 1020 */ 423, 40, 263, 420, 189, 1536, 17, 1428, 1442, 1346, - /* 1030 */ 1412, 430, 431, 264, 426, 229, 1413, 1414, 1417, 1345, - /* 1040 */ 1535, 435, 436, 200, 428, 198, 437, 57, 1385, 206, - /* 1050 */ 212, 1428, 59, 473, 1412, 474, 206, 488, 426, 1261, - /* 1060 */ 473, 445, 214, 507, 222, 1413, 1414, 1417, 428, 208, - /* 1070 */ 220, 39, 1385, 216, 475, 1428, 1379, 221, 1412, 218, - /* 1080 */ 1378, 475, 426, 278, 1375, 280, 281, 984, 231, 1413, - /* 1090 */ 1414, 1417, 428, 472, 471, 470, 1385, 469, 985, 1428, - /* 1100 */ 472, 471, 470, 128, 469, 285, 426, 1412, 1373, 287, - /* 1110 */ 288, 289, 223, 1413, 1414, 1417, 428, 1372, 1371, 291, - /* 1120 */ 1385, 293, 1362, 129, 296, 297, 969, 968, 1428, 1356, - /* 1130 */ 1355, 303, 302, 1412, 1354, 426, 232, 1413, 1414, 1417, - /* 1140 */ 1353, 1412, 940, 1330, 1329, 428, 1328, 1327, 1326, 1385, - /* 1150 */ 1325, 1324, 1323, 1322, 1428, 1321, 1320, 1319, 1318, 100, - /* 1160 */ 1317, 426, 1428, 1316, 1315, 1425, 1413, 1414, 1417, 426, - /* 1170 */ 1314, 428, 1313, 1312, 1311, 1385, 1412, 1310, 1309, 428, - /* 1180 */ 1308, 942, 1412, 1385, 1199, 1370, 1364, 1352, 1343, 1239, - /* 1190 */ 811, 1424, 1413, 1414, 1417, 1198, 1196, 1428, 1185, 1423, - /* 1200 */ 1413, 1414, 1417, 1428, 426, 141, 342, 343, 344, 1184, - /* 1210 */ 426, 1412, 1181, 1241, 428, 73, 487, 1412, 1385, 912, - /* 1220 */ 428, 148, 910, 1240, 1385, 1194, 248, 1189, 840, 839, - /* 1230 */ 838, 837, 1428, 249, 239, 1413, 1414, 1417, 1428, 426, - /* 1240 */ 238, 1413, 1414, 1417, 835, 426, 834, 1187, 250, 428, - /* 1250 */ 1180, 371, 369, 1385, 366, 428, 1179, 373, 71, 1385, - /* 1260 */ 1369, 42, 163, 1412, 1363, 381, 976, 110, 382, 240, - /* 1270 */ 1413, 1414, 1417, 1351, 1350, 237, 1413, 1414, 1417, 1342, - /* 1280 */ 169, 166, 14, 1403, 1428, 51, 171, 3, 32, 174, - /* 1290 */ 15, 426, 1113, 116, 34, 54, 36, 37, 185, 124, - /* 1300 */ 178, 428, 1107, 52, 1135, 1385, 1106, 179, 53, 19, - /* 1310 */ 1134, 1085, 35, 10, 1084, 16, 1140, 8, 257, 1139, - /* 1320 */ 1138, 226, 1413, 1414, 1417, 258, 1341, 1024, 13, 433, - /* 1330 */ 1049, 18, 192, 193, 60, 1111, 195, 197, 55, 199, - /* 1340 */ 56, 1005, 1402, 446, 38, 271, 450, 896, 893, 453, - /* 1350 */ 456, 204, 448, 451, 459, 890, 444, 454, 201, 884, - /* 1360 */ 882, 457, 460, 873, 901, 907, 903, 61, 809, 476, - /* 1370 */ 62, 63, 831, 830, 829, 828, 827, 826, 825, 823, - /* 1380 */ 888, 466, 824, 842, 821, 820, 819, 818, 887, 886, - /* 1390 */ 817, 885, 816, 815, 814, 1195, 497, 498, 1183, 1182, - /* 1400 */ 501, 502, 499, 1178, 906, 505, 506, 1153, 992, 215, - /* 1410 */ 509, 510, + /* 0 */ 1250, 441, 441, 265, 441, 1246, 1413, 72, 304, 306, + /* 10 */ 72, 111, 30, 28, 348, 409, 252, 354, 20, 1263, + /* 20 */ 261, 1153, 990, 1261, 1261, 269, 1261, 1429, 31, 29, + /* 30 */ 27, 26, 25, 1399, 412, 441, 233, 98, 988, 117, + /* 40 */ 409, 305, 1399, 1014, 427, 1395, 1401, 11, 1386, 1429, + /* 50 */ 1301, 30, 28, 1094, 1395, 1401, 425, 1261, 799, 261, + /* 60 */ 798, 990, 98, 409, 69, 1414, 1415, 1418, 1462, 1, + /* 70 */ 96, 276, 254, 1458, 120, 1413, 1308, 988, 800, 411, + /* 80 */ 121, 1469, 1470, 1308, 1474, 98, 11, 1340, 399, 251, + /* 90 */ 30, 28, 507, 1490, 1306, 96, 1429, 1252, 261, 325, + /* 100 */ 990, 440, 1523, 425, 989, 122, 1469, 1470, 1, 1474, + /* 110 */ 22, 24, 191, 427, 1399, 125, 988, 1386, 96, 1521, + /* 120 */ 31, 29, 27, 26, 25, 11, 1395, 1402, 123, 1469, + /* 130 */ 1470, 507, 1474, 70, 1414, 1415, 1418, 1462, 991, 1386, + /* 140 */ 440, 1461, 1458, 989, 135, 134, 1413, 1, 31, 29, + /* 150 */ 27, 26, 25, 186, 65, 994, 995, 1038, 1039, 1040, + /* 160 */ 1041, 1042, 1043, 1044, 1045, 99, 232, 1429, 1010, 1406, + /* 170 */ 507, 299, 1253, 298, 425, 318, 126, 991, 330, 1010, + /* 180 */ 428, 1404, 989, 264, 427, 1429, 1348, 331, 1386, 12, + /* 190 */ 1413, 398, 425, 131, 994, 995, 1038, 1039, 1040, 1041, + /* 200 */ 1042, 1043, 1044, 1045, 69, 1414, 1415, 1418, 1462, 441, + /* 210 */ 126, 1429, 254, 1458, 1535, 311, 991, 47, 425, 1308, + /* 220 */ 46, 441, 1336, 1496, 402, 266, 440, 312, 427, 133, + /* 230 */ 1306, 1261, 1386, 994, 995, 1038, 1039, 1040, 1041, 1042, + /* 240 */ 1043, 1044, 1045, 1261, 404, 400, 30, 28, 70, 1414, + /* 250 */ 1415, 1418, 1462, 64, 261, 329, 990, 1459, 324, 323, + /* 260 */ 322, 321, 320, 60, 317, 316, 315, 314, 310, 309, + /* 270 */ 308, 307, 988, 84, 1118, 12, 83, 82, 81, 80, + /* 280 */ 79, 78, 77, 76, 75, 30, 28, 426, 1156, 361, + /* 290 */ 111, 356, 300, 261, 360, 990, 126, 160, 1264, 357, + /* 300 */ 355, 126, 358, 7, 395, 1116, 1117, 1119, 1120, 84, + /* 310 */ 6, 988, 83, 82, 81, 80, 79, 78, 77, 76, + /* 320 */ 75, 441, 1308, 1523, 30, 28, 507, 338, 273, 503, + /* 330 */ 502, 441, 261, 1306, 990, 126, 125, 1258, 989, 114, + /* 340 */ 1521, 1225, 7, 1261, 272, 31, 29, 27, 26, 25, + /* 350 */ 988, 1339, 1341, 1261, 874, 464, 463, 462, 878, 461, + /* 360 */ 880, 881, 460, 883, 457, 507, 889, 454, 891, 892, + /* 370 */ 451, 448, 991, 27, 26, 25, 113, 989, 1167, 213, + /* 380 */ 1413, 7, 1291, 1375, 31, 29, 27, 26, 25, 994, + /* 390 */ 995, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 441, + /* 400 */ 1523, 1429, 1201, 1026, 507, 438, 339, 384, 425, 1239, + /* 410 */ 126, 991, 1178, 1522, 428, 1012, 989, 1521, 427, 283, + /* 420 */ 1349, 1261, 1386, 31, 29, 27, 26, 25, 994, 995, + /* 430 */ 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 115, 1414, + /* 440 */ 1415, 1418, 1089, 31, 29, 27, 26, 25, 385, 271, + /* 450 */ 991, 361, 1015, 356, 1386, 339, 360, 111, 235, 160, + /* 460 */ 245, 357, 355, 1177, 358, 1263, 1413, 994, 995, 1038, + /* 470 */ 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1537, 235, 1523, + /* 480 */ 30, 28, 159, 9, 8, 403, 351, 1429, 261, 1057, + /* 490 */ 990, 1155, 125, 49, 425, 291, 1521, 246, 375, 244, + /* 500 */ 243, 1026, 350, 1148, 427, 1386, 988, 353, 1386, 1057, + /* 510 */ 293, 1257, 1176, 413, 1175, 93, 92, 91, 90, 89, + /* 520 */ 88, 87, 86, 85, 68, 1414, 1415, 1418, 1462, 1523, + /* 530 */ 49, 274, 234, 1458, 159, 1058, 416, 1, 351, 111, + /* 540 */ 95, 1093, 125, 1174, 1523, 441, 1521, 1263, 1256, 1013, + /* 550 */ 1059, 439, 1070, 1062, 1386, 1058, 1386, 125, 476, 353, + /* 560 */ 507, 1521, 172, 1308, 441, 441, 1173, 1261, 1063, 1147, + /* 570 */ 205, 275, 989, 1062, 1307, 1172, 23, 259, 1052, 1053, + /* 580 */ 1054, 1055, 1056, 1060, 1061, 1386, 1261, 1261, 1011, 1476, + /* 590 */ 1194, 21, 1171, 1170, 1169, 1166, 23, 259, 1052, 1053, + /* 600 */ 1054, 1055, 1056, 1060, 1061, 1198, 991, 1473, 1386, 1101, + /* 610 */ 1165, 147, 362, 1248, 119, 1012, 112, 1386, 1481, 1089, + /* 620 */ 344, 219, 146, 994, 995, 1038, 1039, 1040, 1041, 1042, + /* 630 */ 1043, 1044, 1045, 217, 1386, 1386, 1386, 1386, 1164, 1163, + /* 640 */ 1162, 1161, 1160, 136, 1159, 1158, 50, 9, 8, 144, + /* 650 */ 1237, 1413, 1386, 420, 495, 494, 493, 492, 491, 490, + /* 660 */ 489, 488, 207, 485, 484, 483, 482, 481, 480, 479, + /* 670 */ 478, 477, 1429, 1476, 1476, 486, 58, 467, 294, 425, + /* 680 */ 1386, 1386, 1386, 1386, 1386, 798, 1386, 1386, 417, 427, + /* 690 */ 424, 1472, 1471, 1386, 1254, 152, 476, 154, 150, 1413, + /* 700 */ 153, 346, 373, 143, 67, 138, 998, 140, 383, 69, + /* 710 */ 1414, 1415, 1418, 1462, 1092, 371, 1244, 254, 1458, 1535, + /* 720 */ 1429, 1150, 1151, 165, 137, 415, 466, 412, 1519, 45, + /* 730 */ 44, 303, 156, 130, 1189, 155, 104, 427, 297, 387, + /* 740 */ 158, 1386, 1413, 157, 1168, 1226, 241, 1187, 289, 396, + /* 750 */ 285, 281, 127, 188, 1302, 345, 364, 69, 1414, 1415, + /* 760 */ 1418, 1462, 181, 1429, 1492, 254, 1458, 120, 43, 367, + /* 770 */ 425, 1115, 1001, 175, 126, 997, 177, 1430, 1413, 187, + /* 780 */ 427, 410, 421, 190, 1386, 391, 1489, 32, 32, 32, + /* 790 */ 1064, 1023, 957, 194, 2, 1010, 196, 278, 1049, 1429, + /* 800 */ 69, 1414, 1415, 1418, 1462, 1413, 425, 282, 254, 1458, + /* 810 */ 1535, 101, 240, 242, 433, 418, 427, 841, 102, 1480, + /* 820 */ 1386, 202, 966, 211, 313, 413, 1429, 409, 510, 1338, + /* 830 */ 132, 104, 319, 425, 867, 327, 224, 1414, 1415, 1418, + /* 840 */ 326, 1000, 210, 427, 328, 94, 332, 1386, 1019, 98, + /* 850 */ 333, 499, 43, 209, 446, 862, 1523, 895, 102, 103, + /* 860 */ 1413, 899, 905, 70, 1414, 1415, 1418, 1462, 413, 125, + /* 870 */ 334, 423, 1458, 1521, 1018, 335, 139, 66, 1017, 336, + /* 880 */ 203, 1429, 96, 104, 102, 337, 904, 105, 425, 142, + /* 890 */ 48, 145, 184, 1469, 408, 340, 407, 1016, 427, 1523, + /* 900 */ 347, 349, 1386, 74, 1413, 1251, 376, 149, 1247, 1413, + /* 910 */ 437, 377, 125, 151, 106, 107, 1521, 1249, 115, 1414, + /* 920 */ 1415, 1418, 1245, 108, 109, 1429, 352, 359, 378, 250, + /* 930 */ 1429, 1015, 425, 382, 167, 390, 388, 425, 168, 379, + /* 940 */ 170, 397, 427, 1503, 431, 386, 1386, 427, 1493, 260, + /* 950 */ 389, 1386, 995, 974, 392, 164, 414, 1536, 1502, 5, + /* 960 */ 1483, 173, 228, 1414, 1415, 1418, 1413, 228, 1414, 1415, + /* 970 */ 1418, 1413, 406, 394, 393, 176, 253, 4, 401, 1089, + /* 980 */ 97, 1014, 33, 183, 180, 422, 255, 1429, 118, 1477, + /* 990 */ 419, 429, 1429, 182, 425, 17, 1347, 1346, 430, 425, + /* 1000 */ 263, 434, 1444, 435, 427, 198, 1538, 1413, 1386, 427, + /* 1010 */ 200, 57, 436, 1386, 1520, 1413, 258, 59, 1262, 212, + /* 1020 */ 473, 189, 444, 214, 227, 1414, 1415, 1418, 1429, 228, + /* 1030 */ 1414, 1415, 1418, 487, 208, 425, 1429, 506, 216, 220, + /* 1040 */ 221, 1413, 39, 425, 218, 427, 1380, 1379, 277, 1386, + /* 1050 */ 1413, 279, 262, 427, 1376, 280, 405, 1386, 984, 985, + /* 1060 */ 128, 284, 1429, 1374, 286, 228, 1414, 1415, 1418, 425, + /* 1070 */ 287, 1429, 288, 226, 1414, 1415, 1418, 1413, 425, 427, + /* 1080 */ 1373, 290, 1372, 1386, 1363, 1413, 292, 129, 427, 295, + /* 1090 */ 296, 969, 1386, 968, 1357, 1356, 302, 1355, 1429, 229, + /* 1100 */ 1414, 1415, 1418, 301, 1354, 425, 1429, 1331, 222, 1414, + /* 1110 */ 1415, 1418, 940, 425, 1330, 427, 1329, 1328, 1327, 1386, + /* 1120 */ 1326, 1413, 1325, 427, 1324, 1323, 1413, 1386, 1322, 1321, + /* 1130 */ 1320, 1319, 100, 1318, 1317, 230, 1414, 1415, 1418, 1316, + /* 1140 */ 1315, 1314, 1429, 223, 1414, 1415, 1418, 1429, 1313, 425, + /* 1150 */ 1312, 1311, 1310, 1309, 425, 1200, 1371, 942, 1365, 427, + /* 1160 */ 1353, 1344, 1413, 1386, 427, 1240, 141, 1413, 1386, 811, + /* 1170 */ 1199, 1197, 1186, 1413, 341, 343, 1185, 1182, 342, 231, + /* 1180 */ 1414, 1415, 1418, 1429, 1426, 1414, 1415, 1418, 1429, 1242, + /* 1190 */ 425, 73, 912, 486, 1429, 425, 910, 1241, 1195, 1190, + /* 1200 */ 427, 425, 1238, 148, 1386, 427, 840, 839, 247, 1386, + /* 1210 */ 248, 427, 365, 838, 1413, 1386, 837, 835, 834, 1188, + /* 1220 */ 1425, 1414, 1415, 1418, 1236, 1424, 1414, 1415, 1418, 1413, + /* 1230 */ 249, 238, 1414, 1415, 1418, 1429, 1181, 370, 268, 267, + /* 1240 */ 1413, 368, 425, 1180, 372, 71, 1370, 163, 1003, 42, + /* 1250 */ 1429, 976, 427, 1364, 990, 1413, 1386, 425, 110, 206, + /* 1260 */ 380, 1429, 1352, 472, 996, 166, 1351, 427, 425, 1343, + /* 1270 */ 988, 1386, 237, 1414, 1415, 1418, 1429, 381, 427, 36, + /* 1280 */ 366, 206, 1386, 425, 474, 472, 51, 239, 1414, 1415, + /* 1290 */ 1418, 169, 171, 427, 3, 374, 32, 1386, 236, 1414, + /* 1300 */ 1415, 1418, 37, 471, 470, 469, 474, 468, 116, 162, + /* 1310 */ 174, 1114, 369, 225, 1414, 1415, 1418, 363, 442, 178, + /* 1320 */ 161, 1108, 1107, 14, 507, 471, 470, 469, 52, 468, + /* 1330 */ 999, 179, 53, 34, 19, 1404, 989, 15, 1086, 185, + /* 1340 */ 1085, 35, 124, 1141, 41, 16, 10, 40, 54, 1136, + /* 1350 */ 1135, 256, 1140, 1139, 257, 8, 1050, 1024, 192, 13, + /* 1360 */ 18, 432, 193, 1112, 1004, 195, 197, 55, 1342, 199, + /* 1370 */ 991, 56, 201, 60, 1005, 38, 1403, 896, 445, 270, + /* 1380 */ 204, 1007, 995, 443, 449, 452, 447, 994, 995, 893, + /* 1390 */ 450, 873, 890, 453, 455, 884, 456, 458, 882, 459, + /* 1400 */ 61, 907, 888, 887, 886, 465, 62, 63, 906, 903, + /* 1410 */ 901, 475, 885, 809, 831, 830, 829, 828, 827, 826, + /* 1420 */ 825, 824, 823, 842, 821, 820, 1196, 819, 818, 817, + /* 1430 */ 1184, 816, 815, 814, 496, 497, 500, 1183, 501, 1179, + /* 1440 */ 498, 504, 505, 1154, 992, 215, 508, 509, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 246, 267, 268, 210, 210, 207, 208, 229, 225, 216, - /* 10 */ 216, 229, 258, 259, 231, 210, 223, 223, 18, 236, - /* 20 */ 20, 0, 0, 210, 246, 232, 232, 27, 246, 216, - /* 30 */ 30, 12, 13, 14, 15, 16, 258, 259, 242, 39, - /* 40 */ 258, 259, 237, 21, 248, 232, 24, 25, 26, 27, - /* 50 */ 28, 29, 30, 31, 32, 234, 12, 13, 14, 15, - /* 60 */ 16, 242, 241, 242, 245, 218, 47, 248, 221, 20, - /* 70 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - /* 80 */ 59, 60, 61, 62, 63, 64, 65, 66, 38, 70, - /* 90 */ 12, 13, 14, 15, 16, 21, 20, 78, 24, 25, - /* 100 */ 26, 27, 28, 29, 30, 31, 32, 107, 0, 210, - /* 110 */ 110, 111, 112, 113, 114, 216, 116, 117, 118, 119, - /* 120 */ 120, 121, 122, 123, 210, 47, 14, 15, 16, 46, - /* 130 */ 216, 232, 24, 25, 26, 27, 28, 29, 30, 31, - /* 140 */ 32, 225, 49, 124, 51, 69, 232, 54, 232, 20, - /* 150 */ 57, 210, 59, 60, 104, 62, 78, 216, 20, 128, - /* 160 */ 210, 142, 83, 84, 85, 86, 87, 88, 89, 90, - /* 170 */ 91, 92, 93, 232, 95, 96, 97, 98, 99, 100, - /* 180 */ 264, 176, 232, 20, 165, 166, 167, 168, 169, 170, - /* 190 */ 171, 172, 173, 2, 18, 204, 137, 20, 139, 23, - /* 200 */ 210, 217, 124, 12, 13, 14, 15, 16, 210, 225, - /* 210 */ 225, 35, 181, 182, 216, 265, 225, 233, 233, 134, - /* 220 */ 142, 45, 204, 232, 274, 275, 276, 277, 204, 279, - /* 230 */ 232, 46, 251, 242, 149, 176, 203, 246, 205, 204, - /* 240 */ 20, 251, 251, 165, 166, 167, 168, 169, 170, 171, - /* 250 */ 172, 173, 176, 262, 263, 264, 265, 266, 12, 13, - /* 260 */ 225, 270, 271, 282, 246, 210, 20, 232, 22, 80, - /* 270 */ 246, 216, 282, 282, 226, 204, 295, 242, 22, 57, - /* 280 */ 299, 246, 106, 61, 38, 295, 295, 232, 155, 299, - /* 290 */ 299, 0, 212, 47, 38, 43, 225, 262, 263, 264, - /* 300 */ 265, 266, 222, 232, 82, 270, 271, 131, 132, 133, - /* 310 */ 230, 135, 226, 242, 0, 69, 140, 246, 185, 186, - /* 320 */ 187, 188, 189, 210, 148, 210, 150, 36, 152, 153, - /* 330 */ 154, 216, 20, 262, 263, 264, 265, 266, 92, 176, - /* 340 */ 63, 270, 271, 272, 225, 232, 226, 232, 92, 212, - /* 350 */ 104, 232, 176, 210, 20, 284, 12, 13, 14, 216, - /* 360 */ 104, 290, 291, 49, 20, 51, 22, 230, 54, 12, - /* 370 */ 13, 57, 204, 59, 60, 232, 62, 20, 265, 22, - /* 380 */ 226, 69, 38, 264, 138, 108, 109, 204, 275, 276, - /* 390 */ 277, 47, 279, 225, 138, 38, 1, 2, 224, 0, - /* 400 */ 232, 155, 156, 157, 47, 159, 213, 217, 215, 235, - /* 410 */ 242, 155, 156, 69, 246, 225, 204, 12, 13, 14, - /* 420 */ 15, 16, 176, 233, 209, 204, 69, 175, 201, 246, - /* 430 */ 262, 263, 264, 265, 266, 220, 92, 225, 270, 271, - /* 440 */ 272, 225, 227, 282, 232, 46, 225, 231, 104, 92, - /* 450 */ 226, 225, 236, 232, 242, 124, 295, 231, 246, 291, - /* 460 */ 299, 104, 236, 242, 130, 70, 158, 246, 160, 161, - /* 470 */ 162, 163, 164, 142, 262, 263, 264, 265, 251, 226, - /* 480 */ 12, 13, 138, 262, 263, 264, 265, 266, 20, 204, - /* 490 */ 22, 270, 271, 272, 44, 138, 165, 204, 0, 155, - /* 500 */ 156, 157, 281, 159, 204, 254, 38, 217, 3, 282, - /* 510 */ 225, 204, 155, 156, 157, 225, 159, 232, 68, 210, - /* 520 */ 20, 71, 295, 233, 225, 225, 299, 242, 204, 22, - /* 530 */ 4, 246, 232, 204, 260, 236, 131, 69, 20, 246, - /* 540 */ 22, 232, 242, 205, 46, 38, 246, 262, 263, 264, - /* 550 */ 265, 266, 278, 246, 225, 270, 271, 272, 40, 302, - /* 560 */ 92, 232, 262, 263, 264, 265, 281, 14, 215, 204, - /* 570 */ 246, 242, 104, 20, 265, 246, 210, 204, 260, 12, - /* 580 */ 13, 14, 216, 232, 275, 276, 277, 20, 279, 22, - /* 590 */ 239, 262, 263, 264, 265, 266, 278, 204, 232, 270, - /* 600 */ 271, 272, 197, 260, 2, 38, 138, 204, 204, 251, - /* 610 */ 281, 246, 12, 13, 12, 13, 14, 15, 16, 246, - /* 620 */ 20, 278, 22, 155, 156, 157, 210, 159, 225, 209, - /* 630 */ 130, 204, 216, 204, 204, 232, 69, 204, 38, 246, - /* 640 */ 282, 204, 204, 67, 176, 242, 70, 227, 232, 246, - /* 650 */ 246, 12, 13, 295, 251, 174, 175, 299, 204, 92, - /* 660 */ 67, 22, 12, 13, 19, 262, 263, 264, 265, 69, - /* 670 */ 20, 104, 22, 246, 67, 246, 246, 38, 33, 246, - /* 680 */ 293, 36, 225, 246, 246, 282, 204, 42, 38, 44, - /* 690 */ 204, 204, 92, 236, 1, 2, 73, 192, 295, 76, - /* 700 */ 246, 235, 299, 177, 104, 138, 204, 73, 73, 73, - /* 710 */ 76, 76, 76, 68, 0, 0, 71, 0, 67, 69, - /* 720 */ 47, 70, 155, 156, 157, 69, 159, 225, 246, 199, - /* 730 */ 200, 92, 246, 246, 232, 79, 22, 22, 138, 22, - /* 740 */ 287, 69, 92, 104, 242, 21, 101, 67, 246, 67, - /* 750 */ 70, 38, 70, 81, 104, 155, 156, 157, 34, 159, - /* 760 */ 210, 207, 204, 280, 262, 263, 264, 265, 266, 261, - /* 770 */ 225, 126, 270, 271, 129, 67, 67, 138, 70, 70, - /* 780 */ 296, 20, 232, 225, 283, 210, 36, 194, 138, 144, - /* 790 */ 232, 146, 257, 38, 155, 156, 67, 214, 136, 70, - /* 800 */ 242, 251, 210, 196, 246, 155, 156, 157, 252, 159, - /* 810 */ 204, 67, 115, 67, 70, 265, 70, 104, 210, 240, - /* 820 */ 262, 263, 264, 265, 124, 275, 276, 277, 155, 279, - /* 830 */ 210, 225, 282, 12, 13, 14, 15, 16, 232, 67, - /* 840 */ 20, 35, 70, 67, 33, 295, 70, 36, 242, 299, - /* 850 */ 238, 67, 246, 42, 70, 44, 238, 67, 300, 301, - /* 860 */ 70, 256, 67, 57, 204, 70, 242, 61, 262, 263, - /* 870 */ 264, 265, 266, 204, 20, 250, 212, 271, 72, 68, - /* 880 */ 74, 75, 71, 77, 67, 225, 232, 70, 82, 20, - /* 890 */ 67, 70, 232, 70, 225, 67, 243, 212, 70, 212, - /* 900 */ 210, 232, 242, 212, 20, 206, 246, 204, 225, 249, - /* 910 */ 232, 242, 214, 225, 225, 246, 210, 225, 249, 225, - /* 920 */ 225, 225, 262, 263, 264, 265, 225, 225, 225, 225, - /* 930 */ 225, 262, 263, 264, 265, 232, 125, 214, 127, 206, - /* 940 */ 129, 145, 256, 204, 209, 242, 209, 20, 232, 246, - /* 950 */ 184, 204, 242, 292, 4, 243, 156, 146, 183, 292, - /* 960 */ 191, 250, 255, 289, 225, 262, 263, 264, 265, 19, - /* 970 */ 190, 232, 225, 247, 246, 178, 246, 204, 247, 232, - /* 980 */ 261, 242, 246, 33, 179, 246, 36, 175, 232, 242, - /* 990 */ 20, 41, 0, 246, 44, 115, 249, 294, 225, 0, - /* 1000 */ 260, 262, 263, 264, 265, 232, 204, 286, 288, 262, - /* 1010 */ 263, 264, 265, 198, 273, 242, 298, 285, 68, 246, - /* 1020 */ 195, 71, 249, 193, 297, 303, 69, 225, 269, 247, - /* 1030 */ 204, 246, 246, 246, 232, 262, 263, 264, 265, 247, - /* 1040 */ 301, 127, 244, 209, 242, 232, 243, 209, 246, 57, - /* 1050 */ 221, 225, 69, 61, 204, 214, 57, 214, 232, 232, - /* 1060 */ 61, 228, 210, 206, 262, 263, 264, 265, 242, 209, - /* 1070 */ 219, 253, 246, 211, 82, 225, 0, 219, 204, 202, - /* 1080 */ 0, 82, 232, 60, 0, 38, 151, 38, 262, 263, - /* 1090 */ 264, 265, 242, 101, 102, 103, 246, 105, 38, 225, - /* 1100 */ 101, 102, 103, 38, 105, 151, 232, 204, 0, 38, - /* 1110 */ 38, 151, 262, 263, 264, 265, 242, 0, 0, 38, - /* 1120 */ 246, 38, 0, 69, 142, 141, 104, 138, 225, 0, - /* 1130 */ 0, 134, 50, 204, 0, 232, 262, 263, 264, 265, - /* 1140 */ 0, 204, 81, 0, 0, 242, 0, 0, 0, 246, - /* 1150 */ 0, 0, 0, 0, 225, 0, 0, 0, 0, 115, - /* 1160 */ 0, 232, 225, 0, 0, 262, 263, 264, 265, 232, - /* 1170 */ 0, 242, 0, 0, 0, 246, 204, 0, 0, 242, - /* 1180 */ 0, 22, 204, 246, 0, 0, 0, 0, 0, 0, - /* 1190 */ 48, 262, 263, 264, 265, 0, 0, 225, 0, 262, - /* 1200 */ 263, 264, 265, 225, 232, 43, 38, 36, 43, 0, - /* 1210 */ 232, 204, 0, 0, 242, 78, 67, 204, 246, 38, - /* 1220 */ 242, 76, 22, 0, 246, 0, 22, 0, 38, 38, - /* 1230 */ 38, 38, 225, 22, 262, 263, 264, 265, 225, 232, - /* 1240 */ 262, 263, 264, 265, 38, 232, 38, 0, 22, 242, - /* 1250 */ 0, 22, 38, 246, 39, 242, 0, 22, 20, 246, - /* 1260 */ 0, 130, 147, 204, 0, 22, 38, 143, 130, 262, - /* 1270 */ 263, 264, 265, 0, 0, 262, 263, 264, 265, 0, - /* 1280 */ 43, 127, 180, 81, 225, 69, 125, 67, 67, 70, - /* 1290 */ 180, 232, 70, 69, 174, 4, 130, 67, 81, 81, - /* 1300 */ 69, 242, 70, 69, 38, 246, 70, 67, 69, 67, - /* 1310 */ 38, 70, 67, 180, 70, 67, 70, 2, 38, 38, - /* 1320 */ 38, 262, 263, 264, 265, 38, 0, 70, 69, 128, - /* 1330 */ 155, 69, 81, 70, 79, 70, 69, 69, 69, 43, - /* 1340 */ 69, 22, 81, 38, 69, 38, 38, 70, 70, 38, - /* 1350 */ 38, 81, 69, 69, 38, 70, 80, 69, 125, 70, - /* 1360 */ 70, 69, 69, 22, 22, 38, 38, 69, 48, 47, - /* 1370 */ 69, 69, 22, 38, 38, 38, 38, 38, 38, 22, - /* 1380 */ 94, 82, 38, 38, 38, 38, 38, 38, 94, 94, - /* 1390 */ 38, 94, 38, 38, 38, 0, 38, 36, 0, 0, - /* 1400 */ 38, 37, 43, 0, 104, 22, 21, 304, 22, 22, - /* 1410 */ 21, 20, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1420 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1430 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1440 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 0 */ 226, 210, 210, 217, 210, 226, 204, 216, 216, 210, + /* 10 */ 216, 225, 12, 13, 223, 210, 229, 223, 2, 233, + /* 20 */ 20, 201, 22, 232, 232, 229, 232, 225, 12, 13, + /* 30 */ 14, 15, 16, 246, 232, 210, 237, 232, 38, 224, + /* 40 */ 210, 216, 246, 20, 242, 258, 259, 47, 246, 225, + /* 50 */ 235, 12, 13, 14, 258, 259, 232, 232, 20, 20, + /* 60 */ 22, 22, 232, 210, 262, 263, 264, 265, 266, 69, + /* 70 */ 265, 251, 270, 271, 272, 204, 225, 38, 40, 274, + /* 80 */ 275, 276, 277, 225, 279, 232, 47, 236, 264, 231, + /* 90 */ 12, 13, 92, 291, 236, 265, 225, 204, 20, 63, + /* 100 */ 22, 20, 282, 232, 104, 275, 276, 277, 69, 279, + /* 110 */ 2, 267, 268, 242, 246, 295, 38, 246, 265, 299, + /* 120 */ 12, 13, 14, 15, 16, 47, 258, 259, 275, 276, + /* 130 */ 277, 92, 279, 262, 263, 264, 265, 266, 138, 246, + /* 140 */ 20, 270, 271, 104, 108, 109, 204, 69, 12, 13, + /* 150 */ 14, 15, 16, 130, 209, 155, 156, 157, 158, 159, + /* 160 */ 160, 161, 162, 163, 164, 220, 18, 225, 20, 69, + /* 170 */ 92, 137, 227, 139, 232, 27, 176, 138, 30, 20, + /* 180 */ 242, 81, 104, 245, 242, 225, 248, 39, 246, 69, + /* 190 */ 204, 128, 232, 44, 155, 156, 157, 158, 159, 160, + /* 200 */ 161, 162, 163, 164, 262, 263, 264, 265, 266, 210, + /* 210 */ 176, 225, 270, 271, 272, 216, 138, 68, 232, 225, + /* 220 */ 71, 210, 232, 281, 264, 231, 20, 216, 242, 239, + /* 230 */ 236, 232, 246, 155, 156, 157, 158, 159, 160, 161, + /* 240 */ 162, 163, 164, 232, 181, 182, 12, 13, 262, 263, + /* 250 */ 264, 265, 266, 69, 20, 107, 22, 271, 110, 111, + /* 260 */ 112, 113, 114, 79, 116, 117, 118, 119, 120, 121, + /* 270 */ 122, 123, 38, 21, 155, 69, 24, 25, 26, 27, + /* 280 */ 28, 29, 30, 31, 32, 12, 13, 14, 0, 49, + /* 290 */ 225, 51, 251, 20, 54, 22, 176, 57, 233, 59, + /* 300 */ 60, 176, 62, 69, 185, 186, 187, 188, 189, 21, + /* 310 */ 43, 38, 24, 25, 26, 27, 28, 29, 30, 31, + /* 320 */ 32, 210, 225, 282, 12, 13, 92, 216, 231, 207, + /* 330 */ 208, 210, 20, 236, 22, 176, 295, 216, 104, 213, + /* 340 */ 299, 215, 69, 232, 234, 12, 13, 14, 15, 16, + /* 350 */ 38, 241, 242, 232, 83, 84, 85, 86, 87, 88, + /* 360 */ 89, 90, 91, 92, 93, 92, 95, 96, 97, 98, + /* 370 */ 99, 100, 138, 14, 15, 16, 203, 104, 205, 218, + /* 380 */ 204, 69, 221, 0, 12, 13, 14, 15, 16, 155, + /* 390 */ 156, 157, 158, 159, 160, 161, 162, 163, 164, 210, + /* 400 */ 282, 225, 0, 70, 92, 216, 46, 210, 232, 0, + /* 410 */ 176, 138, 204, 295, 242, 20, 104, 299, 242, 36, + /* 420 */ 248, 232, 246, 12, 13, 14, 15, 16, 155, 156, + /* 430 */ 157, 158, 159, 160, 161, 162, 163, 164, 262, 263, + /* 440 */ 264, 265, 175, 12, 13, 14, 15, 16, 251, 217, + /* 450 */ 138, 49, 20, 51, 246, 46, 54, 225, 47, 57, + /* 460 */ 35, 59, 60, 204, 62, 233, 204, 155, 156, 157, + /* 470 */ 158, 159, 160, 161, 162, 163, 164, 301, 47, 282, + /* 480 */ 12, 13, 57, 1, 2, 20, 61, 225, 20, 78, + /* 490 */ 22, 0, 295, 212, 232, 134, 299, 72, 251, 74, + /* 500 */ 75, 70, 77, 131, 242, 246, 38, 82, 246, 78, + /* 510 */ 149, 230, 204, 251, 204, 24, 25, 26, 27, 28, + /* 520 */ 29, 30, 31, 32, 262, 263, 264, 265, 266, 282, + /* 530 */ 212, 217, 270, 271, 57, 124, 3, 69, 61, 225, + /* 540 */ 222, 4, 295, 204, 282, 210, 299, 233, 230, 20, + /* 550 */ 124, 216, 70, 142, 246, 124, 246, 295, 46, 82, + /* 560 */ 92, 299, 130, 225, 210, 210, 204, 232, 142, 197, + /* 570 */ 216, 216, 104, 142, 236, 204, 165, 166, 167, 168, + /* 580 */ 169, 170, 171, 172, 173, 246, 232, 232, 20, 260, + /* 590 */ 0, 165, 204, 204, 204, 204, 165, 166, 167, 168, + /* 600 */ 169, 170, 171, 172, 173, 0, 138, 278, 246, 14, + /* 610 */ 204, 33, 22, 226, 36, 20, 18, 246, 174, 175, + /* 620 */ 42, 23, 44, 155, 156, 157, 158, 159, 160, 161, + /* 630 */ 162, 163, 164, 35, 246, 246, 246, 246, 204, 204, + /* 640 */ 204, 204, 204, 45, 204, 204, 68, 1, 2, 71, + /* 650 */ 0, 204, 246, 67, 49, 50, 51, 52, 53, 54, + /* 660 */ 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + /* 670 */ 65, 66, 225, 260, 260, 67, 209, 80, 70, 232, + /* 680 */ 246, 246, 246, 246, 246, 22, 246, 246, 67, 242, + /* 690 */ 47, 278, 278, 246, 227, 73, 46, 73, 76, 204, + /* 700 */ 76, 38, 21, 125, 106, 127, 38, 129, 254, 262, + /* 710 */ 263, 264, 265, 266, 177, 34, 226, 270, 271, 272, + /* 720 */ 225, 199, 200, 226, 146, 192, 226, 232, 281, 131, + /* 730 */ 132, 133, 73, 135, 0, 76, 67, 242, 140, 70, + /* 740 */ 73, 246, 204, 76, 205, 215, 148, 0, 150, 293, + /* 750 */ 152, 153, 154, 302, 235, 207, 22, 262, 263, 264, + /* 760 */ 265, 266, 287, 225, 261, 270, 271, 272, 67, 22, + /* 770 */ 232, 70, 104, 67, 176, 38, 70, 225, 204, 284, + /* 780 */ 242, 280, 196, 296, 246, 290, 291, 67, 67, 67, + /* 790 */ 70, 70, 70, 67, 283, 20, 70, 210, 155, 225, + /* 800 */ 262, 263, 264, 265, 266, 204, 232, 36, 270, 271, + /* 810 */ 272, 67, 257, 214, 70, 194, 242, 38, 67, 281, + /* 820 */ 246, 70, 136, 252, 210, 251, 225, 210, 19, 210, + /* 830 */ 115, 67, 240, 232, 70, 124, 262, 263, 264, 265, + /* 840 */ 238, 104, 33, 242, 238, 36, 210, 246, 20, 232, + /* 850 */ 256, 42, 67, 44, 67, 70, 282, 70, 67, 67, + /* 860 */ 204, 70, 70, 262, 263, 264, 265, 266, 251, 295, + /* 870 */ 242, 270, 271, 299, 20, 250, 212, 68, 20, 232, + /* 880 */ 71, 225, 265, 67, 67, 243, 70, 70, 232, 212, + /* 890 */ 212, 212, 275, 276, 277, 210, 279, 20, 242, 282, + /* 900 */ 206, 225, 246, 210, 204, 225, 232, 225, 225, 204, + /* 910 */ 101, 256, 295, 225, 225, 225, 299, 225, 262, 263, + /* 920 */ 264, 265, 225, 225, 225, 225, 214, 214, 145, 206, + /* 930 */ 225, 20, 232, 242, 209, 126, 232, 232, 129, 255, + /* 940 */ 209, 184, 242, 292, 183, 250, 246, 242, 261, 249, + /* 950 */ 243, 246, 156, 144, 249, 146, 300, 301, 292, 191, + /* 960 */ 289, 247, 262, 263, 264, 265, 204, 262, 263, 264, + /* 970 */ 265, 204, 190, 246, 179, 247, 246, 178, 246, 175, + /* 980 */ 232, 20, 115, 273, 288, 195, 198, 225, 286, 260, + /* 990 */ 193, 246, 225, 285, 232, 69, 247, 247, 246, 232, + /* 1000 */ 246, 127, 269, 244, 242, 232, 303, 204, 246, 242, + /* 1010 */ 209, 209, 243, 246, 298, 204, 249, 69, 232, 221, + /* 1020 */ 214, 297, 228, 210, 262, 263, 264, 265, 225, 262, + /* 1030 */ 263, 264, 265, 214, 209, 232, 225, 206, 211, 219, + /* 1040 */ 219, 204, 253, 232, 202, 242, 0, 0, 60, 246, + /* 1050 */ 204, 38, 249, 242, 0, 151, 294, 246, 38, 38, + /* 1060 */ 38, 151, 225, 0, 38, 262, 263, 264, 265, 232, + /* 1070 */ 38, 225, 151, 262, 263, 264, 265, 204, 232, 242, + /* 1080 */ 0, 38, 0, 246, 0, 204, 38, 69, 242, 142, + /* 1090 */ 141, 104, 246, 138, 0, 0, 134, 0, 225, 262, + /* 1100 */ 263, 264, 265, 50, 0, 232, 225, 0, 262, 263, + /* 1110 */ 264, 265, 81, 232, 0, 242, 0, 0, 0, 246, + /* 1120 */ 0, 204, 0, 242, 0, 0, 204, 246, 0, 0, + /* 1130 */ 0, 0, 115, 0, 0, 262, 263, 264, 265, 0, + /* 1140 */ 0, 0, 225, 262, 263, 264, 265, 225, 0, 232, + /* 1150 */ 0, 0, 0, 0, 232, 0, 0, 22, 0, 242, + /* 1160 */ 0, 0, 204, 246, 242, 0, 43, 204, 246, 48, + /* 1170 */ 0, 0, 0, 204, 38, 43, 0, 0, 36, 262, + /* 1180 */ 263, 264, 265, 225, 262, 263, 264, 265, 225, 0, + /* 1190 */ 232, 78, 38, 67, 225, 232, 22, 0, 0, 0, + /* 1200 */ 242, 232, 0, 76, 246, 242, 38, 38, 22, 246, + /* 1210 */ 22, 242, 39, 38, 204, 246, 38, 38, 38, 0, + /* 1220 */ 262, 263, 264, 265, 0, 262, 263, 264, 265, 204, + /* 1230 */ 22, 262, 263, 264, 265, 225, 0, 22, 12, 13, + /* 1240 */ 204, 38, 232, 0, 22, 20, 0, 147, 22, 130, + /* 1250 */ 225, 38, 242, 0, 22, 204, 246, 232, 143, 57, + /* 1260 */ 22, 225, 0, 61, 38, 127, 0, 242, 232, 0, + /* 1270 */ 38, 246, 262, 263, 264, 265, 225, 130, 242, 130, + /* 1280 */ 4, 57, 246, 232, 82, 61, 69, 262, 263, 264, + /* 1290 */ 265, 43, 125, 242, 67, 19, 67, 246, 262, 263, + /* 1300 */ 264, 265, 67, 101, 102, 103, 82, 105, 69, 33, + /* 1310 */ 70, 70, 36, 262, 263, 264, 265, 41, 92, 69, + /* 1320 */ 44, 70, 70, 180, 92, 101, 102, 103, 69, 105, + /* 1330 */ 104, 67, 69, 174, 67, 81, 104, 180, 70, 81, + /* 1340 */ 70, 67, 81, 70, 68, 67, 180, 71, 4, 38, + /* 1350 */ 38, 38, 38, 38, 38, 2, 155, 70, 81, 69, + /* 1360 */ 69, 128, 70, 70, 138, 69, 69, 69, 0, 43, + /* 1370 */ 138, 69, 125, 79, 22, 69, 81, 70, 38, 38, + /* 1380 */ 81, 155, 156, 80, 38, 38, 69, 155, 156, 70, + /* 1390 */ 69, 22, 70, 69, 38, 70, 69, 38, 70, 69, + /* 1400 */ 69, 38, 94, 94, 94, 82, 69, 69, 104, 38, + /* 1410 */ 22, 47, 94, 48, 22, 38, 38, 38, 38, 38, + /* 1420 */ 38, 38, 22, 38, 38, 38, 0, 38, 38, 38, + /* 1430 */ 0, 38, 38, 38, 38, 36, 38, 0, 37, 0, + /* 1440 */ 43, 22, 21, 304, 22, 22, 21, 20, 304, 304, /* 1450 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1460 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1470 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, @@ -516,145 +519,148 @@ static const YYCODETYPE yy_lookahead[] = { /* 1580 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1590 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, /* 1600 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, - /* 1610 */ 304, 304, 304, + /* 1610 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1620 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1630 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, 304, + /* 1640 */ 304, 304, 304, 304, 304, 304, 304, 304, 304, }; -#define YY_SHIFT_COUNT (511) +#define YY_SHIFT_COUNT (510) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1403) +#define YY_SHIFT_MAX (1439) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 176, 246, 344, 357, 357, 357, 357, 468, 357, 357, - /* 10 */ 600, 650, 76, 567, 600, 600, 600, 600, 600, 600, - /* 20 */ 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, - /* 30 */ 600, 600, 600, 312, 312, 312, 163, 639, 639, 59, - /* 40 */ 49, 49, 5, 639, 49, 49, 49, 49, 49, 49, - /* 50 */ 83, 129, 138, 138, 5, 177, 129, 49, 49, 129, - /* 60 */ 49, 129, 177, 129, 129, 49, 185, 0, 19, 78, - /* 70 */ 78, 74, 806, 256, 93, 256, 256, 256, 256, 256, - /* 80 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, - /* 90 */ 256, 256, 256, 256, 518, 399, 334, 334, 334, 498, - /* 100 */ 220, 177, 129, 129, 129, 189, 79, 79, 79, 79, - /* 110 */ 79, 79, 645, 22, 314, 405, 133, 222, 31, 507, - /* 120 */ 500, 481, 252, 481, 553, 505, 526, 761, 750, 755, - /* 130 */ 662, 761, 761, 697, 700, 700, 761, 820, 177, 854, - /* 140 */ 83, 220, 869, 83, 83, 761, 83, 884, 129, 129, - /* 150 */ 129, 129, 129, 129, 129, 129, 129, 129, 129, 755, - /* 160 */ 755, 761, 884, 220, 820, 796, 177, 854, 185, 220, - /* 170 */ 869, 185, 927, 766, 775, 800, 766, 775, 800, 800, - /* 180 */ 769, 780, 805, 797, 812, 220, 970, 880, 815, 825, - /* 190 */ 830, 957, 129, 775, 800, 800, 775, 800, 914, 220, - /* 200 */ 869, 185, 189, 185, 220, 983, 755, 755, 761, 185, - /* 210 */ 884, 1412, 1412, 1412, 1412, 1412, 21, 811, 108, 950, - /* 220 */ 992, 999, 191, 602, 308, 821, 44, 44, 44, 44, - /* 230 */ 44, 44, 44, 450, 277, 395, 331, 112, 112, 112, - /* 240 */ 112, 291, 85, 576, 623, 634, 635, 636, 714, 715, - /* 250 */ 717, 724, 651, 680, 682, 693, 530, 593, 607, 708, - /* 260 */ 673, 709, 672, 729, 744, 746, 772, 776, 50, 713, - /* 270 */ 784, 790, 795, 817, 823, 828, 656, 1076, 1080, 1023, - /* 280 */ 1084, 1047, 935, 1049, 1060, 1065, 954, 1108, 1071, 1072, - /* 290 */ 960, 1117, 1081, 1118, 1083, 1122, 1054, 982, 984, 1022, - /* 300 */ 989, 1129, 1130, 1082, 997, 1134, 1140, 1061, 1143, 1144, - /* 310 */ 1146, 1147, 1148, 1150, 1151, 1152, 1153, 1155, 1156, 1157, - /* 320 */ 1158, 1044, 1160, 1163, 1164, 1170, 1172, 1173, 1159, 1174, - /* 330 */ 1177, 1178, 1180, 1184, 1185, 1186, 1187, 1188, 1162, 1189, - /* 340 */ 1142, 1195, 1196, 1168, 1171, 1165, 1198, 1209, 1212, 1213, - /* 350 */ 1137, 1145, 1181, 1149, 1200, 1223, 1190, 1191, 1192, 1193, - /* 360 */ 1149, 1206, 1208, 1225, 1204, 1227, 1211, 1215, 1247, 1226, - /* 370 */ 1214, 1250, 1229, 1256, 1235, 1238, 1260, 1131, 1115, 1228, - /* 380 */ 1264, 1124, 1243, 1138, 1154, 1273, 1274, 1166, 1279, 1216, - /* 390 */ 1237, 1161, 1220, 1221, 1102, 1219, 1230, 1222, 1224, 1231, - /* 400 */ 1232, 1234, 1236, 1240, 1202, 1239, 1242, 1110, 1241, 1244, - /* 410 */ 1217, 1120, 1245, 1218, 1246, 1248, 1133, 1291, 1266, 1272, - /* 420 */ 1280, 1281, 1282, 1287, 1315, 1175, 1251, 1257, 1259, 1262, - /* 430 */ 1263, 1265, 1267, 1268, 1201, 1269, 1326, 1296, 1233, 1271, - /* 440 */ 1255, 1261, 1270, 1319, 1275, 1276, 1277, 1305, 1307, 1283, - /* 450 */ 1278, 1308, 1284, 1285, 1311, 1288, 1289, 1312, 1292, 1290, - /* 460 */ 1316, 1293, 1286, 1294, 1295, 1297, 1341, 1299, 1298, 1327, - /* 470 */ 1300, 1301, 1302, 1328, 1149, 1342, 1320, 1322, 1350, 1335, - /* 480 */ 1336, 1337, 1338, 1339, 1340, 1344, 1357, 1345, 1149, 1346, - /* 490 */ 1347, 1348, 1349, 1352, 1354, 1355, 1356, 1395, 1358, 1361, - /* 500 */ 1359, 1398, 1362, 1364, 1399, 1403, 1383, 1385, 1386, 1387, - /* 510 */ 1389, 1391, + /* 0 */ 598, 0, 39, 78, 78, 78, 78, 234, 78, 78, + /* 10 */ 312, 468, 120, 273, 312, 312, 312, 312, 312, 312, + /* 20 */ 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, + /* 30 */ 312, 312, 312, 206, 206, 206, 159, 1226, 1226, 34, + /* 40 */ 81, 81, 125, 1226, 81, 81, 81, 81, 81, 81, + /* 50 */ 360, 395, 465, 465, 125, 529, 395, 81, 81, 395, + /* 60 */ 81, 395, 529, 395, 395, 81, 512, 148, 431, 411, + /* 70 */ 411, 252, 425, 1232, 240, 1232, 1232, 1232, 1232, 1232, + /* 80 */ 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, + /* 90 */ 1232, 1232, 1232, 1232, 38, 409, 23, 23, 23, 650, + /* 100 */ 568, 529, 395, 395, 395, 597, 271, 271, 271, 271, + /* 110 */ 271, 271, 809, 288, 402, 372, 119, 477, 63, 663, + /* 120 */ 432, 444, 267, 444, 595, 533, 537, 775, 771, 779, + /* 130 */ 686, 775, 775, 715, 711, 711, 775, 828, 529, 854, + /* 140 */ 360, 568, 858, 360, 360, 775, 360, 877, 395, 395, + /* 150 */ 395, 395, 395, 395, 395, 395, 395, 395, 395, 779, + /* 160 */ 779, 775, 877, 568, 828, 783, 529, 854, 512, 568, + /* 170 */ 858, 512, 911, 757, 761, 796, 757, 761, 796, 796, + /* 180 */ 768, 782, 795, 799, 804, 568, 961, 867, 788, 790, + /* 190 */ 797, 926, 395, 761, 796, 796, 761, 796, 874, 568, + /* 200 */ 858, 512, 597, 512, 568, 948, 779, 779, 775, 512, + /* 210 */ 877, 1448, 1448, 1448, 1448, 1448, 605, 578, 491, 1276, + /* 220 */ 1202, 1224, 16, 108, 333, 136, 136, 136, 136, 136, + /* 230 */ 136, 136, 149, 36, 482, 426, 359, 359, 359, 359, + /* 240 */ 383, 361, 608, 622, 624, 659, 667, 590, 734, 747, + /* 250 */ 681, 669, 701, 706, 646, 522, 621, 586, 720, 643, + /* 260 */ 721, 100, 722, 726, 744, 751, 764, 668, 737, 785, + /* 270 */ 787, 791, 792, 816, 817, 184, 1046, 1047, 988, 1054, + /* 280 */ 1013, 904, 1020, 1021, 1022, 910, 1063, 1026, 1032, 921, + /* 290 */ 1080, 1043, 1082, 1048, 1084, 1018, 947, 949, 987, 955, + /* 300 */ 1094, 1095, 1053, 962, 1097, 1104, 1031, 1107, 1114, 1116, + /* 310 */ 1117, 1118, 1120, 1122, 1124, 1125, 1128, 1129, 1130, 1131, + /* 320 */ 1017, 1133, 1134, 1139, 1140, 1141, 1148, 1135, 1150, 1151, + /* 330 */ 1152, 1153, 1155, 1156, 1158, 1160, 1161, 1123, 1165, 1121, + /* 340 */ 1170, 1171, 1136, 1142, 1132, 1172, 1176, 1177, 1189, 1113, + /* 350 */ 1127, 1154, 1126, 1174, 1197, 1168, 1169, 1175, 1178, 1126, + /* 360 */ 1179, 1180, 1198, 1186, 1199, 1188, 1173, 1219, 1208, 1203, + /* 370 */ 1236, 1215, 1243, 1222, 1225, 1246, 1119, 1100, 1213, 1253, + /* 380 */ 1115, 1238, 1147, 1138, 1262, 1266, 1149, 1269, 1217, 1248, + /* 390 */ 1167, 1227, 1229, 1143, 1240, 1235, 1241, 1239, 1250, 1251, + /* 400 */ 1259, 1252, 1264, 1254, 1263, 1267, 1157, 1268, 1270, 1258, + /* 410 */ 1159, 1274, 1261, 1273, 1278, 1166, 1344, 1311, 1312, 1313, + /* 420 */ 1314, 1315, 1316, 1353, 1201, 1277, 1287, 1290, 1291, 1292, + /* 430 */ 1293, 1296, 1297, 1233, 1298, 1368, 1326, 1247, 1302, 1294, + /* 440 */ 1295, 1299, 1352, 1306, 1303, 1307, 1340, 1341, 1317, 1319, + /* 450 */ 1346, 1321, 1322, 1347, 1324, 1325, 1356, 1327, 1328, 1359, + /* 460 */ 1330, 1308, 1309, 1310, 1318, 1369, 1323, 1331, 1363, 1304, + /* 470 */ 1337, 1338, 1371, 1126, 1388, 1365, 1364, 1392, 1377, 1378, + /* 480 */ 1379, 1380, 1381, 1382, 1383, 1400, 1385, 1126, 1386, 1387, + /* 490 */ 1389, 1390, 1391, 1393, 1394, 1395, 1426, 1396, 1399, 1397, + /* 500 */ 1430, 1398, 1401, 1437, 1439, 1419, 1421, 1422, 1423, 1425, + /* 510 */ 1427, }; #define YY_REDUCE_COUNT (215) -#define YY_REDUCE_MIN (-266) -#define YY_REDUCE_MAX (1059) +#define YY_REDUCE_MIN (-226) +#define YY_REDUCE_MAX (1051) static const short yy_reduce_ofst[] = { - /* 0 */ 227, -9, 71, 168, 221, 285, 329, 403, 35, 502, - /* 10 */ 558, 606, 550, 660, 669, 703, 739, 747, 773, 212, - /* 20 */ 300, 802, 826, 850, 874, 903, 929, 937, 972, 978, - /* 30 */ 1007, 1013, 1059, -50, 113, 309, -10, -222, -218, -19, - /* 40 */ -207, -206, 358, -246, -187, -101, -86, -59, -2, 55, - /* 50 */ 80, -217, -84, 119, 161, -181, -16, 115, 143, 216, - /* 60 */ 366, 190, -179, 226, 290, 416, 215, -195, -266, -266, - /* 70 */ -266, 33, 174, 18, 193, 24, 183, 293, 307, 324, - /* 80 */ 365, 373, 393, 404, 427, 429, 430, 433, 437, 438, - /* 90 */ 454, 482, 486, 487, -202, 137, 274, 318, 343, 420, - /* 100 */ 351, -204, -15, 299, 457, -153, 48, 86, 120, 154, - /* 110 */ 224, 253, 251, 338, 353, 257, 387, 466, 453, 554, - /* 120 */ 508, 483, 483, 483, 545, 484, 501, 575, 535, 583, - /* 130 */ 556, 592, 608, 579, 612, 618, 620, 605, 624, 625, - /* 140 */ 664, 654, 653, 685, 687, 690, 691, 699, 683, 688, - /* 150 */ 689, 692, 694, 695, 696, 701, 702, 704, 705, 698, - /* 160 */ 723, 706, 733, 678, 686, 707, 710, 711, 735, 716, - /* 170 */ 712, 737, 719, 661, 726, 728, 667, 731, 730, 736, - /* 180 */ 674, 720, 721, 732, 483, 756, 740, 741, 722, 718, - /* 190 */ 727, 759, 545, 782, 785, 786, 792, 787, 798, 813, - /* 200 */ 803, 834, 829, 838, 827, 833, 841, 843, 852, 860, - /* 210 */ 857, 818, 851, 858, 862, 877, + /* 0 */ -180, 262, 495, -198, -58, 447, 538, 574, -129, 601, + /* 10 */ 656, -14, 617, 700, 705, 762, 176, 767, 803, 811, + /* 20 */ 837, 846, 873, 881, 917, 922, 958, 963, 969, 1010, + /* 30 */ 1025, 1036, 1051, -195, -170, -147, 197, -213, -204, 41, + /* 40 */ -209, -206, 247, -132, -208, -175, -1, 11, 111, 121, + /* 50 */ 318, -142, -176, -40, 118, -62, -214, 189, 335, -6, + /* 60 */ 354, 232, 110, 97, 314, 355, -55, -201, -156, -156, + /* 70 */ -156, 173, -185, -107, 126, 208, 259, 308, 310, 339, + /* 80 */ 362, 371, 388, 389, 390, 391, 406, 434, 435, 436, + /* 90 */ 437, 438, 440, 441, 122, 281, 329, 413, 414, 467, + /* 100 */ -10, 172, 65, -149, 338, 161, -226, -221, 387, 490, + /* 110 */ 497, 500, 454, 539, 530, 451, 456, 519, 475, 548, + /* 120 */ 503, 501, 501, 501, 552, 487, 511, 587, 555, 599, + /* 130 */ 571, 614, 619, 592, 602, 606, 636, 594, 628, 625, + /* 140 */ 664, 647, 642, 677, 678, 685, 679, 694, 676, 680, + /* 150 */ 682, 683, 688, 689, 690, 692, 697, 698, 699, 712, + /* 160 */ 713, 693, 723, 674, 655, 684, 691, 695, 725, 704, + /* 170 */ 707, 731, 687, 651, 714, 727, 666, 728, 730, 732, + /* 180 */ 671, 696, 702, 708, 501, 748, 729, 710, 703, 716, + /* 190 */ 724, 733, 552, 749, 745, 752, 750, 754, 759, 773, + /* 200 */ 769, 801, 798, 802, 786, 794, 806, 819, 813, 825, + /* 210 */ 831, 789, 820, 821, 827, 842, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 10 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 20 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 30 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 40 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 50 */ 1204, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 60 */ 1151, 1151, 1151, 1151, 1151, 1151, 1202, 1331, 1151, 1462, - /* 70 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 80 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 90 */ 1151, 1151, 1151, 1151, 1151, 1204, 1473, 1473, 1473, 1202, - /* 100 */ 1151, 1151, 1151, 1151, 1151, 1289, 1151, 1151, 1151, 1151, - /* 110 */ 1151, 1151, 1365, 1151, 1151, 1537, 1151, 1242, 1497, 1151, - /* 120 */ 1489, 1465, 1479, 1466, 1151, 1522, 1482, 1151, 1151, 1151, - /* 130 */ 1357, 1151, 1151, 1336, 1333, 1333, 1151, 1151, 1151, 1151, - /* 140 */ 1204, 1151, 1151, 1204, 1204, 1151, 1204, 1151, 1151, 1151, - /* 150 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 160 */ 1151, 1151, 1151, 1151, 1151, 1367, 1151, 1151, 1202, 1151, - /* 170 */ 1151, 1202, 1151, 1504, 1502, 1151, 1504, 1502, 1151, 1151, - /* 180 */ 1516, 1512, 1495, 1493, 1479, 1151, 1151, 1151, 1540, 1528, - /* 190 */ 1524, 1151, 1151, 1502, 1151, 1151, 1502, 1151, 1344, 1151, - /* 200 */ 1151, 1202, 1151, 1202, 1151, 1258, 1151, 1151, 1151, 1202, - /* 210 */ 1151, 1359, 1292, 1292, 1205, 1156, 1151, 1151, 1151, 1151, - /* 220 */ 1151, 1151, 1151, 1151, 1151, 1151, 1427, 1515, 1514, 1426, - /* 230 */ 1439, 1438, 1437, 1151, 1151, 1151, 1151, 1421, 1422, 1420, - /* 240 */ 1419, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 250 */ 1151, 1151, 1151, 1151, 1151, 1463, 1151, 1525, 1529, 1151, - /* 260 */ 1151, 1151, 1404, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 270 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 280 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 290 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 300 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 310 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 320 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 330 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 340 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 350 */ 1151, 1151, 1151, 1303, 1151, 1151, 1151, 1151, 1151, 1151, - /* 360 */ 1228, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 370 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 380 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 390 */ 1151, 1151, 1486, 1496, 1151, 1151, 1151, 1151, 1151, 1151, - /* 400 */ 1151, 1151, 1151, 1151, 1404, 1151, 1513, 1151, 1472, 1468, - /* 410 */ 1151, 1151, 1464, 1151, 1151, 1523, 1151, 1151, 1151, 1151, - /* 420 */ 1151, 1151, 1151, 1151, 1458, 1151, 1151, 1151, 1151, 1151, - /* 430 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 440 */ 1151, 1403, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1286, - /* 450 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 460 */ 1151, 1151, 1271, 1269, 1268, 1267, 1151, 1264, 1151, 1151, - /* 470 */ 1151, 1151, 1151, 1151, 1294, 1151, 1151, 1151, 1151, 1151, - /* 480 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1214, 1151, - /* 490 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 500 */ 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, 1151, - /* 510 */ 1151, 1151, + /* 0 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 10 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 20 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 30 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 40 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 50 */ 1205, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 60 */ 1152, 1152, 1152, 1152, 1152, 1152, 1203, 1332, 1152, 1464, + /* 70 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 80 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 90 */ 1152, 1152, 1152, 1152, 1152, 1205, 1475, 1475, 1475, 1203, + /* 100 */ 1152, 1152, 1152, 1152, 1152, 1290, 1152, 1152, 1152, 1152, + /* 110 */ 1152, 1152, 1366, 1152, 1152, 1539, 1152, 1243, 1499, 1152, + /* 120 */ 1491, 1467, 1481, 1468, 1152, 1524, 1484, 1152, 1152, 1152, + /* 130 */ 1358, 1152, 1152, 1337, 1334, 1334, 1152, 1152, 1152, 1152, + /* 140 */ 1205, 1152, 1152, 1205, 1205, 1152, 1205, 1152, 1152, 1152, + /* 150 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 160 */ 1152, 1152, 1152, 1152, 1152, 1368, 1152, 1152, 1203, 1152, + /* 170 */ 1152, 1203, 1152, 1506, 1504, 1152, 1506, 1504, 1152, 1152, + /* 180 */ 1518, 1514, 1497, 1495, 1481, 1152, 1152, 1152, 1542, 1530, + /* 190 */ 1526, 1152, 1152, 1504, 1152, 1152, 1504, 1152, 1345, 1152, + /* 200 */ 1152, 1203, 1152, 1203, 1152, 1259, 1152, 1152, 1152, 1203, + /* 210 */ 1152, 1360, 1293, 1293, 1206, 1157, 1152, 1152, 1152, 1152, + /* 220 */ 1152, 1152, 1152, 1152, 1152, 1428, 1517, 1516, 1427, 1441, + /* 230 */ 1440, 1439, 1152, 1152, 1152, 1152, 1422, 1423, 1421, 1420, + /* 240 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 250 */ 1152, 1152, 1152, 1152, 1465, 1152, 1527, 1531, 1152, 1152, + /* 260 */ 1152, 1405, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 270 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 280 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 290 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 300 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 310 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 320 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 330 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 340 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 350 */ 1152, 1152, 1304, 1152, 1152, 1152, 1152, 1152, 1152, 1229, + /* 360 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 370 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 380 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 390 */ 1152, 1488, 1498, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 400 */ 1152, 1152, 1152, 1405, 1152, 1515, 1152, 1474, 1470, 1152, + /* 410 */ 1152, 1466, 1152, 1152, 1525, 1152, 1152, 1152, 1152, 1152, + /* 420 */ 1152, 1152, 1152, 1460, 1152, 1152, 1152, 1152, 1152, 1152, + /* 430 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 440 */ 1404, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1287, 1152, + /* 450 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 460 */ 1152, 1272, 1270, 1269, 1268, 1152, 1265, 1152, 1152, 1152, + /* 470 */ 1152, 1152, 1152, 1295, 1152, 1152, 1152, 1152, 1152, 1152, + /* 480 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1215, 1152, 1152, + /* 490 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 500 */ 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, + /* 510 */ 1152, }; /********** End of lemon-generated parsing tables *****************************/ @@ -918,7 +924,7 @@ static const char *const yyTokenName[] = { /* 154 */ "SYNCDB", /* 155 */ "NULL", /* 156 */ "NK_VARIABLE", - /* 157 */ "NK_UNDERLINE", + /* 157 */ "NOW", /* 158 */ "ROWTS", /* 159 */ "TBNAME", /* 160 */ "QSTARTTS", @@ -1348,119 +1354,120 @@ static const char *const yyRuleName[] = { /* 273 */ "expression_list ::= expression_list NK_COMMA expression", /* 274 */ "column_reference ::= column_name", /* 275 */ "column_reference ::= table_name NK_DOT column_name", - /* 276 */ "pseudo_column ::= NK_UNDERLINE ROWTS", - /* 277 */ "pseudo_column ::= TBNAME", - /* 278 */ "pseudo_column ::= NK_UNDERLINE QSTARTTS", - /* 279 */ "pseudo_column ::= NK_UNDERLINE QENDTS", - /* 280 */ "pseudo_column ::= NK_UNDERLINE WSTARTTS", - /* 281 */ "pseudo_column ::= NK_UNDERLINE WENDTS", - /* 282 */ "pseudo_column ::= NK_UNDERLINE WDURATION", - /* 283 */ "predicate ::= expression compare_op expression", - /* 284 */ "predicate ::= expression BETWEEN expression AND expression", - /* 285 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 286 */ "predicate ::= expression IS NULL", - /* 287 */ "predicate ::= expression IS NOT NULL", - /* 288 */ "predicate ::= expression in_op in_predicate_value", - /* 289 */ "compare_op ::= NK_LT", - /* 290 */ "compare_op ::= NK_GT", - /* 291 */ "compare_op ::= NK_LE", - /* 292 */ "compare_op ::= NK_GE", - /* 293 */ "compare_op ::= NK_NE", - /* 294 */ "compare_op ::= NK_EQ", - /* 295 */ "compare_op ::= LIKE", - /* 296 */ "compare_op ::= NOT LIKE", - /* 297 */ "compare_op ::= MATCH", - /* 298 */ "compare_op ::= NMATCH", - /* 299 */ "in_op ::= IN", - /* 300 */ "in_op ::= NOT IN", - /* 301 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 302 */ "boolean_value_expression ::= boolean_primary", - /* 303 */ "boolean_value_expression ::= NOT boolean_primary", - /* 304 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 305 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 306 */ "boolean_primary ::= predicate", - /* 307 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 308 */ "common_expression ::= expression", - /* 309 */ "common_expression ::= boolean_value_expression", - /* 310 */ "from_clause ::= FROM table_reference_list", - /* 311 */ "table_reference_list ::= table_reference", - /* 312 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 313 */ "table_reference ::= table_primary", - /* 314 */ "table_reference ::= joined_table", - /* 315 */ "table_primary ::= table_name alias_opt", - /* 316 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 317 */ "table_primary ::= subquery alias_opt", - /* 318 */ "table_primary ::= parenthesized_joined_table", - /* 319 */ "alias_opt ::=", - /* 320 */ "alias_opt ::= table_alias", - /* 321 */ "alias_opt ::= AS table_alias", - /* 322 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 323 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 324 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 325 */ "join_type ::=", - /* 326 */ "join_type ::= INNER", - /* 327 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 328 */ "set_quantifier_opt ::=", - /* 329 */ "set_quantifier_opt ::= DISTINCT", - /* 330 */ "set_quantifier_opt ::= ALL", - /* 331 */ "select_list ::= NK_STAR", - /* 332 */ "select_list ::= select_sublist", - /* 333 */ "select_sublist ::= select_item", - /* 334 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 335 */ "select_item ::= common_expression", - /* 336 */ "select_item ::= common_expression column_alias", - /* 337 */ "select_item ::= common_expression AS column_alias", - /* 338 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 339 */ "where_clause_opt ::=", - /* 340 */ "where_clause_opt ::= WHERE search_condition", - /* 341 */ "partition_by_clause_opt ::=", - /* 342 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 343 */ "twindow_clause_opt ::=", - /* 344 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 345 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 346 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 347 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 348 */ "sliding_opt ::=", - /* 349 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 350 */ "fill_opt ::=", - /* 351 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 352 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 353 */ "fill_mode ::= NONE", - /* 354 */ "fill_mode ::= PREV", - /* 355 */ "fill_mode ::= NULL", - /* 356 */ "fill_mode ::= LINEAR", - /* 357 */ "fill_mode ::= NEXT", - /* 358 */ "group_by_clause_opt ::=", - /* 359 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 360 */ "group_by_list ::= expression", - /* 361 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 362 */ "having_clause_opt ::=", - /* 363 */ "having_clause_opt ::= HAVING search_condition", - /* 364 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 365 */ "query_expression_body ::= query_primary", - /* 366 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 367 */ "query_primary ::= query_specification", - /* 368 */ "order_by_clause_opt ::=", - /* 369 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 370 */ "slimit_clause_opt ::=", - /* 371 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 372 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 373 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 374 */ "limit_clause_opt ::=", - /* 375 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 376 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 377 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 378 */ "subquery ::= NK_LP query_expression NK_RP", - /* 379 */ "search_condition ::= common_expression", - /* 380 */ "sort_specification_list ::= sort_specification", - /* 381 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 382 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 383 */ "ordering_specification_opt ::=", - /* 384 */ "ordering_specification_opt ::= ASC", - /* 385 */ "ordering_specification_opt ::= DESC", - /* 386 */ "null_ordering_opt ::=", - /* 387 */ "null_ordering_opt ::= NULLS FIRST", - /* 388 */ "null_ordering_opt ::= NULLS LAST", + /* 276 */ "pseudo_column ::= NOW", + /* 277 */ "pseudo_column ::= ROWTS", + /* 278 */ "pseudo_column ::= TBNAME", + /* 279 */ "pseudo_column ::= QSTARTTS", + /* 280 */ "pseudo_column ::= QENDTS", + /* 281 */ "pseudo_column ::= WSTARTTS", + /* 282 */ "pseudo_column ::= WENDTS", + /* 283 */ "pseudo_column ::= WDURATION", + /* 284 */ "predicate ::= expression compare_op expression", + /* 285 */ "predicate ::= expression BETWEEN expression AND expression", + /* 286 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 287 */ "predicate ::= expression IS NULL", + /* 288 */ "predicate ::= expression IS NOT NULL", + /* 289 */ "predicate ::= expression in_op in_predicate_value", + /* 290 */ "compare_op ::= NK_LT", + /* 291 */ "compare_op ::= NK_GT", + /* 292 */ "compare_op ::= NK_LE", + /* 293 */ "compare_op ::= NK_GE", + /* 294 */ "compare_op ::= NK_NE", + /* 295 */ "compare_op ::= NK_EQ", + /* 296 */ "compare_op ::= LIKE", + /* 297 */ "compare_op ::= NOT LIKE", + /* 298 */ "compare_op ::= MATCH", + /* 299 */ "compare_op ::= NMATCH", + /* 300 */ "in_op ::= IN", + /* 301 */ "in_op ::= NOT IN", + /* 302 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 303 */ "boolean_value_expression ::= boolean_primary", + /* 304 */ "boolean_value_expression ::= NOT boolean_primary", + /* 305 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 306 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 307 */ "boolean_primary ::= predicate", + /* 308 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 309 */ "common_expression ::= expression", + /* 310 */ "common_expression ::= boolean_value_expression", + /* 311 */ "from_clause ::= FROM table_reference_list", + /* 312 */ "table_reference_list ::= table_reference", + /* 313 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 314 */ "table_reference ::= table_primary", + /* 315 */ "table_reference ::= joined_table", + /* 316 */ "table_primary ::= table_name alias_opt", + /* 317 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 318 */ "table_primary ::= subquery alias_opt", + /* 319 */ "table_primary ::= parenthesized_joined_table", + /* 320 */ "alias_opt ::=", + /* 321 */ "alias_opt ::= table_alias", + /* 322 */ "alias_opt ::= AS table_alias", + /* 323 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 324 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 325 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 326 */ "join_type ::=", + /* 327 */ "join_type ::= INNER", + /* 328 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 329 */ "set_quantifier_opt ::=", + /* 330 */ "set_quantifier_opt ::= DISTINCT", + /* 331 */ "set_quantifier_opt ::= ALL", + /* 332 */ "select_list ::= NK_STAR", + /* 333 */ "select_list ::= select_sublist", + /* 334 */ "select_sublist ::= select_item", + /* 335 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 336 */ "select_item ::= common_expression", + /* 337 */ "select_item ::= common_expression column_alias", + /* 338 */ "select_item ::= common_expression AS column_alias", + /* 339 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 340 */ "where_clause_opt ::=", + /* 341 */ "where_clause_opt ::= WHERE search_condition", + /* 342 */ "partition_by_clause_opt ::=", + /* 343 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 344 */ "twindow_clause_opt ::=", + /* 345 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 346 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 347 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 348 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 349 */ "sliding_opt ::=", + /* 350 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 351 */ "fill_opt ::=", + /* 352 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 353 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 354 */ "fill_mode ::= NONE", + /* 355 */ "fill_mode ::= PREV", + /* 356 */ "fill_mode ::= NULL", + /* 357 */ "fill_mode ::= LINEAR", + /* 358 */ "fill_mode ::= NEXT", + /* 359 */ "group_by_clause_opt ::=", + /* 360 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 361 */ "group_by_list ::= expression", + /* 362 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 363 */ "having_clause_opt ::=", + /* 364 */ "having_clause_opt ::= HAVING search_condition", + /* 365 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 366 */ "query_expression_body ::= query_primary", + /* 367 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 368 */ "query_primary ::= query_specification", + /* 369 */ "order_by_clause_opt ::=", + /* 370 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 371 */ "slimit_clause_opt ::=", + /* 372 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 373 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 374 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 375 */ "limit_clause_opt ::=", + /* 376 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 377 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 378 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 379 */ "subquery ::= NK_LP query_expression NK_RP", + /* 380 */ "search_condition ::= common_expression", + /* 381 */ "sort_specification_list ::= sort_specification", + /* 382 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 383 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 384 */ "ordering_specification_opt ::=", + /* 385 */ "ordering_specification_opt ::= ASC", + /* 386 */ "ordering_specification_opt ::= DESC", + /* 387 */ "null_ordering_opt ::=", + /* 388 */ "null_ordering_opt ::= NULLS FIRST", + /* 389 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2308,119 +2315,120 @@ static const struct { { 249, -3 }, /* (273) expression_list ::= expression_list NK_COMMA expression */ { 264, -1 }, /* (274) column_reference ::= column_name */ { 264, -3 }, /* (275) column_reference ::= table_name NK_DOT column_name */ - { 263, -2 }, /* (276) pseudo_column ::= NK_UNDERLINE ROWTS */ - { 263, -1 }, /* (277) pseudo_column ::= TBNAME */ - { 263, -2 }, /* (278) pseudo_column ::= NK_UNDERLINE QSTARTTS */ - { 263, -2 }, /* (279) pseudo_column ::= NK_UNDERLINE QENDTS */ - { 263, -2 }, /* (280) pseudo_column ::= NK_UNDERLINE WSTARTTS */ - { 263, -2 }, /* (281) pseudo_column ::= NK_UNDERLINE WENDTS */ - { 263, -2 }, /* (282) pseudo_column ::= NK_UNDERLINE WDURATION */ - { 266, -3 }, /* (283) predicate ::= expression compare_op expression */ - { 266, -5 }, /* (284) predicate ::= expression BETWEEN expression AND expression */ - { 266, -6 }, /* (285) predicate ::= expression NOT BETWEEN expression AND expression */ - { 266, -3 }, /* (286) predicate ::= expression IS NULL */ - { 266, -4 }, /* (287) predicate ::= expression IS NOT NULL */ - { 266, -3 }, /* (288) predicate ::= expression in_op in_predicate_value */ - { 267, -1 }, /* (289) compare_op ::= NK_LT */ - { 267, -1 }, /* (290) compare_op ::= NK_GT */ - { 267, -1 }, /* (291) compare_op ::= NK_LE */ - { 267, -1 }, /* (292) compare_op ::= NK_GE */ - { 267, -1 }, /* (293) compare_op ::= NK_NE */ - { 267, -1 }, /* (294) compare_op ::= NK_EQ */ - { 267, -1 }, /* (295) compare_op ::= LIKE */ - { 267, -2 }, /* (296) compare_op ::= NOT LIKE */ - { 267, -1 }, /* (297) compare_op ::= MATCH */ - { 267, -1 }, /* (298) compare_op ::= NMATCH */ - { 268, -1 }, /* (299) in_op ::= IN */ - { 268, -2 }, /* (300) in_op ::= NOT IN */ - { 269, -3 }, /* (301) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 270, -1 }, /* (302) boolean_value_expression ::= boolean_primary */ - { 270, -2 }, /* (303) boolean_value_expression ::= NOT boolean_primary */ - { 270, -3 }, /* (304) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 270, -3 }, /* (305) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 271, -1 }, /* (306) boolean_primary ::= predicate */ - { 271, -3 }, /* (307) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 272, -1 }, /* (308) common_expression ::= expression */ - { 272, -1 }, /* (309) common_expression ::= boolean_value_expression */ - { 273, -2 }, /* (310) from_clause ::= FROM table_reference_list */ - { 274, -1 }, /* (311) table_reference_list ::= table_reference */ - { 274, -3 }, /* (312) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 275, -1 }, /* (313) table_reference ::= table_primary */ - { 275, -1 }, /* (314) table_reference ::= joined_table */ - { 276, -2 }, /* (315) table_primary ::= table_name alias_opt */ - { 276, -4 }, /* (316) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 276, -2 }, /* (317) table_primary ::= subquery alias_opt */ - { 276, -1 }, /* (318) table_primary ::= parenthesized_joined_table */ - { 278, 0 }, /* (319) alias_opt ::= */ - { 278, -1 }, /* (320) alias_opt ::= table_alias */ - { 278, -2 }, /* (321) alias_opt ::= AS table_alias */ - { 279, -3 }, /* (322) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 279, -3 }, /* (323) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 277, -6 }, /* (324) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 280, 0 }, /* (325) join_type ::= */ - { 280, -1 }, /* (326) join_type ::= INNER */ - { 282, -9 }, /* (327) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 283, 0 }, /* (328) set_quantifier_opt ::= */ - { 283, -1 }, /* (329) set_quantifier_opt ::= DISTINCT */ - { 283, -1 }, /* (330) set_quantifier_opt ::= ALL */ - { 284, -1 }, /* (331) select_list ::= NK_STAR */ - { 284, -1 }, /* (332) select_list ::= select_sublist */ - { 290, -1 }, /* (333) select_sublist ::= select_item */ - { 290, -3 }, /* (334) select_sublist ::= select_sublist NK_COMMA select_item */ - { 291, -1 }, /* (335) select_item ::= common_expression */ - { 291, -2 }, /* (336) select_item ::= common_expression column_alias */ - { 291, -3 }, /* (337) select_item ::= common_expression AS column_alias */ - { 291, -3 }, /* (338) select_item ::= table_name NK_DOT NK_STAR */ - { 285, 0 }, /* (339) where_clause_opt ::= */ - { 285, -2 }, /* (340) where_clause_opt ::= WHERE search_condition */ - { 286, 0 }, /* (341) partition_by_clause_opt ::= */ - { 286, -3 }, /* (342) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 287, 0 }, /* (343) twindow_clause_opt ::= */ - { 287, -6 }, /* (344) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 287, -4 }, /* (345) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 287, -6 }, /* (346) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 287, -8 }, /* (347) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 247, 0 }, /* (348) sliding_opt ::= */ - { 247, -4 }, /* (349) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 292, 0 }, /* (350) fill_opt ::= */ - { 292, -4 }, /* (351) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 292, -6 }, /* (352) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 293, -1 }, /* (353) fill_mode ::= NONE */ - { 293, -1 }, /* (354) fill_mode ::= PREV */ - { 293, -1 }, /* (355) fill_mode ::= NULL */ - { 293, -1 }, /* (356) fill_mode ::= LINEAR */ - { 293, -1 }, /* (357) fill_mode ::= NEXT */ - { 288, 0 }, /* (358) group_by_clause_opt ::= */ - { 288, -3 }, /* (359) group_by_clause_opt ::= GROUP BY group_by_list */ - { 294, -1 }, /* (360) group_by_list ::= expression */ - { 294, -3 }, /* (361) group_by_list ::= group_by_list NK_COMMA expression */ - { 289, 0 }, /* (362) having_clause_opt ::= */ - { 289, -2 }, /* (363) having_clause_opt ::= HAVING search_condition */ - { 251, -4 }, /* (364) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 295, -1 }, /* (365) query_expression_body ::= query_primary */ - { 295, -4 }, /* (366) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 299, -1 }, /* (367) query_primary ::= query_specification */ - { 296, 0 }, /* (368) order_by_clause_opt ::= */ - { 296, -3 }, /* (369) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 297, 0 }, /* (370) slimit_clause_opt ::= */ - { 297, -2 }, /* (371) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 297, -4 }, /* (372) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 297, -4 }, /* (373) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 298, 0 }, /* (374) limit_clause_opt ::= */ - { 298, -2 }, /* (375) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 298, -4 }, /* (376) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 298, -4 }, /* (377) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 265, -3 }, /* (378) subquery ::= NK_LP query_expression NK_RP */ - { 281, -1 }, /* (379) search_condition ::= common_expression */ - { 300, -1 }, /* (380) sort_specification_list ::= sort_specification */ - { 300, -3 }, /* (381) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 301, -3 }, /* (382) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 302, 0 }, /* (383) ordering_specification_opt ::= */ - { 302, -1 }, /* (384) ordering_specification_opt ::= ASC */ - { 302, -1 }, /* (385) ordering_specification_opt ::= DESC */ - { 303, 0 }, /* (386) null_ordering_opt ::= */ - { 303, -2 }, /* (387) null_ordering_opt ::= NULLS FIRST */ - { 303, -2 }, /* (388) null_ordering_opt ::= NULLS LAST */ + { 263, -1 }, /* (276) pseudo_column ::= NOW */ + { 263, -1 }, /* (277) pseudo_column ::= ROWTS */ + { 263, -1 }, /* (278) pseudo_column ::= TBNAME */ + { 263, -1 }, /* (279) pseudo_column ::= QSTARTTS */ + { 263, -1 }, /* (280) pseudo_column ::= QENDTS */ + { 263, -1 }, /* (281) pseudo_column ::= WSTARTTS */ + { 263, -1 }, /* (282) pseudo_column ::= WENDTS */ + { 263, -1 }, /* (283) pseudo_column ::= WDURATION */ + { 266, -3 }, /* (284) predicate ::= expression compare_op expression */ + { 266, -5 }, /* (285) predicate ::= expression BETWEEN expression AND expression */ + { 266, -6 }, /* (286) predicate ::= expression NOT BETWEEN expression AND expression */ + { 266, -3 }, /* (287) predicate ::= expression IS NULL */ + { 266, -4 }, /* (288) predicate ::= expression IS NOT NULL */ + { 266, -3 }, /* (289) predicate ::= expression in_op in_predicate_value */ + { 267, -1 }, /* (290) compare_op ::= NK_LT */ + { 267, -1 }, /* (291) compare_op ::= NK_GT */ + { 267, -1 }, /* (292) compare_op ::= NK_LE */ + { 267, -1 }, /* (293) compare_op ::= NK_GE */ + { 267, -1 }, /* (294) compare_op ::= NK_NE */ + { 267, -1 }, /* (295) compare_op ::= NK_EQ */ + { 267, -1 }, /* (296) compare_op ::= LIKE */ + { 267, -2 }, /* (297) compare_op ::= NOT LIKE */ + { 267, -1 }, /* (298) compare_op ::= MATCH */ + { 267, -1 }, /* (299) compare_op ::= NMATCH */ + { 268, -1 }, /* (300) in_op ::= IN */ + { 268, -2 }, /* (301) in_op ::= NOT IN */ + { 269, -3 }, /* (302) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 270, -1 }, /* (303) boolean_value_expression ::= boolean_primary */ + { 270, -2 }, /* (304) boolean_value_expression ::= NOT boolean_primary */ + { 270, -3 }, /* (305) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 270, -3 }, /* (306) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 271, -1 }, /* (307) boolean_primary ::= predicate */ + { 271, -3 }, /* (308) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 272, -1 }, /* (309) common_expression ::= expression */ + { 272, -1 }, /* (310) common_expression ::= boolean_value_expression */ + { 273, -2 }, /* (311) from_clause ::= FROM table_reference_list */ + { 274, -1 }, /* (312) table_reference_list ::= table_reference */ + { 274, -3 }, /* (313) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 275, -1 }, /* (314) table_reference ::= table_primary */ + { 275, -1 }, /* (315) table_reference ::= joined_table */ + { 276, -2 }, /* (316) table_primary ::= table_name alias_opt */ + { 276, -4 }, /* (317) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 276, -2 }, /* (318) table_primary ::= subquery alias_opt */ + { 276, -1 }, /* (319) table_primary ::= parenthesized_joined_table */ + { 278, 0 }, /* (320) alias_opt ::= */ + { 278, -1 }, /* (321) alias_opt ::= table_alias */ + { 278, -2 }, /* (322) alias_opt ::= AS table_alias */ + { 279, -3 }, /* (323) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 279, -3 }, /* (324) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 277, -6 }, /* (325) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 280, 0 }, /* (326) join_type ::= */ + { 280, -1 }, /* (327) join_type ::= INNER */ + { 282, -9 }, /* (328) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 283, 0 }, /* (329) set_quantifier_opt ::= */ + { 283, -1 }, /* (330) set_quantifier_opt ::= DISTINCT */ + { 283, -1 }, /* (331) set_quantifier_opt ::= ALL */ + { 284, -1 }, /* (332) select_list ::= NK_STAR */ + { 284, -1 }, /* (333) select_list ::= select_sublist */ + { 290, -1 }, /* (334) select_sublist ::= select_item */ + { 290, -3 }, /* (335) select_sublist ::= select_sublist NK_COMMA select_item */ + { 291, -1 }, /* (336) select_item ::= common_expression */ + { 291, -2 }, /* (337) select_item ::= common_expression column_alias */ + { 291, -3 }, /* (338) select_item ::= common_expression AS column_alias */ + { 291, -3 }, /* (339) select_item ::= table_name NK_DOT NK_STAR */ + { 285, 0 }, /* (340) where_clause_opt ::= */ + { 285, -2 }, /* (341) where_clause_opt ::= WHERE search_condition */ + { 286, 0 }, /* (342) partition_by_clause_opt ::= */ + { 286, -3 }, /* (343) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 287, 0 }, /* (344) twindow_clause_opt ::= */ + { 287, -6 }, /* (345) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 287, -4 }, /* (346) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 287, -6 }, /* (347) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 287, -8 }, /* (348) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 247, 0 }, /* (349) sliding_opt ::= */ + { 247, -4 }, /* (350) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 292, 0 }, /* (351) fill_opt ::= */ + { 292, -4 }, /* (352) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 292, -6 }, /* (353) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 293, -1 }, /* (354) fill_mode ::= NONE */ + { 293, -1 }, /* (355) fill_mode ::= PREV */ + { 293, -1 }, /* (356) fill_mode ::= NULL */ + { 293, -1 }, /* (357) fill_mode ::= LINEAR */ + { 293, -1 }, /* (358) fill_mode ::= NEXT */ + { 288, 0 }, /* (359) group_by_clause_opt ::= */ + { 288, -3 }, /* (360) group_by_clause_opt ::= GROUP BY group_by_list */ + { 294, -1 }, /* (361) group_by_list ::= expression */ + { 294, -3 }, /* (362) group_by_list ::= group_by_list NK_COMMA expression */ + { 289, 0 }, /* (363) having_clause_opt ::= */ + { 289, -2 }, /* (364) having_clause_opt ::= HAVING search_condition */ + { 251, -4 }, /* (365) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 295, -1 }, /* (366) query_expression_body ::= query_primary */ + { 295, -4 }, /* (367) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 299, -1 }, /* (368) query_primary ::= query_specification */ + { 296, 0 }, /* (369) order_by_clause_opt ::= */ + { 296, -3 }, /* (370) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 297, 0 }, /* (371) slimit_clause_opt ::= */ + { 297, -2 }, /* (372) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 297, -4 }, /* (373) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 297, -4 }, /* (374) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 298, 0 }, /* (375) limit_clause_opt ::= */ + { 298, -2 }, /* (376) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 298, -4 }, /* (377) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 298, -4 }, /* (378) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 265, -3 }, /* (379) subquery ::= NK_LP query_expression NK_RP */ + { 281, -1 }, /* (380) search_condition ::= common_expression */ + { 300, -1 }, /* (381) sort_specification_list ::= sort_specification */ + { 300, -3 }, /* (382) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 301, -3 }, /* (383) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 302, 0 }, /* (384) ordering_specification_opt ::= */ + { 302, -1 }, /* (385) ordering_specification_opt ::= ASC */ + { 302, -1 }, /* (386) ordering_specification_opt ::= DESC */ + { 303, 0 }, /* (387) null_ordering_opt ::= */ + { 303, -2 }, /* (388) null_ordering_opt ::= NULLS FIRST */ + { 303, -2 }, /* (389) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2639,7 +2647,7 @@ static YYACTIONTYPE yy_reduce( case 50: /* exists_opt ::= */ yytestcase(yyruleno==50); case 203: /* analyze_opt ::= */ yytestcase(yyruleno==203); case 211: /* agg_func_opt ::= */ yytestcase(yyruleno==211); - case 328: /* set_quantifier_opt ::= */ yytestcase(yyruleno==328); + case 329: /* set_quantifier_opt ::= */ yytestcase(yyruleno==329); { yymsp[1].minor.yy185 = false; } break; case 49: /* exists_opt ::= IF EXISTS */ @@ -2823,8 +2831,8 @@ static YYACTIONTYPE yy_reduce( case 184: /* func_name_list ::= func_name */ yytestcase(yyruleno==184); case 193: /* func_list ::= func */ yytestcase(yyruleno==193); case 246: /* literal_list ::= signed_literal */ yytestcase(yyruleno==246); - case 333: /* select_sublist ::= select_item */ yytestcase(yyruleno==333); - case 380: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==380); + case 334: /* select_sublist ::= select_item */ yytestcase(yyruleno==334); + case 381: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==381); { yylhsminor.yy312 = createNodeList(pCxt, yymsp[0].minor.yy104); } yymsp[0].minor.yy312 = yylhsminor.yy312; break; @@ -2843,9 +2851,9 @@ static YYACTIONTYPE yy_reduce( break; case 104: /* specific_tags_opt ::= */ case 135: /* tags_def_opt ::= */ yytestcase(yyruleno==135); - case 341: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==341); - case 358: /* group_by_clause_opt ::= */ yytestcase(yyruleno==358); - case 368: /* order_by_clause_opt ::= */ yytestcase(yyruleno==368); + case 342: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==342); + case 359: /* group_by_clause_opt ::= */ yytestcase(yyruleno==359); + case 369: /* order_by_clause_opt ::= */ yytestcase(yyruleno==369); { yymsp[1].minor.yy312 = NULL; } break; case 105: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ @@ -2864,8 +2872,8 @@ static YYACTIONTYPE yy_reduce( case 185: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==185); case 194: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==194); case 247: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==247); - case 334: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==334); - case 381: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==381); + case 335: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==335); + case 382: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==382); { yylhsminor.yy312 = addNodeToList(pCxt, yymsp[-2].minor.yy312, yymsp[0].minor.yy104); } yymsp[-2].minor.yy312 = yylhsminor.yy312; break; @@ -2945,7 +2953,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[-5].minor.yy336 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 136: /* tags_def_opt ::= tags_def */ - case 332: /* select_list ::= select_sublist */ yytestcase(yyruleno==332); + case 333: /* select_list ::= select_sublist */ yytestcase(yyruleno==333); { yylhsminor.yy312 = yymsp[0].minor.yy312; } yymsp[0].minor.yy312 = yylhsminor.yy312; break; @@ -3083,13 +3091,13 @@ static YYACTIONTYPE yy_reduce( break; case 179: /* like_pattern_opt ::= */ case 190: /* index_options ::= */ yytestcase(yyruleno==190); - case 339: /* where_clause_opt ::= */ yytestcase(yyruleno==339); - case 343: /* twindow_clause_opt ::= */ yytestcase(yyruleno==343); - case 348: /* sliding_opt ::= */ yytestcase(yyruleno==348); - case 350: /* fill_opt ::= */ yytestcase(yyruleno==350); - case 362: /* having_clause_opt ::= */ yytestcase(yyruleno==362); - case 370: /* slimit_clause_opt ::= */ yytestcase(yyruleno==370); - case 374: /* limit_clause_opt ::= */ yytestcase(yyruleno==374); + case 340: /* where_clause_opt ::= */ yytestcase(yyruleno==340); + case 344: /* twindow_clause_opt ::= */ yytestcase(yyruleno==344); + case 349: /* sliding_opt ::= */ yytestcase(yyruleno==349); + case 351: /* fill_opt ::= */ yytestcase(yyruleno==351); + case 363: /* having_clause_opt ::= */ yytestcase(yyruleno==363); + case 371: /* slimit_clause_opt ::= */ yytestcase(yyruleno==371); + case 375: /* limit_clause_opt ::= */ yytestcase(yyruleno==375); { yymsp[1].minor.yy104 = NULL; } break; case 180: /* like_pattern_opt ::= LIKE NK_STRING */ @@ -3146,7 +3154,7 @@ static YYACTIONTYPE yy_reduce( break; case 204: /* analyze_opt ::= ANALYZE */ case 212: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==212); - case 329: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==329); + case 330: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==330); { yymsp[0].minor.yy185 = true; } break; case 205: /* explain_options ::= */ @@ -3228,16 +3236,16 @@ static YYACTIONTYPE yy_reduce( case 259: /* expression ::= pseudo_column */ yytestcase(yyruleno==259); case 260: /* expression ::= column_reference */ yytestcase(yyruleno==260); case 263: /* expression ::= subquery */ yytestcase(yyruleno==263); - case 302: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==302); - case 306: /* boolean_primary ::= predicate */ yytestcase(yyruleno==306); - case 308: /* common_expression ::= expression */ yytestcase(yyruleno==308); - case 309: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==309); - case 311: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==311); - case 313: /* table_reference ::= table_primary */ yytestcase(yyruleno==313); - case 314: /* table_reference ::= joined_table */ yytestcase(yyruleno==314); - case 318: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==318); - case 365: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==365); - case 367: /* query_primary ::= query_specification */ yytestcase(yyruleno==367); + case 303: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==303); + case 307: /* boolean_primary ::= predicate */ yytestcase(yyruleno==307); + case 309: /* common_expression ::= expression */ yytestcase(yyruleno==309); + case 310: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==310); + case 312: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==312); + case 314: /* table_reference ::= table_primary */ yytestcase(yyruleno==314); + case 315: /* table_reference ::= joined_table */ yytestcase(yyruleno==315); + case 319: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==319); + case 366: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==366); + case 368: /* query_primary ::= query_specification */ yytestcase(yyruleno==368); { yylhsminor.yy104 = yymsp[0].minor.yy104; } yymsp[0].minor.yy104 = yylhsminor.yy104; break; @@ -3291,7 +3299,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[-1].minor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; case 244: /* signed_literal ::= duration_literal */ - case 379: /* search_condition ::= common_expression */ yytestcase(yyruleno==379); + case 380: /* search_condition ::= common_expression */ yytestcase(yyruleno==380); { yylhsminor.yy104 = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); } yymsp[0].minor.yy104 = yylhsminor.yy104; break; @@ -3307,7 +3315,7 @@ static YYACTIONTYPE yy_reduce( yymsp[-3].minor.yy104 = yylhsminor.yy104; break; case 264: /* expression ::= NK_LP expression NK_RP */ - case 307: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==307); + case 308: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==308); { yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; @@ -3321,7 +3329,7 @@ static YYACTIONTYPE yy_reduce( case 266: /* expression ::= NK_MINUS expression */ { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), NULL)); + yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), NULL)); } yymsp[-1].minor.yy104 = yylhsminor.yy104; break; @@ -3381,25 +3389,19 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129, createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129)); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 276: /* pseudo_column ::= NK_UNDERLINE ROWTS */ - case 278: /* pseudo_column ::= NK_UNDERLINE QSTARTTS */ yytestcase(yyruleno==278); - case 279: /* pseudo_column ::= NK_UNDERLINE QENDTS */ yytestcase(yyruleno==279); - case 280: /* pseudo_column ::= NK_UNDERLINE WSTARTTS */ yytestcase(yyruleno==280); - case 281: /* pseudo_column ::= NK_UNDERLINE WENDTS */ yytestcase(yyruleno==281); - case 282: /* pseudo_column ::= NK_UNDERLINE WDURATION */ yytestcase(yyruleno==282); -{ - SToken t = yymsp[-1].minor.yy0; - t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy104 = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL)); - } - yymsp[-1].minor.yy104 = yylhsminor.yy104; - break; - case 277: /* pseudo_column ::= TBNAME */ + case 276: /* pseudo_column ::= NOW */ + case 277: /* pseudo_column ::= ROWTS */ yytestcase(yyruleno==277); + case 278: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==278); + case 279: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==279); + case 280: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==280); + case 281: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==281); + case 282: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==282); + case 283: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==283); { yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } yymsp[0].minor.yy104 = yylhsminor.yy104; break; - case 283: /* predicate ::= expression compare_op expression */ - case 288: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==288); + case 284: /* predicate ::= expression compare_op expression */ + case 289: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==289); { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); @@ -3407,7 +3409,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 284: /* predicate ::= expression BETWEEN expression AND expression */ + case 285: /* predicate ::= expression BETWEEN expression AND expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy104); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); @@ -3415,7 +3417,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy104 = yylhsminor.yy104; break; - case 285: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 286: /* predicate ::= expression NOT BETWEEN expression AND expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy104); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); @@ -3423,68 +3425,68 @@ static YYACTIONTYPE yy_reduce( } yymsp[-5].minor.yy104 = yylhsminor.yy104; break; - case 286: /* predicate ::= expression IS NULL */ + case 287: /* predicate ::= expression IS NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), NULL)); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 287: /* predicate ::= expression IS NOT NULL */ + case 288: /* predicate ::= expression IS NOT NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy104); yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), NULL)); } yymsp[-3].minor.yy104 = yylhsminor.yy104; break; - case 289: /* compare_op ::= NK_LT */ + case 290: /* compare_op ::= NK_LT */ { yymsp[0].minor.yy60 = OP_TYPE_LOWER_THAN; } break; - case 290: /* compare_op ::= NK_GT */ + case 291: /* compare_op ::= NK_GT */ { yymsp[0].minor.yy60 = OP_TYPE_GREATER_THAN; } break; - case 291: /* compare_op ::= NK_LE */ + case 292: /* compare_op ::= NK_LE */ { yymsp[0].minor.yy60 = OP_TYPE_LOWER_EQUAL; } break; - case 292: /* compare_op ::= NK_GE */ + case 293: /* compare_op ::= NK_GE */ { yymsp[0].minor.yy60 = OP_TYPE_GREATER_EQUAL; } break; - case 293: /* compare_op ::= NK_NE */ + case 294: /* compare_op ::= NK_NE */ { yymsp[0].minor.yy60 = OP_TYPE_NOT_EQUAL; } break; - case 294: /* compare_op ::= NK_EQ */ + case 295: /* compare_op ::= NK_EQ */ { yymsp[0].minor.yy60 = OP_TYPE_EQUAL; } break; - case 295: /* compare_op ::= LIKE */ + case 296: /* compare_op ::= LIKE */ { yymsp[0].minor.yy60 = OP_TYPE_LIKE; } break; - case 296: /* compare_op ::= NOT LIKE */ + case 297: /* compare_op ::= NOT LIKE */ { yymsp[-1].minor.yy60 = OP_TYPE_NOT_LIKE; } break; - case 297: /* compare_op ::= MATCH */ + case 298: /* compare_op ::= MATCH */ { yymsp[0].minor.yy60 = OP_TYPE_MATCH; } break; - case 298: /* compare_op ::= NMATCH */ + case 299: /* compare_op ::= NMATCH */ { yymsp[0].minor.yy60 = OP_TYPE_NMATCH; } break; - case 299: /* in_op ::= IN */ + case 300: /* in_op ::= IN */ { yymsp[0].minor.yy60 = OP_TYPE_IN; } break; - case 300: /* in_op ::= NOT IN */ + case 301: /* in_op ::= NOT IN */ { yymsp[-1].minor.yy60 = OP_TYPE_NOT_IN; } break; - case 301: /* in_predicate_value ::= NK_LP expression_list NK_RP */ + case 302: /* in_predicate_value ::= NK_LP expression_list NK_RP */ { yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy312)); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 303: /* boolean_value_expression ::= NOT boolean_primary */ + case 304: /* boolean_value_expression ::= NOT boolean_primary */ { SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), NULL)); } yymsp[-1].minor.yy104 = yylhsminor.yy104; break; - case 304: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 305: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); @@ -3492,7 +3494,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 305: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 306: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); @@ -3500,52 +3502,52 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 310: /* from_clause ::= FROM table_reference_list */ - case 340: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==340); - case 363: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==363); + case 311: /* from_clause ::= FROM table_reference_list */ + case 341: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==341); + case 364: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==364); { yymsp[-1].minor.yy104 = yymsp[0].minor.yy104; } break; - case 312: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ + case 313: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ { yylhsminor.yy104 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy104, yymsp[0].minor.yy104, NULL); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 315: /* table_primary ::= table_name alias_opt */ + case 316: /* table_primary ::= table_name alias_opt */ { yylhsminor.yy104 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } yymsp[-1].minor.yy104 = yylhsminor.yy104; break; - case 316: /* table_primary ::= db_name NK_DOT table_name alias_opt */ + case 317: /* table_primary ::= db_name NK_DOT table_name alias_opt */ { yylhsminor.yy104 = createRealTableNode(pCxt, &yymsp[-3].minor.yy129, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } yymsp[-3].minor.yy104 = yylhsminor.yy104; break; - case 317: /* table_primary ::= subquery alias_opt */ + case 318: /* table_primary ::= subquery alias_opt */ { yylhsminor.yy104 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104), &yymsp[0].minor.yy129); } yymsp[-1].minor.yy104 = yylhsminor.yy104; break; - case 319: /* alias_opt ::= */ + case 320: /* alias_opt ::= */ { yymsp[1].minor.yy129 = nil_token; } break; - case 320: /* alias_opt ::= table_alias */ + case 321: /* alias_opt ::= table_alias */ { yylhsminor.yy129 = yymsp[0].minor.yy129; } yymsp[0].minor.yy129 = yylhsminor.yy129; break; - case 321: /* alias_opt ::= AS table_alias */ + case 322: /* alias_opt ::= AS table_alias */ { yymsp[-1].minor.yy129 = yymsp[0].minor.yy129; } break; - case 322: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 323: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==323); + case 323: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 324: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==324); { yymsp[-2].minor.yy104 = yymsp[-1].minor.yy104; } break; - case 324: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + case 325: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ { yylhsminor.yy104 = createJoinTableNode(pCxt, yymsp[-4].minor.yy532, yymsp[-5].minor.yy104, yymsp[-2].minor.yy104, yymsp[0].minor.yy104); } yymsp[-5].minor.yy104 = yylhsminor.yy104; break; - case 325: /* join_type ::= */ + case 326: /* join_type ::= */ { yymsp[1].minor.yy532 = JOIN_TYPE_INNER; } break; - case 326: /* join_type ::= INNER */ + case 327: /* join_type ::= INNER */ { yymsp[0].minor.yy532 = JOIN_TYPE_INNER; } break; - case 327: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 328: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { yymsp[-8].minor.yy104 = createSelectStmt(pCxt, yymsp[-7].minor.yy185, yymsp[-6].minor.yy312, yymsp[-5].minor.yy104); yymsp[-8].minor.yy104 = addWhereClause(pCxt, yymsp[-8].minor.yy104, yymsp[-4].minor.yy104); @@ -3555,81 +3557,81 @@ static YYACTIONTYPE yy_reduce( yymsp[-8].minor.yy104 = addHavingClause(pCxt, yymsp[-8].minor.yy104, yymsp[0].minor.yy104); } break; - case 330: /* set_quantifier_opt ::= ALL */ + case 331: /* set_quantifier_opt ::= ALL */ { yymsp[0].minor.yy185 = false; } break; - case 331: /* select_list ::= NK_STAR */ + case 332: /* select_list ::= NK_STAR */ { yymsp[0].minor.yy312 = NULL; } break; - case 335: /* select_item ::= common_expression */ + case 336: /* select_item ::= common_expression */ { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); yylhsminor.yy104 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), &t); } yymsp[0].minor.yy104 = yylhsminor.yy104; break; - case 336: /* select_item ::= common_expression column_alias */ + case 337: /* select_item ::= common_expression column_alias */ { yylhsminor.yy104 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104), &yymsp[0].minor.yy129); } yymsp[-1].minor.yy104 = yylhsminor.yy104; break; - case 337: /* select_item ::= common_expression AS column_alias */ + case 338: /* select_item ::= common_expression AS column_alias */ { yylhsminor.yy104 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), &yymsp[0].minor.yy129); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 338: /* select_item ::= table_name NK_DOT NK_STAR */ + case 339: /* select_item ::= table_name NK_DOT NK_STAR */ { yylhsminor.yy104 = createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 342: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 359: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==359); - case 369: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==369); + case 343: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 360: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==360); + case 370: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==370); { yymsp[-2].minor.yy312 = yymsp[0].minor.yy312; } break; - case 344: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + case 345: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ { yymsp[-5].minor.yy104 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } break; - case 345: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + case 346: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ { yymsp[-3].minor.yy104 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } break; - case 346: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + case 347: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-5].minor.yy104 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), NULL, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } break; - case 347: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + case 348: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-7].minor.yy104 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy104), releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } break; - case 349: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + case 350: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ { yymsp[-3].minor.yy104 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy104); } break; - case 351: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ + case 352: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ { yymsp[-3].minor.yy104 = createFillNode(pCxt, yymsp[-1].minor.yy550, NULL); } break; - case 352: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + case 353: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ { yymsp[-5].minor.yy104 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy312)); } break; - case 353: /* fill_mode ::= NONE */ + case 354: /* fill_mode ::= NONE */ { yymsp[0].minor.yy550 = FILL_MODE_NONE; } break; - case 354: /* fill_mode ::= PREV */ + case 355: /* fill_mode ::= PREV */ { yymsp[0].minor.yy550 = FILL_MODE_PREV; } break; - case 355: /* fill_mode ::= NULL */ + case 356: /* fill_mode ::= NULL */ { yymsp[0].minor.yy550 = FILL_MODE_NULL; } break; - case 356: /* fill_mode ::= LINEAR */ + case 357: /* fill_mode ::= LINEAR */ { yymsp[0].minor.yy550 = FILL_MODE_LINEAR; } break; - case 357: /* fill_mode ::= NEXT */ + case 358: /* fill_mode ::= NEXT */ { yymsp[0].minor.yy550 = FILL_MODE_NEXT; } break; - case 360: /* group_by_list ::= expression */ + case 361: /* group_by_list ::= expression */ { yylhsminor.yy312 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); } yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 361: /* group_by_list ::= group_by_list NK_COMMA expression */ + case 362: /* group_by_list ::= group_by_list NK_COMMA expression */ { yylhsminor.yy312 = addNodeToList(pCxt, yymsp[-2].minor.yy312, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); } yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 364: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 365: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { yylhsminor.yy104 = addOrderByClause(pCxt, yymsp[-3].minor.yy104, yymsp[-2].minor.yy312); yylhsminor.yy104 = addSlimitClause(pCxt, yylhsminor.yy104, yymsp[-1].minor.yy104); @@ -3637,46 +3639,46 @@ static YYACTIONTYPE yy_reduce( } yymsp[-3].minor.yy104 = yylhsminor.yy104; break; - case 366: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + case 367: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { yylhsminor.yy104 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy104, yymsp[0].minor.yy104); } yymsp[-3].minor.yy104 = yylhsminor.yy104; break; - case 371: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 375: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==375); + case 372: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 376: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==376); { yymsp[-1].minor.yy104 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 372: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 376: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==376); + case 373: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 377: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==377); { yymsp[-3].minor.yy104 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 373: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 377: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==377); + case 374: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 378: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==378); { yymsp[-3].minor.yy104 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 378: /* subquery ::= NK_LP query_expression NK_RP */ + case 379: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy104); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 382: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + case 383: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ { yylhsminor.yy104 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), yymsp[-1].minor.yy354, yymsp[0].minor.yy489); } yymsp[-2].minor.yy104 = yylhsminor.yy104; break; - case 383: /* ordering_specification_opt ::= */ + case 384: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy354 = ORDER_ASC; } break; - case 384: /* ordering_specification_opt ::= ASC */ + case 385: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy354 = ORDER_ASC; } break; - case 385: /* ordering_specification_opt ::= DESC */ + case 386: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy354 = ORDER_DESC; } break; - case 386: /* null_ordering_opt ::= */ + case 387: /* null_ordering_opt ::= */ { yymsp[1].minor.yy489 = NULL_ORDER_DEFAULT; } break; - case 387: /* null_ordering_opt ::= NULLS FIRST */ + case 388: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy489 = NULL_ORDER_FIRST; } break; - case 388: /* null_ordering_opt ::= NULLS LAST */ + case 389: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy489 = NULL_ORDER_LAST; } break; default: diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 6dc3b53ae3..72670bb79a 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -54,7 +54,7 @@ private: static const int max_err_len = 1024; bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = doParse(&cxt_, &query_); + int32_t code = parse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == parseCode); @@ -63,7 +63,7 @@ private: return false; } parsedAstStr_ = toString(query_->pRoot); - code = doTranslate(&cxt_, query_); + code = translate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == translateCode); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index 90e2ba3db2..d292fcf8b0 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -62,7 +62,7 @@ protected: void dumpReslut() { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); size_t num = taosArrayGetSize(pStmt->pDataBlocks); - cout << "schemaAttache:" << (int32_t)pStmt->schemaAttache << ", payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; + cout << "payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, i); cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; @@ -81,7 +81,6 @@ protected: void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); - ASSERT_EQ(pStmt->schemaAttache, 0); ASSERT_EQ(pStmt->payloadType, PAYLOAD_TYPE_KV); ASSERT_EQ(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); size_t num = taosArrayGetSize(pStmt->pDataBlocks); @@ -168,6 +167,18 @@ TEST_F(InsertTest, multiTableMultiRowTest) { checkReslut(2, 3, 2); } +// INSERT INTO +// tb1_name USING st1_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +// tb2_name USING st2_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +TEST_F(InsertTest, autoCreateTableTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 using st1 tags(1, 'wxy') values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + TEST_F(InsertTest, toleranceTest) { setDatabase("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f57f476d51..149071bad2 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - nodesWalkList(pExprs, doNameExpr, NULL); + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx return TSDB_CODE_OUT_OF_MEMORY; } - nodesWalkList(pExprs, doCreateColumn, &cxt); + nodesWalkExprs(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); return cxt.errCode; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..04fe0d7da4 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -264,7 +264,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkNode(pRes, doSetSlotId, &cxt); + nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return cxt.errCode; @@ -285,7 +285,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkList(pRes, doSetSlotId, &cxt); + nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return cxt.errCode; @@ -419,7 +419,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + SQueryNodeAddr addr = { .nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet }; taosArrayPush(pCxt->pExecNodeList, &addr); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; @@ -606,7 +606,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; - nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { nodesDestroyList(cxt.pPrecalcExprs); *pPrecalcExprs = NULL; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index d19d88f455..32b221c4fb 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, selectConstant) { + setDatabase("root", "test"); + + bind("SELECT 2-1 FROM t1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, stSimple) { setDatabase("root", "test"); @@ -195,6 +202,12 @@ TEST_F(PlannerTest, interval) { bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)"); ASSERT_TRUE(run()); + + bind("SELECT count(*) FROM t1 interval(10s) fill(linear)"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), sum(c1) FROM t1 interval(10s) fill(value, 10, 20)"); + ASSERT_TRUE(run()); } TEST_F(PlannerTest, sessionWindow) { diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 3e04e7b30a..977fab2e85 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -229,7 +229,7 @@ typedef struct SFltBuildGroupCtx { int32_t code; } SFltBuildGroupCtx; -typedef struct SFilterInfo { +struct SFilterInfo { bool scalarMode; SFltScalarCtx sclCtx; uint32_t options; @@ -254,7 +254,7 @@ typedef struct SFilterInfo { SArray *blkList; SFilterPCtx pctx; -} SFilterInfo; +}; #define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) #define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index cf34fc24a9..58b62bb05e 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -47,7 +47,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); #define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) -#define GET_PARAM_BYTES(_c) ((_c)->pColumnInfoData->info.bytes) +#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) void sclFreeParam(SScalarParam *param); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b0632bbc34..64f99aa916 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { resGroup = taosArrayInit(4, sizeof(SFilterGroup)); SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); @@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (LOGIC_COND_TYPE_OR == node->condType) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); cell = cell->pNext; @@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { filterInitUnitsFields(info); SFltBuildGroupCtx tctx = {.info = info, .group = group}; - nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); @@ -3566,7 +3566,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat); FLT_RET(pStat->code); } @@ -3638,16 +3638,16 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) info = *pInfo; info->options = options; - SFltTreeStat stat = {0}; - FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + SFltTreeStat stat1 = {0}; + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat1)); - info->scalarMode = stat.scalarMode; + info->scalarMode = stat1.scalarMode; if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { info->sclCtx.node = pNode; - FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat1)); } return code; @@ -3664,25 +3664,16 @@ _return: bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; + + SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + output.columnData = createColumnInfoData(&type, pSrc->info.rows); + + *p = (int8_t *)output.columnData->pData; SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); - - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); taosArrayDestroy(pList); - // TODO Fix it -// *p = output.orig.data; -// output.orig.data = NULL; -// -// sclFreeParam(&output); -// -// int8_t *r = output.data; -// for (int32_t i = 0; i < output.num; ++i) { -// if (0 == *(r+i)) { -// return false; -// } -// } - return true; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1cc259d4da..d77ada3b4e 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -29,7 +29,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = blockDataEnsureColumnCapacity(pColumnData, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -44,7 +44,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - blockDataEnsureColumnCapacity(out->columnData, 1); + colInfoDataEnsureCapacity(out->columnData, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); @@ -163,7 +163,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t param->numOfRows = 1; param->columnData = createColumnInfoData(&valueNode->node.resType, 1); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { - colDataAppend(param->columnData, 0, NULL, true); + colDataAppendNULL(param->columnData, 0); } else { colDataAppend(param->columnData, 0, nodesGetValueFromNode(valueNode), false); } @@ -311,12 +311,10 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } -// for (int32_t i = 0; i < rowNum; ++i) { - code = (*ffpSet.process)(params, node->pParameterList->length, output); - if (code) { - sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - SCL_ERR_JRET(code); -// } + code = (*ffpSet.process)(params, node->pParameterList->length, output); + if (code) { + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); } _return: @@ -447,7 +445,6 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { *pNode = (SNode*)res; sclFreeParam(&output); - return DEAL_RES_CONTINUE; } @@ -673,7 +670,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); + nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; @@ -689,14 +686,15 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { int32_t code = 0; SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList}; + // TODO: OPT performance - ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); if (pDst) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 429bc6e66f..c02729a084 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4,7 +4,18 @@ #include "sclInt.h" #include "sclvector.h" +typedef float (*_float_fn)(float); +typedef double (*_double_fn)(double); +typedef double (*_double_fn_2)(double, double); +typedef int (*_conv_fn)(int); +typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); +typedef int16_t (*_len_fn)(char *, int32_t); + /** Math functions **/ +double tlog(double v, double base) { + return log(v) / log(base); +} + int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -102,14 +113,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu return TSDB_CODE_SUCCESS; } -typedef float (*_float_fn)(float); -typedef double (*_double_fn)(double); -typedef double (*_double_fn_2)(double, double); - -double tlog(double v, double base) { - return log(v) / log(base); -} - int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { @@ -211,6 +214,407 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p return TSDB_CODE_SUCCESS; } +/** String functions **/ +int16_t tlength(char *input, int32_t type) { + return varDataLen(input); +} + +int16_t tcharlength(char *input, int32_t type) { + if (type == TSDB_DATA_TYPE_VARCHAR) { + return varDataLen(input); + } else { //NCHAR + return varDataLen(input) / TSDB_NCHAR_SIZE; + } +} + +void tltrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = 0; i < charLen; ++i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = 0; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); + } + + varDataSetLen(output, resLen); +} + +void trtrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = charLen - 1; i >= 0; --i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = charLen - 1; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + } + memcpy(varDataVal(output), varDataVal(input), resLen); + + varDataSetLen(output, resLen); +} + +int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char *in = pInputData->pData; + int16_t *out = (int16_t *)pOutputData->pData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + out[i] = lenFn(in, type); + in += varDataTLen(in); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) { + pOutputData->pData = taosMemoryCalloc(len, sizeof(char)); + pOutputData->info.type = type; + pOutputData->info.bytes = len; + pOutputData->varmeta.length = len; + pOutputData->varmeta.allocLen = len; +} + +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *outputBuf = NULL; + + int32_t inputLen = 0; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } + for (int32_t i = 0; i < inputNum; ++i) { + pInputData[i] = pInput[i].columnData; + input[i] = pInputData[i]->pData; + if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } + } + + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + outputBuf = taosMemoryCalloc(outputLen, 1); + char *output = outputBuf; + + bool hasNull = false; + for (int32_t k = 0; k < numOfRows; ++k) { + for (int32_t i = 0; i < inputNum; ++i) { + if (colDataIsNull_s(pInputData[i], k)) { + colDataAppendNULL(pOutputData, k); + hasNull = true; + break; + } + } + + if (hasNull) { + continue; + } + + int16_t dataLen = 0; + for (int32_t i = 0; i < inputNum; ++i) { + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } + } + varDataSetLen(output, dataLen); + colDataAppend(pOutputData, k, output, false); + output += varDataTLen(output); + } + + pOutput->numOfRows = numOfRows; + taosMemoryFree(input); + taosMemoryFree(outputBuf); + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *outputBuf = NULL; + + int32_t inputLen = 0; + int32_t numOfRows = 0; + for (int32_t i = 1; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) { + return TSDB_CODE_FAILED; + } + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } + for (int32_t i = 0; i < inputNum; ++i) { + pInputData[i] = pInput[i].columnData; + if (i == 0) { + // calculate required separator space + int32_t factor = (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1; + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor; + } else if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } + input[i] = pInputData[i]->pData; + } + + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + outputBuf = taosMemoryCalloc(outputLen, 1); + char *output = outputBuf; + + for (int32_t k = 0; k < numOfRows; ++k) { + if (colDataIsNull_s(pInputData[0], k)) { + colDataAppendNULL(pOutputData, k); + continue; + } + + int16_t dataLen = 0; + for (int32_t i = 1; i < inputNum; ++i) { + if (colDataIsNull_s(pInputData[i], k)) { + continue; + } + + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } + + if (i < inputNum - 1) { + //insert the separator + char *sep = pInputData[0]->pData; + memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); + dataLen += varDataLen(sep); + } + } + varDataSetLen(output, dataLen); + colDataAppend(pOutputData, k, output, false); + output += varDataTLen(output); + } + + pOutput->numOfRows = numOfRows; + taosMemoryFree(input); + taosMemoryFree(outputBuf); + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char *input = pInputData->pData; + char *output = NULL; + + int32_t outputLen = pInputData->varmeta.length; + char *outputBuf = taosMemoryCalloc(outputLen, 1); + output = outputBuf; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + int32_t len = varDataLen(input); + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t j = 0; j < len; ++j) { + *(varDataVal(output) + j) = convFn(*(varDataVal(input) + j)); + } + } else { //NCHAR + for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) { + *((uint32_t *)varDataVal(output) + j) = convFn(*((uint32_t *)varDataVal(input) + j)); + } + } + varDataSetLen(output, len); + colDataAppend(pOutputData, i, output, false); + input += varDataTLen(input); + output += varDataTLen(output); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(outputBuf); + + return TSDB_CODE_SUCCESS; +} + + +int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char *input = pInputData->pData; + char *output = NULL; + + int32_t outputLen = pInputData->varmeta.length; + char *outputBuf = taosMemoryCalloc(outputLen, 1); + output = outputBuf; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + int32_t len = varDataLen(input); + int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; + trimFn(input, output, type, charLen); + + varDataSetLen(output, len); + colDataAppend(pOutputData, i, output, false); + input += varDataTLen(input); + output += varDataTLen(output); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(outputBuf); + + return TSDB_CODE_SUCCESS; +} + +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 && inputNum!= 3) { + return TSDB_CODE_FAILED; + } + + int32_t subPos = 0; + GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + if (subPos == 0) { //subPos needs to be positive or negative values; + return TSDB_CODE_FAILED; + } + + int32_t subLen = INT16_MAX; + if (inputNum == 3) { + GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + if (subLen < 0) { //subLen cannot be negative + return TSDB_CODE_FAILED; + } + subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char *input = pInputData->pData; + char *output = NULL; + + int32_t outputLen = pInputData->varmeta.length; + char *outputBuf = taosMemoryCalloc(outputLen, 1); + output = outputBuf; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + int32_t len = varDataLen(input); + int32_t startPosBytes; + + if (subPos > 0) { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; + startPosBytes = MIN(startPosBytes, len); + } else { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; + startPosBytes = MAX(startPosBytes, 0); + } + + subLen = MIN(subLen, len - startPosBytes); + if (subLen > 0) { + memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen); + } + + varDataSetLen(output, subLen); + colDataAppend(pOutputData, i , output, false); + input += varDataTLen(input); + output += varDataTLen(output); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(outputBuf); + + return TSDB_CODE_SUCCESS; +} + + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } @@ -259,57 +663,28 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return doScalarFunction(pInput, inputNum, pOutput, roundf, round); } -static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput == 1); -#if 0 - int64_t* out = (int64_t*) pOutput->data; - char* s = pLeft->data; - - for(int32_t i = 0; i < pLeft->num; ++i) { - out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); - } -#endif +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, tolower); } -static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput > 0); -#if 0 - int32_t rowLen = 0; - int32_t num = 1; - for(int32_t i = 0; i < numOfInput; ++i) { - rowLen += pLeft[i].bytes; - - if (pLeft[i].num > 1) { - num = pLeft[i].num; - } - } - - pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); - assert(pOutput->data); - - char* rstart = pOutput->data; - for(int32_t i = 0; i < num; ++i) { - - char* s = rstart; - varDataSetLen(s, 0); - for (int32_t j = 0; j < numOfInput; ++j) { - char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes); - - memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1)); - varDataLen(s) += varDataLen(p1); - } - - rstart += rowLen; - } -#endif +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, toupper); } -static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, tltrim); } -static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, trtrim); +} +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tlength); +} + +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tcharlength); } static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 33be65832c..ef3be80893 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -158,23 +158,23 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { int64_t value = strtoll(buf, NULL, 10); - colDataAppend(pOut->columnData, rowIndex, (char*) &value, false); + colDataAppendInt64(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { uint64_t value = strtoull(buf, NULL, 10); - colDataAppend(pOut->columnData, rowIndex, (char*) &value, false); + colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*) &value); } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { double value = strtod(buf, NULL); - colDataAppend(pOut->columnData, rowIndex, (char*) &value, false); + colDataAppendDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { int64_t value = strtoll(buf, NULL, 10); bool v = (value != 0)? true:false; - colDataAppend(pOut->columnData, rowIndex, (char*) &v, false); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { @@ -198,7 +198,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in pOut->numOfRows = pIn->numOfRows; for (int32_t i = 0; i < pIn->numOfRows; ++i) { if (colDataIsNull(pIn->columnData, pIn->numOfRows, i, NULL)) { - colDataAppend(pOut->columnData, i, NULL, true); + colDataAppendNULL(pOut->columnData, i); continue; } @@ -242,13 +242,13 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_BOOL: { for (int32_t i = 0; i < pIn->numOfRows; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } bool value = 0; GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); - colDataAppend(pOutputCol, i, (char*) &value, false); + colDataAppendInt8(pOutputCol, i, (int8_t*) &value); } break; } @@ -259,13 +259,13 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_TIMESTAMP: { for (int32_t i = 0; i < pIn->numOfRows; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); - colDataAppend(pOutputCol, i, (char *)&value, false); + colDataAppendInt64(pOutputCol, i, &value); } break; } @@ -275,26 +275,26 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_UBIGINT: for (int32_t i = 0; i < pIn->numOfRows; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); - colDataAppend(pOutputCol, i, (char*) &value, false); + colDataAppendInt64(pOutputCol, i, (int64_t*)&value); } break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: for (int32_t i = 0; i < pIn->numOfRows; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } double value = 0; GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); - colDataAppend(pOutputCol, i, (char*) &value, false); + colDataAppendDouble(pOutputCol, i, &value); } break; default: @@ -445,7 +445,7 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0); @@ -527,7 +527,7 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { *output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor; @@ -586,7 +586,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* double *output = (double *)pOutputCol->pData; if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0); @@ -666,7 +666,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } else if (pLeft->numOfRows == 1) { if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { *output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i); @@ -678,7 +678,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } } else if (pRight->numOfRows == 1) { if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0); @@ -714,14 +714,14 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } @@ -730,17 +730,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { if (colDataIsNull_f(pRightCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } @@ -750,17 +750,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); if (isnan(lx) || isinf(lx)) { - colDataAppend(pOutputCol, i, NULL, true); + colDataAppendNULL(pOutputCol, i); continue; } @@ -831,7 +831,7 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh double *output = (double *)pOutputCol->pData; if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0); @@ -888,7 +888,7 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight int64_t *output = (int64_t *)pOutputCol->pData; if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value - // TODO set numOfRows NULL value + colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { @@ -947,56 +947,51 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull(pLeft->columnData, pLeft->numOfRows, i, NULL) /*|| - colDataIsNull(pRight->columnData, pRight->numOfRows, i, NULL)*/) { + if (colDataIsNull_s(pLeft->columnData, i)) { continue; } char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); - colDataAppend(pOut->columnData, i, (char *)&res, false); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } return; } if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { - if (colDataIsNull(pLeft->columnData, pLeft->numOfRows, i, NULL) || - colDataIsNull(pRight->columnData, pRight->numOfRows, i, NULL)) { + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { continue; // TODO set null or ignore } char *pLeftData = colDataGetData(pLeft->columnData, i); char *pRightData = colDataGetData(pRight->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppend(pOut->columnData, i, (char *)&res, false); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pRight->numOfRows == 1) { char *pRightData = colDataGetData(pRight->columnData, 0); ASSERT(pLeft->pHashFilter == NULL); for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull(pLeft->columnData, pLeft->numOfRows, i, NULL) /*|| - colDataIsNull(pRight->columnData, pRight->numOfRows, i, NULL)*/) { + if (colDataIsNull_s(pLeft->columnData, i)) { continue; } char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppend(pOut->columnData, i, (char *)&res, false); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pLeft->numOfRows == 1) { char *pLeftData = colDataGetData(pLeft->columnData, 0); - for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (colDataIsNull(pRight->columnData, pRight->numOfRows, i, NULL) /*|| - colDataIsNull(pRight->columnData, pRight->numOfRows, i, NULL)*/) { + if (colDataIsNull_s(pRight->columnData, i)) { continue; } char *pRightData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppend(pOut->columnData, i, (char *)&res, false); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } } @@ -1077,23 +1072,16 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = 0; - if (colDataIsNull(pLeft->columnData, pLeft->numOfRows, i, NULL)) { - v = 1; - } - colDataAppend(pOut->columnData, i, (char*) &v, false); + int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; + colDataAppendInt8(pOut->columnData, i, &v); } - pOut->numOfRows = pLeft->numOfRows; } void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = 1; - if (colDataIsNull(pLeft->columnData, pLeft->numOfRows, i, NULL)) { - v = 0; - } - colDataAppend(pOut->columnData, i, (char*) &v, false); + int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; + colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 54f82eae2d..1a82c8f955 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -155,7 +155,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - blockDataEnsureColumnCapacity(pColumn, rowNum); + colInfoDataEnsureCapacity(pColumn, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -275,7 +275,6 @@ TEST(timerangeTest, greater_and_lower) { nodesDestroyNode(logicNode); } - TEST(columnTest, smallint_column_greater_double_value) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int16_t leftv[5]= {1, 2, 3, 4, 5}; @@ -386,7 +385,6 @@ TEST(columnTest, int_column_greater_smallint_value) { blockDataDestroy(src); } - TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; @@ -432,8 +430,6 @@ TEST(columnTest, int_column_in_double_list) { blockDataDestroy(src); } - - TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; @@ -497,7 +493,6 @@ TEST(columnTest, binary_column_in_binary_list) { blockDataDestroy(src); } - TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; @@ -546,7 +541,6 @@ TEST(columnTest, binary_column_like_binary) { blockDataDestroy(src); } - TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; @@ -641,8 +635,6 @@ TEST(columnTest, binary_column_is_not_null) { blockDataDestroy(src); } - - TEST(opTest, smallint_column_greater_int_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int16_t leftv[5] = {1, -6, -2, 11, 101}; @@ -680,7 +672,6 @@ TEST(opTest, smallint_column_greater_int_column) { blockDataDestroy(src); } - TEST(opTest, smallint_value_add_int_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int32_t leftv = 1; @@ -719,8 +710,6 @@ TEST(opTest, smallint_value_add_int_column) { blockDataDestroy(src); } - - TEST(opTest, bigint_column_multi_binary_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int64_t leftv[5]= {1, 2, 3, 4, 5}; @@ -845,8 +834,6 @@ TEST(opTest, smallint_column_or_float_column) { blockDataDestroy(src); } - - TEST(opTest, smallint_column_or_double_value) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int16_t leftv[5]= {0, 2, 3, 0, -1}; @@ -885,7 +872,6 @@ TEST(opTest, smallint_column_or_double_value) { blockDataDestroy(src); } - TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; @@ -930,7 +916,6 @@ TEST(opTest, binary_column_is_true) { blockDataDestroy(src); } - TEST(filterModelogicTest, diff_columns_and_or_and) { flttInitLogFile(); @@ -1071,7 +1056,6 @@ TEST(filterModelogicTest, same_column_and_or_and) { blockDataDestroy(src); } - TEST(filterModelogicTest, diff_columns_or_and_or) { SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; SNode *logicNode1 = NULL, *logicNode2 = NULL; @@ -1210,8 +1194,6 @@ TEST(filterModelogicTest, same_column_or_and_or) { blockDataDestroy(src); } - - TEST(scalarModelogicTest, diff_columns_or_and_or) { flttInitLogFile(); @@ -1283,8 +1265,6 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { blockDataDestroy(src); } - - int main(int argc, char** argv) { taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 153222516c..61ef2fdce2 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -99,7 +99,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s SColumnInfoData idata = {0}; idata.info = *colInfo; - blockDataEnsureColumnCapacity(&idata, rows); + colInfoDataEnsureCapacity(&idata, rows); taosArrayPush(res->pDataBlock, &idata); @@ -186,7 +186,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - blockDataEnsureColumnCapacity(pColumn, rowNum); + colInfoDataEnsureCapacity(pColumn, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -1467,7 +1467,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - blockDataEnsureColumnCapacity(input->columnData, num); + colInfoDataEnsureCapacity(input->columnData, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 1b73f96896..a7855539a4 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t nleft = count; int64_t nwritten = 0; @@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -429,7 +429,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); @@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif @@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif @@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return ftruncate(pFile->fd, l_size); #endif @@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { if (pFile == NULL) { return NULL; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); return ptr; diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index 6b52fa30be..1cdd41ad78 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -23,9 +23,9 @@ int32_t taosNewProc(char **args) { int32_t pid = fork(); if (pid == 0) { args[0] = tsProcPath; - close(STDIN_FILENO); + // close(STDIN_FILENO); close(STDOUT_FILENO); - close(STDERR_FILENO); + // close(STDERR_FILENO); return execvp(tsProcPath, args); } else { return pid; @@ -33,7 +33,7 @@ int32_t taosNewProc(char **args) { } void taosWaitProc(int32_t pid) { - int32_t status = 0; + int32_t status = -1; waitpid(pid, &status, 0); } diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index ba184c1f5d..bf784f14ac 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -17,10 +17,10 @@ #define _DEFAULT_SOURCE #include "os.h" -int32_t taosCreateShm(SShm* pShm, int32_t shmsize) { +int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { pShm->id = -1; - int32_t shmid = shmget(0X95279527, shmsize, IPC_CREAT | 0600); + int32_t shmid = shmget(0X95270000 + key, shmsize, IPC_CREAT | 0600); if (shmid < 0) { return -1; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index d4e6cb3318..1d7fa517e5 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -59,7 +59,7 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { struct sigaction act; memset(&act, 0, sizeof(act)); #if 1 - act.sa_flags = SA_SIGINFO; + act.sa_flags = SA_SIGINFO | SA_RESTART; act.sa_sigaction = (FLinuxSignalHandler)sigfp; #else act.sa_handler = sigfp; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f6fec47196..46e2d567ce 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -214,7 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosSsleep(2); + taosSsleep(3); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -348,13 +348,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (pFile == NULL) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - tsLogObj.logHandle->pFile = pFile; taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 1d41bd4a48..600413068c 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -434,7 +434,9 @@ int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen, ftype); } -int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - ProcFuncType ftype) { - return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, ftype); +void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, + ProcFuncType ftype) { + while (taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, ftype) != 0) { + taosMsleep(1); + } } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9cf2e9318b..b756976394 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,6 +5,7 @@ ./test.sh -f tsim/user/basic1.sim # ---- db +./test.sh -f tsim/db/create_all_options.sim ./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim @@ -40,6 +41,8 @@ # ---- tmq ./test.sh -f tsim/tmq/basic.sim ./test.sh -f tsim/tmq/basic1.sim +#./test.sh -f tsim/tmq/oneTopic.sim +#./test.sh -f tsim/tmq/multiTopic.sim # --- stable ./test.sh -f tsim/stable/disk.sim @@ -50,4 +53,10 @@ ./test.sh -f tsim/stable/values.sim ./test.sh -f tsim/stable/vnode3.sim + +# --- for multi process mode +./test.sh -f tsim/user/basic1.sim -m +./test.sh -f tsim/stable/vnode3.sim -m +./test.sh -f tsim/tmq/basic.sim -m + #======================b1-end=============== diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 1bbfccf989..38b6d9aadb 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -5,18 +5,12 @@ set +e echo "Executing deploy.sh" -if [ $# != 4 ]; then - echo "argument list need input : " - echo " -n nodeName" - echo " -i nodePort" - exit 1 -fi - UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` NODE_NAME= NODE= -while getopts "n:i:" arg +MULTIPROCESS=0 +while getopts "n:i:m" arg do case $arg in n) @@ -25,6 +19,9 @@ do i) NODE=$OPTARG ;; + m) + MULTIPROCESS=1 + ;; ?) echo "unkonw argument" ;; @@ -145,5 +142,5 @@ echo "statusInterval 1" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "telemetryReporting 0" >> $TAOS_CFG -echo "multiProcess 0" >> $TAOS_CFG +echo "multiProcess ${MULTIPROCESS}" >> $TAOS_CFG echo " " >> $TAOS_CFG diff --git a/tests/script/test.sh b/tests/script/test.sh index f89b9fb1a2..8b77575e18 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -7,7 +7,6 @@ ################################################## set +e -#set -x FILE_NAME= RELEASE=0 @@ -16,7 +15,8 @@ VALGRIND=0 UNIQUE=0 UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` -while getopts "f:avu" arg +MULTIPROCESS=0 +while getopts "f:avum" arg do case $arg in f) @@ -28,6 +28,9 @@ do u) UNIQUE=1 ;; + m) + MULTIPROCESS=1 + ;; ?) echo "unknow argument" ;; @@ -125,8 +128,13 @@ if [ -n "$FILE_NAME" ]; then echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME else - echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME - $PROGRAM -c $CFG_DIR -f $FILE_NAME + if [[ $MULTIPROCESS -eq 1 ]];then + echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME + $PROGRAM -m -c $CFG_DIR -f $FILE_NAME + else + echo "ExcuteCmd(singleprocess):" $PROGRAM -c $CFG_DIR -f $FILE_NAME + $PROGRAM -c $CFG_DIR -f $FILE_NAME + fi fi else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim new file mode 100644 index 0000000000..7f39474f4d --- /dev/null +++ b/tests/script/tsim/db/create_all_options.sim @@ -0,0 +1,486 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$loop_cnt = 0 +check_dnode_ready_1: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data00 != 1 then + return -1 +endi +if $data01 != localhost:7100 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready_1 +endi +if $data14 != ready then + goto check_dnode_ready_1 +endi +if $data24 != ready then + goto check_dnode_ready_1 +endi + +print ============= create database with all options +#database_option: { +# | BLOCKS value [3~1000, default: 6] +# | CACHE value [default: 16] +# | CACHELAST value [0, 1, 2, 3, default: 0] +# | COMP [0 | 1 | 2, default: 2] +# | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +# | FSYNC value [0 ~ 180000 ms, default: 3000] +# | MAXROWS value [200~10000, default: 4096] +# | MINROWS value [10~1000, default: 100] +# | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +# | PRECISION ['ms' | 'us' | 'ns', default: ms] +# | QUORUM value [1 | 2, default: 1] +# | REPLICA value [1 | 3, default: 1] +# | TTL value [1d ~ , default: 1] +# | WAL value [1 | 2, default: 1] +# | VGROUPS value [default: 2] +# | SINGLE_STABLE [0 | 1, default: ] +# | STREAM_MODE [0 | 1, default: ] +# +#$data0_db : name +#$data1_db : create_time +#$data2_db : vgroups +#$data3_db : ntables +#$data4_db : replica +#$data5_db : quorum +#$data6_db : days +#$data7_db : keep +#$data8_db : cache +#$data9_db : blocks +#$data10_db : minrows +#$data11_db : maxrows +#$data12_db : wal +#$data13_db : fsync +#$data14_db : comp +#$data15_db : cachelast +#$data16_db : precision + +print ====> create database db, with default +sql create database db +sql show databases +print rows: $rows +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $rows != 2 then + return -1 +endi +if $data0_db != db then # name + return -1 +endi +if $data2_db != 2 then # vgroups + return -1 +endi +if $data3_db != 0 then # ntables + return -1 +endi +if $data4_db != 1 then # replica + return -1 +endi +if $data5_db != 1 then # quorum + return -1 +endi +if $data6_db != 14400 then # days + return -1 +endi +if $data7_db != 5256000,5256000,5256000 then # keep + return -1 +endi +if $data8_db != 16 then # cache + return -1 +endi +if $data9_db != 6 then # blocks + return -1 +endi +if $data10_db != 100 then # minrows + return -1 +endi +if $data11_db != 4096 then # maxrows + return -1 +endi +if $data12_db != 1 then # wal + return -1 +endi +if $data13_db != 3000 then # fsync + return -1 +endi +if $data14_db != 2 then # comp + return -1 +endi +if $data15_db != 0 then # cachelast + return -1 +endi +if $data16_db != ms then # precision + return -1 +endi +sql drop database db + +print ====> BLOCKS value [3~1000, default: 6] +sql create database db BLOCKS 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 3 then + return -1 +endi +sql drop database db + +sql create database db BLOCKS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 1000 then + return -1 +endi +sql drop database db +sql_error create database db BLOCKS 2 +sql_error create database db BLOCKS 0 +sql_error create database db BLOCKS -1 + +print ====> CACHE value [default: 16] +sql create database db CACHE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHE 128 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 128 then + return -1 +endi +sql drop database db + +print ====> CACHELAST value [0, 1, 2, 3, default: 0] +sql create database db CACHELAST 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 2 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 3 then + return -1 +endi +sql drop database db +sql_error create database db CACHELAST 4 +sql_error create database db CACHELAST -1 + +print ====> COMP [0 | 1 | 2, default: 2] +sql create database db COMP 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 1 then + return -1 +endi +sql drop database db + +sql create database db COMP 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 0 then + return -1 +endi +sql drop database db +sql_error create database db COMP 3 +sql_error create database db COMP -1 + +#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +#sql create database db DAYS 60m KEEP 60m +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 60,60,60 then +# return -1 +#endi +#sql drop database db +#sql create database db DAYS 60m KEEP 1d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 1440,1440,1440 then +# return -1 +#endi +#sql create database db DAYS 3650d KEEP 365000d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 5256000 then +# return -1 +#endi +#if $data7_db != 525600000,525600000,525600000 then +# return -1 +#endi +#sql drop database db +#sql_error create database db DAYS -59m +#sql_error create database db DAYS 59m +#sql_error create database db DAYS 5256001m +#sql_error create database db DAYS 3651d +#sql_error create database db KEEP -59m +#sql_error create database db KEEP 14399m +#sql_error create database db KEEP 525600001m +#sql_error create database db KEEP 365001d + +print ====> FSYNC value [0 ~ 180000 ms, default: 3000] +sql create database db FSYNC 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 0 then + return -1 +endi +sql drop database db + +sql create database db FSYNC 180000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 180000 then + return -1 +endi +sql drop database db +sql_error create database db FSYNC 180001 +sql_error create database db FSYNC -1 + +print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100] +sql create database db MAXROWS 10000 MINROWS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 1000 then + return -1 +endi +if $data11_db != 10000 then + return -1 +endi +sql drop database db + +sql create database db MAXROWS 200 MINROWS 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 10 then + return -1 +endi +if $data11_db != 200 then + return -1 +endi +sql drop database db +sql_error create database db MAXROWS -1 +sql_error create database db MAXROWS 0 +sql_error create database db MAXROWS 199 +sql_error create database db MAXROWS 10001 +sql_error create database db MINROWS -1 +sql_error create database db MINROWS 0 +sql_error create database db MINROWS 9 +sql_error create database db MINROWS 1001 +sql_error create database db MAXROWS 500 MINROWS 1000 + +print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] +sql create database db PRECISION 'us' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != us then + return -1 +endi +sql drop database db + +sql create database db PRECISION 'ns' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != ns then + return -1 +endi +sql drop database db +sql_error create database db PRECISION 'as' +sql_error create database db PRECISION -1 + +print ====> QUORUM value [1 | 2, default: 1] +#sql create database db QUORUM 2 +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data5_db != 2 then +# return -1 +#endi +#sql drop database db + +sql create database db QUORUM 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data5_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db QUORUM 3 +sql_error create database db QUORUM 0 +sql_error create database db QUORUM -1 + +print ====> REPLICA value [1 | 3, default: 1] +sql create database db REPLICA 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 3 then + return -1 +endi +sql drop database db + +sql create database db REPLICA 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db REPLICA 2 +sql_error create database db REPLICA 0 +sql_error create database db REPLICA -1 +sql_error create database db REPLICA 4 + +print ====> TTL value [1d ~ , default: 1] +sql create database db TTL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db TTL 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 10 then +# return -1 +#endi +sql drop database db +sql_error create database db TTL 0 +sql_error create database db TTL -1 + +print ====> WAL value [1 | 2, default: 1] +sql create database db WAL 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 2 then + return -1 +endi +sql drop database db + +sql create database db WAL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db WAL 3 +sql_error create database db WAL -1 +sql_error create database db WAL 0 + +print ====> VGROUPS value [1~4096, default: 2] +sql create database db VGROUPS 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 1 then + return -1 +endi +sql drop database db + +sql create database db VGROUPS 16 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 16 then + return -1 +endi +sql drop database db +sql_error create database db VGROUPS 4097 +sql_error create database db VGROUPS -1 +sql_error create database db VGROUPS 0 + +print ====> SINGLE_STABLE [0 | 1, default: ] +sql create database db SINGLE_STABLE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db SINGLE_STABLE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db SINGLE_STABLE 2 +sql_error create database db SINGLE_STABLE -1 + +print ====> STREAM_MODE [0 | 1, default: ] +sql create database db STREAM_MODE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db STREAM_MODE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db STREAM_MODE 2 +sql_error create database db STREAM_MODE -1 + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index e26f849b13..f073200a05 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -112,21 +112,21 @@ print $data90 $data91 $data92 $data93 if $rows != 10 then return -1 endi -#if $data00 != 10 then -# return -1 -#endi +if $data00 != 10 then + return -1 +endi if $data01 != 0 then return -1 endi -#if $data10 != 10 then -# return -1 -#endi +if $data10 != 10 then + return -1 +endi if $data11 != 1 then return -1 endi -#if $data90 != 10 then -# return -1 -#endi +if $data90 != 10 then + return -1 +endi if $data91 != 9 then return -1 endi @@ -167,15 +167,6 @@ print $data90 $data91 $data92 $data93 return - - - - - - - - - sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1; if $row != 20 then return -1 diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim new file mode 100644 index 0000000000..635e6a60d7 --- /dev/null +++ b/tests/script/tsim/query/session.sim @@ -0,0 +1,340 @@ +#### session windows + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$vgroups = 4 +$dbNamme = d0 + +print ====> create database d1 precision 'us' +sql create database d1 precision 'us' +sql use d1 +sql create table dev_001 (ts timestamp ,i timestamp ,j int) +sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +sql create table secondts(ts timestamp,t2 timestamp,i int) +sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) + +print ====> create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +#print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +sql use $dbNamme + +print =============== create super table, child table and insert data +sql create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50), tag2 binary(16)) +sql create table if not exists dev_001 using st tags("dev_01", "tag_01") +sql create table if not exists dev_002 using st tags("dev_02", "tag_02") + +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.005', 2) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.011', 3) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:01.011', 4) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:01.611', 5) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:00:02.612', 6) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:01:02.612', 7) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:02:02.612', 8) +sql INSERT INTO dev_001 VALUES('2020-05-13 10:03:02.613', 9) +sql INSERT INTO dev_001 VALUES('2020-05-13 11:00:00.000', 10) +sql INSERT INTO dev_001 VALUES('2020-05-13 12:00:00.000', 11) +sql INSERT INTO dev_001 VALUES('2020-05-13 13:00:00.001', 12) +sql INSERT INTO dev_001 VALUES('2020-05-14 13:00:00.001', 13) +sql INSERT INTO dev_001 VALUES('2020-05-15 14:00:00.000', 14) +sql INSERT INTO dev_001 VALUES('2020-05-20 10:00:00.000', 15) +sql INSERT INTO dev_001 VALUES('2020-05-27 10:00:00.001', 16) + +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.000', 1) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.005', 2) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.009', 3) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.0021', 4) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6) +sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7) + +$loop_test = 0 +loop_test_pos: + +# session(ts,5a) +print ====> select count(*) from dev_001 session(ts,5a) +sql select _wstartts, count(*) from dev_001 session(ts,5a) +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != 15 then + return -1 +endi +if $data01 != 2 then + return -1 +endi + + +print ====> select count(*) from (select * from dev_001) session(ts,5a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,5a) +if $rows != 15 then + return -1 +endi +if $data01 != 2 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1s) +sql select _wstartts, count(*) from dev_001 session(ts,1s) +if $rows != 12 then + return -1 +endi +if $data01 != 5 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1s) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1s) +if $rows != 12 then + return -1 +endi +if $data01 != 5 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1000a) +sql select _wstartts, count(*) from dev_001 session(ts,1000a) +if $rows != 12 then + return -1 +endi +if $data01 != 5 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1000a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1000a) +if $rows != 12 then + return -1 +endi +if $data01 != 5 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1m) +sql select _wstartts, count(*) from dev_001 session(ts,1m) +if $rows != 9 then + return -1 +endi +if $data01 != 8 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1m) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1m) +if $rows != 9 then + return -1 +endi +if $data01 != 8 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1h) +sql select _wstartts, count(*) from dev_001 session(ts,1h) +if $rows != 6 then + return -1 +endi +if $data01 != 11 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1h) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1h) +if $rows != 6 then + return -1 +endi +if $data01 != 11 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1d) +sql select _wstartts, count(*) from dev_001 session(ts,1d) +if $rows != 4 then + return -1 +endi +if $data01 != 13 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1d) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1d) +if $rows != 4 then + return -1 +endi +if $data01 != 13 then + return -1 +endi + +print ====> select count(*) from dev_001 session(ts,1w) +sql select _wstartts, count(*) from dev_001 session(ts,1w) +if $rows != 2 then + return -1 +endi +if $data01 != 15 then + return -1 +endi + +print ====> select count(*) from (select * from dev_001) session(ts,1w) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1w) +if $rows != 2 then + return -1 +endi +if $data01 != 15 then + return -1 +endi + + +print ====> leastsquares not supported yet. +print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi +#if $data09 != 12 then +# return -1 +#endi +# $data0-10 != 3.741657387 +# $data0-11 != 1 +# $data1-11 != 14 + +print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi + +sql_error select * from dev_001 session(ts,1w) +sql_error select count(*) from st session(ts,1w) +sql_error select count(*) from dev_001 group by tagtype session(ts,1w) +sql_error select count(*) from dev_001 session(ts,1n) +sql_error select count(*) from dev_001 session(ts,1y) +sql_error select count(*) from dev_001 session(ts,0s) +sql_error select count(*) from dev_001 session(i,1y) +sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' + +print ====> create database d1 precision 'us' +sql create database d1 precision 'us' +sql use d1 +sql create table dev_001 (ts timestamp ,i timestamp ,j int) +sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +print ====> select count(*) from dev_001 session(ts,1u) +sql select count(*) from dev_001 session(ts,1u) +if $rows != 2 then + return -1 +endi +if $data01 != 3 then + return -1 +endi +sql_error select count(*) from dev_001 session(i,1s) +sql create table secondts(ts timestamp,t2 timestamp,i int) +sql_error select count(*) from secondts session(t2,2s) + + +if $loop_test == 0 then + print =============== stop and restart taosd + system sh/exec.sh -n dnode1 -s stop -x SIGINT + system sh/exec.sh -n dnode1 -s start + + $loop_cnt = 0 + check_dnode_ready_0: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi + sql show dnodes + print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 + if $data00 != 1 then + return -1 + endi + if $data04 != ready then + goto check_dnode_ready_0 + endi + + $loop_test = 1 + goto loop_test_pos +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test-all.sh b/tests/test-all.sh index aa7c4240bc..ff9905e209 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -91,20 +91,23 @@ function runSimCaseOneByOnefq { for ((i=$start;i<=$end;i++)) ; do line=`sed -n "$i"p jenkins/basic.txt` if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` + #case=`echo $line | grep sim$ |awk '{print $NF}'` + case=`echo $line | grep -o ".*\.sim" |awk '{print $NF}'` start_time=`date +%s` date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then - echo -n $case - ./test.sh -f $case > case.log 2>&1 \ + #echo -n $case + echo -n $line + $line > case.log 2>&1 \ && \ ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else - echo -n $case - ./test.sh -f $case > ../../sim/case.log 2>&1 && \ + #echo -n $case + echo -n $line + $line > ../../sim/case.log 2>&1 && \ ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log ) diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index 1e2190e308..a667139de1 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -155,6 +155,7 @@ extern int32_t simScriptSucced; extern int32_t simDebugFlag; extern char simScriptDir[]; extern bool abortExecution; +extern bool useMultiProcess; SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 855705e904..9a0b48197a 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -305,25 +305,24 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } -void simReplaceShToBat(char *dst) { - char *sh = strstr(dst, ".sh"); - if (sh != NULL) { +void simReplaceStr(char *buf, char *src, char *dst) { + char *begin = strstr(buf, src); + if (begin != NULL) { + int32_t srcLen = (int32_t)strlen(src); int32_t dstLen = (int32_t)strlen(dst); - char *end = dst + dstLen; - *(end + 1) = 0; + int32_t interval = (dstLen - srcLen); + int32_t remainLen = (int32_t)strlen(buf); + char *end = buf + remainLen; + *(end + interval) = 0; - for (char *p = end; p >= sh; p--) { - *(p + 1) = *p; + for (char *p = end; p >= begin; p--) { + *(p + interval) = *p; } - sh[0] = '.'; - sh[1] = 'b'; - sh[2] = 'a'; - sh[3] = 't'; - sh[4] = ' '; + memcpy(begin, dst, dstLen); } - simDebug("system cmd is %s", dst); + simInfo("system cmd is %s", buf); } bool simExecuteSystemCmd(SScript *script, char *option) { @@ -334,9 +333,13 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simVisuallizeOption(script, option, buf + strlen(buf)); #else sprintf(buf, "%s%s", simScriptDir, option); - simReplaceShToBat(buf); + simReplaceStr(buf, ".sh", ".bat"); #endif + if (useMultiProcess) { + simReplaceStr(buf, "deploy.sh", "deploy.sh -m"); + } + simLogSql(buf, true); int32_t code = system(buf); int32_t repeatTimes = 0; diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index b57299e752..8898f1b201 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -18,6 +18,7 @@ bool simExecSuccess = false; bool abortExecution = false; +bool useMultiProcess = false; void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); @@ -32,6 +33,8 @@ int32_t main(int32_t argc, char *argv[]) { tstrncpy(configDir, argv[++i], 128); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { strcpy(scriptFile, argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + useMultiProcess = true; } else { printf("usage: %s [options] \n", argv[0]); printf(" [-c config]: config directory, default is: %s\n", configDir);