diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 14428bfc43..31cdb28690 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -30,7 +30,7 @@ typedef uint64_t TDRowVerT; typedef int16_t col_id_t; typedef int8_t col_type_t; typedef int32_t col_bytes_t; -typedef uint16_t schema_ver_t; +typedef int32_t schema_ver_t; typedef int32_t func_id_t; #pragma pack(push, 1) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 89fbc92992..3d86adb573 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -146,7 +146,8 @@ bool fmIsBuiltinFunc(const char* pFunc); bool fmIsAggFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId); -bool fmIsNonstandardSQLFunc(int32_t funcId); +bool fmIsVectorFunc(int32_t funcId); +bool fmIsIndefiniteRowsFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); bool fmIsSelectFunc(int32_t funcId); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 291e08fdbf..b9cb708c9c 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -59,10 +59,10 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) -#define DESTORY_LIST(list) \ - do { \ - nodesDestroyList(list); \ - list = NULL; \ +#define DESTORY_LIST(list) \ + do { \ + nodesDestroyList((list)); \ + (list) = NULL; \ } while (0) typedef enum ENodeType { @@ -96,6 +96,7 @@ typedef enum ENodeType { QUERY_NODE_EXPLAIN_OPTIONS, QUERY_NODE_STREAM_OPTIONS, QUERY_NODE_TOPIC_OPTIONS, + QUERY_NODE_LEFT_VALUE, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 7ca4ca9172..6c4d14ffa1 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -54,6 +54,7 @@ typedef struct SScanLogicNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; + SNode* pTagCond; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -343,6 +344,7 @@ typedef struct SSubplan { SNodeList* pParents; // the data destination subplan, get data from current subplan SPhysiNode* pNode; // physical plan of current subplan SDataSinkNode* pDataSink; // data of the subplan flow into the datasink + SNode* pTagCond; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 298dffcc83..606c0acd5b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -81,6 +81,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool notReserved; int16_t placeholderNo; union { bool b; @@ -93,6 +94,10 @@ typedef struct SValueNode { char unit; } SValueNode; +typedef struct SLeftValueNode { + ENodeType type; +} SLeftValueNode; + typedef struct SOperatorNode { SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; @@ -236,7 +241,7 @@ typedef struct SSelectStmt { bool isTimeOrderQuery; bool hasAggFuncs; bool hasRepeatScanFuncs; - bool hasNonstdSQLFunc; + bool hasIndefiniteRowsFunc; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9868c2cc0d..cda08a96a9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -253,6 +253,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) #define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D5) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/include/util/tdef.h b/include/util/tdef.h index 5cc687d7ab..a7ddb0e0bc 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -132,6 +132,7 @@ typedef enum EOperatorType { OP_TYPE_MOD, // unary arithmetic operator OP_TYPE_MINUS, + OP_TYPE_ASSIGN, // bit operator OP_TYPE_BIT_AND, diff --git a/include/util/tencode.h b/include/util/tencode.h index af38d694e2..cbacd59fa7 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -82,7 +82,7 @@ typedef struct { do { \ SEncoder coder = {0}; \ tEncoderInit(&coder, NULL, 0); \ - if ((E)(&coder, S) == 0) { \ + if ((E)(&coder, S) >= 0) { \ SIZE = coder.pos; \ RET = 0; \ } else { \ diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 6e1a9c5726..51bcd05ea1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1538,7 +1538,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, 0); // TODO: use the latest version + tdSRowInit(&rb, pTSchema->version); // TODO: use the latest version for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b381299d27..1108ea1e9f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3817,7 +3817,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) { + if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) { return NULL; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ae41e5d234..4d0846f6c2 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -924,7 +924,7 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); - printf("%s >>>", tag); + printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; SCellVal sVal = {255, NULL}; diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index d7e6f9c87b..c5c4800e02 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -35,7 +35,7 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); -int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); +int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 8b2799833b..1bb003bab9 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -419,7 +419,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SMqTopicObj topicObj = {0}; memcpy(&topicObj, pTopic, sizeof(SMqTopicObj)); topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1; - if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER; + if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER; mndReleaseTopic(pMnode, pTopic); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 01ff08cef9..0cac7fd86b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -448,13 +448,13 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC } mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); - SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -524,13 +524,13 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) { } mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); - SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 6f42d66625..dca07f6a6d 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -153,6 +153,7 @@ int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicNa return -1; } sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + // commit log or redo log? if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { return -1; } @@ -188,7 +189,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { pOffsetObj->offset = pOffset->offset; SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pOffsetRaw); + mndTransAppendCommitlog(pTrans, pOffsetRaw); if (create) { taosMemoryFree(pOffsetObj); } else { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f6043615ab..61f115e2ba 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -743,9 +743,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) { - goto _OVER; - } + if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7b6383a470..9de6138689 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -279,13 +279,13 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast } mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name); - SSdbRaw *pRedoRaw = mndStreamActionEncode(pStream); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c82472eec0..3713bd501a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -479,7 +479,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu SMqTopicObj topicObj = {0}; memcpy(&topicObj, pTopic, sizeof(SMqTopicObj)); topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum; - if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto REB_FAIL; + if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto REB_FAIL; } } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b9b01a4391..ec3d30ff07 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -386,14 +386,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * } mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); taosMemoryFreeClear(topicObj.physicalPlan); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -473,13 +473,13 @@ CREATE_TOPIC_OVER: } static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) { - SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -627,11 +627,11 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl return numOfRows; } -int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { - SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; +int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 821b04a017..e4e67c6e98 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -761,6 +761,12 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } + if (taosArrayGetSize(pTrans->commitLogs) <= 0) { + terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL; + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + return -1; + } + mDebug("trans:%d, prepare transaction", pTrans->id); if (mndTransSync(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 88e646e765..5f2147a5fe 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -272,13 +272,13 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } mDebug("trans:%d, used to create user:%s", pTrans->id, pCreate->user); - SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -352,13 +352,13 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc } mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user); - SSdbRaw *pRedoRaw = mndUserActionEncode(pNew); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndUserActionEncode(pNew); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -559,13 +559,13 @@ static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { } mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); - SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/test/trans/CMakeLists.txt b/source/dnode/mnode/impl/test/trans/CMakeLists.txt index 22ff85563f..dfde1e3d77 100644 --- a/source/dnode/mnode/impl/test/trans/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/trans/CMakeLists.txt @@ -35,3 +35,4 @@ target_include_directories( # NAME transTest2 # COMMAND transTest2 #) + diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b10bee4732..4141485d28 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -48,7 +48,6 @@ target_sources( # tq "src/tq/tq.c" "src/tq/tqCommit.c" - "src/tq/tqMetaStore.c" "src/tq/tqOffset.c" "src/tq/tqPush.c" "src/tq/tqRead.c" diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 3017523aa9..ad3f8cc869 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -20,9 +20,9 @@ #include "executor.h" #include "os.h" -#include "tcache.h" #include "thash.h" #include "tmsg.h" +#include "tqueue.h" #include "trpc.h" #include "ttimer.h" #include "wal.h" @@ -41,45 +41,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -#define TQ_BUFFER_SIZE 4 - -#define TQ_BUCKET_MASK 0xFF -#define TQ_BUCKET_SIZE 256 - -#define TQ_PAGE_SIZE 4096 -// key + offset + size -#define TQ_IDX_SIZE 24 -// 4096 / 24 -#define TQ_MAX_IDX_ONE_PAGE 170 -// 24 * 170 -#define TQ_IDX_PAGE_BODY_SIZE 4080 -// 4096 - 4080 -#define TQ_IDX_PAGE_HEAD_SIZE 16 - -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 -#define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 - -#define TQ_SVER 0 - -// TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 - -#define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 - -static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } - -static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } - -static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; - -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE - -typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; - typedef struct STqOffsetCfg STqOffsetCfg; typedef struct STqOffsetStore STqOffsetStore; @@ -98,53 +59,6 @@ struct STqReadHandle { STSchema* pSchema; }; -typedef struct { - int16_t ver; - int16_t action; - int32_t checksum; - int64_t ssize; - char content[]; -} STqSerializedHead; - -typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); -typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj); -typedef void (*FTqDelete)(void*); - -typedef struct { - int64_t key; - int64_t offset; - int64_t serializedSize; - void* valueInUse; - void* valueInTxn; -} STqMetaHandle; - -typedef struct STqMetaList { - STqMetaHandle handle; - struct STqMetaList* next; - // struct STqMetaList* inTxnPrev; - // struct STqMetaList* inTxnNext; - struct STqMetaList* unpersistPrev; - struct STqMetaList* unpersistNext; -} STqMetaList; - -typedef struct { - STQ* pTq; - STqMetaList* bucket[TQ_BUCKET_SIZE]; - // a table head - STqMetaList* unpersistHead; - // topics that are not connectted - STqMetaList* unconnectTopic; - - TdFilePtr pFile; - TdFilePtr pIdxFile; - - char* dirPath; - int32_t tqConfigFlag; - FTqSerialize pSerializer; - FTqDeserialize pDeserializer; - FTqDelete pDeleter; -} STqMetaStore; - typedef struct { int64_t consumerId; int32_t epoch; @@ -172,6 +86,9 @@ typedef struct { qTaskInfo_t task[5]; } STqExec; +int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec); +int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec); + struct STQ { char* path; SHashObj* pushMgr; // consumerId -> STqExec* @@ -179,7 +96,7 @@ struct STQ { SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; - // TDB* pTdb; + TDB* pTdb; }; typedef struct { @@ -187,89 +104,12 @@ typedef struct { tmr_h timer; } STqMgmt; -static STqMgmt tqMgmt; - -typedef struct { - int8_t status; - int64_t offset; - qTaskInfo_t task; - STqReadHandle* pReadHandle; -} STqTaskItem; - -// new version -typedef struct { - int64_t firstOffset; - int64_t lastOffset; - STqTaskItem output[TQ_BUFFER_SIZE]; -} STqBuffer; - -typedef struct { - char topicName[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - char* qmsg; - STqBuffer buffer; - SWalReadHandle* pReadhandle; -} STqTopic; - -typedef struct { - int64_t consumerId; - int32_t epoch; - char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray -} STqConsumer; - -typedef struct { - int8_t type; - int8_t nodeType; - int8_t reserved[6]; - int64_t streamId; - qTaskInfo_t task; - // TODO sync function -} STqStreamPusher; - -typedef struct { - int8_t inited; - tmr_h timer; -} STqPushMgmt; - -static STqPushMgmt tqPushMgmt; +static STqMgmt tqMgmt = {0}; // init once int tqInit(); void tqCleanUp(); -// open in each vnode -// required by vnode - -int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); -int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); - -static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } - -// tqMetaStore.h -STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, - FTqDelete pDeleter, int32_t tqConfigFlag); -int32_t tqStoreClose(STqMetaStore*); -// int32_t tqStoreDelete(TqMetaStore*); -// int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(STqMetaStore*); -// clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore*); - -void* tqHandleGet(STqMetaStore*, int64_t key); -// make it unpersist -void* tqHandleTouchGet(STqMetaStore*, int64_t key); -int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); -int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); -// delete committed kv pair -// notice that a delete action still needs to be committed -int32_t tqHandleDel(STqMetaStore*, int64_t key); -int32_t tqHandlePurge(STqMetaStore*, int64_t key); -int32_t tqHandleCommit(STqMetaStore*, int64_t key); -int32_t tqHandleAbort(STqMetaStore*, int64_t key); - // tqOffset STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); void STqOffsetClose(STqOffsetStore*); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 2bcb68c82a..c19190e68a 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -278,12 +278,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { pSW = metaGetTableSchema(pMeta, quid, sver, 0); if (!pSW) return NULL; - tdInitTSchemaBuilder(&sb, 0); + tdInitTSchemaBuilder(&sb, sver); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); } pTSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); taosMemoryFree(pSW->pSchema); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bc9a053a43..bd48ed9b4c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,14 +14,37 @@ */ #include "tq.h" -#include "tqueue.h" int32_t tqInit() { - // + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 2); + if (old != 2) break; + } + + if (old == 0) { + tqMgmt.timer = taosTmrInit(10000, 100, 10000, "TQ"); + if (tqMgmt.timer == NULL) { + atomic_store_8(&tqMgmt.inited, 0); + return -1; + } + atomic_store_8(&tqMgmt.inited, 1); + } return 0; } -void tqCleanUp() {} +void tqCleanUp() { + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 2); + if (old != 2) break; + } + + if (old == 1) { + taosTmrCleanUp(tqMgmt.timer); + atomic_store_8(&tqMgmt.inited, 0); + } +} STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { STQ* pTq = taosMemoryMalloc(sizeof(STQ)); @@ -32,9 +55,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; - /*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/ - /*ASSERT(0);*/ - /*}*/ + if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) { + ASSERT(0); + } pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); @@ -51,11 +74,45 @@ void tqClose(STQ* pTq) { taosHashCleanup(pTq->execs); taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); + tdbClose(pTq->pTdb); taosMemoryFree(pTq); } // TODO } +int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeCStr(pEncoder, pExec->subKey) < 0) return -1; + if (tEncodeI64(pEncoder, pExec->consumerId) < 0) return -1; + if (tEncodeI32(pEncoder, pExec->epoch) < 0) return -1; + if (tEncodeI8(pEncoder, pExec->subType) < 0) return -1; + if (tEncodeI8(pEncoder, pExec->withTbName) < 0) return -1; + if (tEncodeI8(pEncoder, pExec->withSchema) < 0) return -1; + if (tEncodeI8(pEncoder, pExec->withTag) < 0) return -1; + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + if (tEncodeCStr(pEncoder, pExec->qmsg) < 0) return -1; + // TODO encode modified exec + } + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pExec->subKey) < 0) return -1; + if (tDecodeI64(pDecoder, &pExec->consumerId) < 0) return -1; + if (tDecodeI32(pDecoder, &pExec->epoch) < 0) return -1; + if (tDecodeI8(pDecoder, &pExec->subType) < 0) return -1; + if (tDecodeI8(pDecoder, &pExec->withTbName) < 0) return -1; + if (tDecodeI8(pDecoder, &pExec->withSchema) < 0) return -1; + if (tDecodeI8(pDecoder, &pExec->withTag) < 0) return -1; + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + if (tDecodeCStrAlloc(pDecoder, &pExec->qmsg) < 0) return -1; + // TODO decode modified exec + } + tEndDecode(pDecoder); + return 0; +} int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { void* pIter = NULL; while (1) { @@ -214,7 +271,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { - // TODO error handle + // TODO handle sma error } void* data = taosMemoryMalloc(msgLen); if (data == NULL) { @@ -230,134 +287,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) int tqCommit(STQ* pTq) { // do nothing - /*return tqStorePersist(pTq->tqMeta);*/ - return 0; -} - -int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { - return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) + - sizeof(int64_t) * 3; -} - -int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) { - int num = taosArrayGetSize(pConsumer->topics); - int32_t sz = 0; - for (int i = 0; i < num; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - sz += tqGetTopicHandleSize(pTopic); - } - return sz; -} - -static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopic->topicName); - /*tlen += taosEncodeString(buf, pTopic->sql);*/ - /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/ - tlen += taosEncodeString(buf, pTopic->qmsg); - /*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/ - /*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/ - /*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/ - return tlen; -} - -static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) { - buf = taosDecodeStringTo(buf, pTopic->topicName); - /*buf = taosDecodeString(buf, &pTopic->sql);*/ - /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/ - buf = taosDecodeString(buf, &pTopic->qmsg); - /*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/ - /*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/ - /*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/ - return buf; -} - -static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) { - int32_t sz; - - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - tlen += taosEncodeFixedI32(buf, pConsumer->epoch); - tlen += taosEncodeString(buf, pConsumer->cgroup); - sz = taosArrayGetSize(pConsumer->topics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - tlen += tEncodeSTqTopic(buf, pTopic); - } - return tlen; -} - -static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) { - int32_t sz; - - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI32(buf, &pConsumer->epoch); - buf = taosDecodeStringTo(buf, pConsumer->cgroup); - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic)); - if (pConsumer->topics == NULL) return NULL; - for (int32_t i = 0; i < sz; i++) { - STqTopic pTopic; - buf = tDecodeSTqTopic(buf, &pTopic); - taosArrayPush(pConsumer->topics, &pTopic); - } - return buf; -} - -int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) { - int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); - - if (sz > (*ppHead)->ssize) { - void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz); - if (tmpPtr == NULL) { - taosMemoryFree(*ppHead); - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - *ppHead = tmpPtr; - (*ppHead)->ssize = sz; - } - - void* ptr = (*ppHead)->content; - void* abuf = ptr; - tEncodeSTqConsumer(&abuf, pConsumer); - - return 0; -} - -int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { - const void* str = pHead->content; - *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (*ppConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - STqConsumer* pConsumer = *ppConsumer; - int32_t sz = taosArrayGetSize(pConsumer->topics); - for (int32_t i = 0; i < sz; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); - if (pTopic->pReadhandle == NULL) { - ASSERT(false); - } - for (int j = 0; j < TQ_BUFFER_SIZE; j++) { - pTopic->buffer.output[j].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - }; - pTopic->buffer.output[j].pReadHandle = pReadHandle; - pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); - } - } - return 0; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c deleted file mode 100644 index ca09cc1dc1..0000000000 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ /dev/null @@ -1,622 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tq.h" -// #include -// #include -// #include -// #include "osDir.h" - -#define TQ_META_NAME "tq.meta" -#define TQ_IDX_NAME "tq.idx" - -static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); -static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); - -static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { - if (pNode->unpersistNext == NULL) { - pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; - pNode->unpersistPrev = pMeta->unpersistHead; - pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; - pMeta->unpersistHead->unpersistNext = pNode; - } -} - -static inline int64_t tqSeekLastPage(TdFilePtr pFile) { - int offset = taosLSeekFile(pFile, 0, SEEK_END); - int pageNo = offset / TQ_PAGE_SIZE; - int curPageOffset = pageNo * TQ_PAGE_SIZE; - return taosLSeekFile(pFile, curPageOffset, SEEK_SET); -} - -// TODO: the struct is tightly coupled with index entry -typedef struct STqIdxPageHead { - int16_t writeOffset; - int8_t unused[14]; -} STqIdxPageHead; - -typedef struct STqIdxPageBuf { - STqIdxPageHead head; - char buffer[TQ_IDX_PAGE_BODY_SIZE]; -} STqIdxPageBuf; - -static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(pFile); - int nBytes; - if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (nBytes == 0) { - memset(pBuf, 0, TQ_PAGE_SIZE); - pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - } - ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - - return taosLSeekFile(pFile, offset, SEEK_SET); -} - -STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, - FTqDelete deleter, int32_t tqConfigFlag) { - STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore)); - if (pMeta == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return NULL; - } - pMeta->pTq = pTq; - - // concat data file name and index file name - size_t pathLen = strlen(path); - pMeta->dirPath = taosMemoryMalloc(pathLen + 1); - if (pMeta->dirPath == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - taosMemoryFree(pMeta); - return NULL; - } - strcpy(pMeta->dirPath, path); - - char* name = taosMemoryMalloc(pathLen + 10); - - strcpy(name, path); - if (!taosDirExist(name) && taosMkDir(name) != 0) { - terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; - tqError("failed to create dir:%s since %s ", name, terrstr()); - } - strcat(name, "/" TQ_IDX_NAME); - TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ); - if (pIdxFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to open file:%s since %s ", name, terrstr()); - // free memory - taosMemoryFree(name); - return NULL; - } - - pMeta->pIdxFile = pIdxFile; - pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pMeta->unpersistHead == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - taosMemoryFree(name); - return NULL; - } - pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; - - strcpy(name, path); - strcat(name, "/" TQ_META_NAME); - TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to open file:%s since %s", name, terrstr()); - taosMemoryFree(name); - return NULL; - } - taosMemoryFree(name); - - pMeta->pFile = pFile; - - pMeta->pSerializer = serializer; - pMeta->pDeserializer = deserializer; - pMeta->pDeleter = deleter; - pMeta->tqConfigFlag = tqConfigFlag; - - // read idx file and load into memory - STqIdxPageBuf idxBuf; - STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE); - if (serializedObj == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - } - int idxRead; - int allocated = TQ_PAGE_SIZE; - bool readEnd = false; - while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { - if (idxRead == -1) { - // TODO: handle error - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to read tq index file since %s", terrstr()); - } - ASSERT(idxBuf.head.writeOffset == idxRead); - // loop read every entry - for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // TODO: free memory - } - memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - - taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); - if (allocated < pNode->handle.serializedSize) { - void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // TODO: free memory - } - serializedObj = ptr; - allocated = pNode->handle.serializedSize; - } - serializedObj->ssize = pNode->handle.serializedSize; - if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { - // TODO: read error - } - if (serializedObj->action == TQ_ACTION_INUSE) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); - } else { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - } - } else if (serializedObj->action == TQ_ACTION_INTXN) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn); - } else { - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - } - } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); - } else { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - } - STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); - if (ptr->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn); - } else { - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - } - } else { - ASSERT(0); - } - - // put into list - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; - if (pBucketNode == NULL) { - pMeta->bucket[bucketKey] = pNode; - } else if (pBucketNode->handle.key == pNode->handle.key) { - pNode->next = pBucketNode->next; - pMeta->bucket[bucketKey] = pNode; - } else { - while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) { - pBucketNode = pBucketNode->next; - } - if (pBucketNode->next) { - ASSERT(pBucketNode->next->handle.key == pNode->handle.key); - STqMetaList* pNodeFound = pBucketNode->next; - pNode->next = pNodeFound->next; - pBucketNode->next = pNode; - pBucketNode = pNodeFound; - } else { - pNode->next = pMeta->bucket[bucketKey]; - pMeta->bucket[bucketKey] = pNode; - pBucketNode = NULL; - } - } - if (pBucketNode) { - if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pBucketNode->handle.valueInUse); - } - if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pBucketNode->handle.valueInTxn); - } - taosMemoryFree(pBucketNode); - } - } - } - taosMemoryFree(serializedObj); - return pMeta; -} - -int32_t tqStoreClose(STqMetaStore* pMeta) { - // commit data and idx - tqStorePersist(pMeta); - ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - taosCloseFile(&pMeta->pFile); - taosCloseFile(&pMeta->pIdxFile); - // free memory - for (int i = 0; i < TQ_BUCKET_SIZE; i++) { - STqMetaList* pNode = pMeta->bucket[i]; - while (pNode) { - ASSERT(pNode->unpersistNext == NULL); - ASSERT(pNode->unpersistPrev == NULL); - if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - STqMetaList* next = pNode->next; - taosMemoryFree(pNode); - pNode = next; - } - } - taosMemoryFree(pMeta->dirPath); - taosMemoryFree(pMeta->unpersistHead); - taosMemoryFree(pMeta); - return 0; -} - -int32_t tqStoreDelete(STqMetaStore* pMeta) { - taosCloseFile(&pMeta->pFile); - taosCloseFile(&pMeta->pIdxFile); - // free memory - for (int i = 0; i < TQ_BUCKET_SIZE; i++) { - STqMetaList* pNode = pMeta->bucket[i]; - pMeta->bucket[i] = NULL; - while (pNode) { - if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - STqMetaList* next = pNode->next; - taosMemoryFree(pNode); - pNode = next; - } - } - taosMemoryFree(pMeta->unpersistHead); - taosRemoveDir(pMeta->dirPath); - taosMemoryFree(pMeta->dirPath); - taosMemoryFree(pMeta); - return 0; -} - -int32_t tqStorePersist(STqMetaStore* pMeta) { - STqIdxPageBuf idxBuf; - int64_t* bufPtr = (int64_t*)idxBuf.buffer; - STqMetaList* pHead = pMeta->unpersistHead; - STqMetaList* pNode = pHead->unpersistNext; - STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead)); - if (pSHead == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pSHead->ver = TQ_SVER; - pSHead->checksum = 0; - pSHead->ssize = sizeof(STqSerializedHead); - /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - - tqReadLastPage(pMeta->pIdxFile, &idxBuf); - - if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); - memset(&idxBuf, 0, TQ_PAGE_SIZE); - idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - } else { - bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); - } - - while (pHead != pNode) { - int nBytes = 0; - - if (pNode->handle.valueInUse) { - if (pNode->handle.valueInTxn) { - pSHead->action = TQ_ACTION_INUSE_CONT; - } else { - pSHead->action = TQ_ACTION_INUSE; - } - - if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(STqSerializedHead); - } else { - pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); - } - nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); - ASSERT(nBytes == pSHead->ssize); - } - - if (pNode->handle.valueInTxn) { - pSHead->action = TQ_ACTION_INTXN; - if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(STqSerializedHead); - } else { - pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); - } - int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); - ASSERT(nBytesTxn == pSHead->ssize); - nBytes += nBytesTxn; - } - pNode->handle.offset = offset; - offset += nBytes; - - // write idx file - // TODO: endian check and convert - *(bufPtr++) = pNode->handle.key; - *(bufPtr++) = pNode->handle.offset; - *(bufPtr++) = (int64_t)nBytes; - idxBuf.head.writeOffset += TQ_IDX_SIZE; - - if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); - // TODO: handle error with tfile - ASSERT(nBytes == TQ_PAGE_SIZE); - memset(&idxBuf, 0, TQ_PAGE_SIZE); - idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - bufPtr = (int64_t*)&idxBuf.buffer; - } - // remove from unpersist list - pHead->unpersistNext = pNode->unpersistNext; - pHead->unpersistNext->unpersistPrev = pHead; - pNode->unpersistPrev = pNode->unpersistNext = NULL; - pNode = pHead->unpersistNext; - - // remove from bucket - if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) { - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; - if (pBucketHead == pNode) { - pMeta->bucket[bucketKey] = pNode->next; - } else { - STqMetaList* pBucketNode = pBucketHead; - while (pBucketNode->next != NULL && pBucketNode->next != pNode) { - pBucketNode = pBucketNode->next; - } - // impossible for pBucket->next == NULL - ASSERT(pBucketNode->next == pNode); - pBucketNode->next = pNode->next; - } - taosMemoryFree(pNode); - } - } - - // write left bytes - taosMemoryFree(pSHead); - // TODO: write new version in tfile - if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); - // TODO: handle error in tfile - ASSERT(nBytes == idxBuf.head.writeOffset); - } - // TODO: using fsync in tfile - taosFsyncFile(pMeta->pIdxFile); - taosFsyncFile(pMeta->pFile); - return 0; -} - -static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - // change pointer ownership - pNode->handle.valueInUse = value; - return 0; - } else { - pNode = pNode->next; - } - } - STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNewNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pNewNode->handle.key = key; - pNewNode->handle.valueInUse = value; - pNewNode->next = pMeta->bucket[bucketKey]; - // put into unpersist list - pNewNode->unpersistPrev = pMeta->unpersistHead; - pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; - pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; - pMeta->unpersistHead->unpersistNext = pNewNode; - return 0; -} - -void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - return pNode->handle.valueInUse; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - tqLinkUnpersist(pMeta, pNode); - return pNode->handle.valueInUse; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn) { - if (tqDupIntxnReject(pMeta->tqConfigFlag)) { - terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN; - return -1; - } - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - } - pNode->handle.valueInTxn = value; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNewNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pNewNode->handle.key = key; - pNewNode->handle.valueInTxn = value; - pNewNode->next = pMeta->bucket[bucketKey]; - pMeta->bucket[bucketKey] = pNewNode; - tqLinkUnpersist(pMeta, pNewNode); - return 0; -} - -int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } - -int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void* vmem = taosMemoryMalloc(vsize); - if (vmem == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - memcpy(vmem, value, vsize); - return tqHandlePutImpl(pMeta, key, vmem); -} - -static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - return pNode->handle.valueInTxn; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn == NULL) { - terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; - return -1; - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - pNode->handle.valueInUse = pNode->handle.valueInTxn; - pNode->handle.valueInTxn = NULL; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - pNode->handle.valueInTxn = NULL; - tqLinkUnpersist(pMeta, pNode); - return 0; - } - terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; - return -1; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if (pNode->handle.valueInTxn) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - tqLinkUnpersist(pMeta, pNode); - return 0; - } - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -// TODO: clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 47896272b5..ee216cb2ab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1638,9 +1638,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); - // TODO: use the real schemaVersion - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); } #ifdef TD_DEBUG_PRINT_ROW @@ -1657,9 +1655,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (row2) { isRow2DataRow = TD_IS_TP_ROW(row2); if (pSchema2 == NULL) { - // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); - // TODO: use the real schemaVersion - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp deleted file mode 100644 index 168daeb19f..0000000000 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ /dev/null @@ -1,279 +0,0 @@ -#include -#include -#include -#include - -#include "tqMetaStore.h" - -struct Foo { - int32_t a; -}; - -int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { - Foo* foo = (Foo*)pObj; - if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { - *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); - (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); - } - *(int32_t*)(*ppHead)->content = foo->a; - return (*ppHead)->ssize; -} - -const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { - if (*ppObj == NULL) { - *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t)); - } - Foo* pFoo = *(Foo**)ppObj; - pFoo->a = *(int32_t*)pHead->content; - return NULL; -} - -void FooDeleter(void* pObj) { taosMemoryFree(pObj); } - -class TqMetaUpdateAppendTest : public ::testing::Test { - protected: - void SetUp() override { - taosRemoveDir(pathName); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - } - - void TearDown() override { tqStoreClose(pMeta); } - - STqMetaStore* pMeta; - const char* pathName = TD_TMP_DIR_PATH "tq_test"; -}; - -TEST_F(TqMetaUpdateAppendTest, copyPutTest) { - Foo foo; - foo.a = 3; - tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); - - Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleCommit(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo->a, 3); -} - -TEST_F(TqMetaUpdateAppendTest, persistTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 2; - tqHandleMovePut(pMeta, 1, pFoo); - Foo* pBar = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pBar == NULL, true); - tqHandleCommit(pMeta, 1); - pBar = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pBar->a, pFoo->a); - pBar = (Foo*)tqHandleGet(pMeta, 2); - EXPECT_EQ(pBar == NULL, true); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pBar = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pBar != NULL, true); - EXPECT_EQ(pBar->a, 2); - - pBar = (Foo*)tqHandleGet(pMeta, 2); - EXPECT_EQ(pBar == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, abortTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleAbort(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, deleteTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleCommit(pMeta, 1); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pFoo != NULL, true); - EXPECT_EQ(pFoo->a, 3); - - tqHandleDel(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pFoo != NULL, true); - EXPECT_EQ(pFoo->a, 3); - - tqHandleCommit(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, intxnPersist) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - tqHandleCommit(pMeta, 1); - - Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pBar->a = 4; - tqHandleMovePut(pMeta, 1, pBar); - - Foo* pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 3); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 3); - - tqHandleCommit(pMeta, 1); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 4); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 4); -} - -TEST_F(TqMetaUpdateAppendTest, multiplePage) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - for (int i = 0; i < 500; i++) { - tqHandleCommit(pMeta, i); - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - for (int i = 500; i < 1000; i++) { - tqHandleCommit(pMeta, i); - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} - -TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 500; i++) { - tqHandleCommit(pMeta, i); - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 500; i < 1000; i++) { - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 1000; i++) { - tqHandleCommit(pMeta, i); - } - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - for (int i = 500; i < 1000; i++) { - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - tqHandleCommit(pMeta, i); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} - -TEST_F(TqMetaUpdateAppendTest, dupCommit) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 1000; i++) { - int ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, 0); - ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, -1); - } - - for (int i = 0; i < 1000; i++) { - int ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, -1); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d6ca5dc15c..1da5770c6d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -106,7 +106,7 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static void releaseQueryBuf(size_t numOfTables); +static void releaseQueryBuf(size_t numOfTables); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); @@ -154,8 +154,9 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, + SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -182,10 +183,10 @@ static int compareRowData(const void* a, const void* b, const void* userData) { int16_t offset = supporter->dataOffset; return 0; -// char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); -// char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); + // char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); + // char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); -// return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; + // return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } // setup the output buffer for each operator @@ -582,8 +583,9 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, + SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { // keep it temporarily bool hasAgg = pCtx[k].input.colDataAggIsSet; @@ -665,8 +667,8 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, - bool createDummyCol) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } else { @@ -717,7 +719,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, - int32_t scanFlag, bool createDummyCol) { + int32_t scanFlag, bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { @@ -725,7 +727,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pCtx[i].input.numOfRows = pBlock->info.rows; pCtx[i].pSrcBlock = pBlock; - pCtx[i].scanFlag = scanFlag; + pCtx[i].scanFlag = scanFlag; SInputColumnInfoData* pInput = &pCtx[i].input; pInput->uid = pBlock->info.uid; @@ -834,7 +836,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); + int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pBlockList); return code; @@ -852,7 +854,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // _rowts/_c0, not tbname column if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) { // do nothing - } else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) { + } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); pfCtx->fpSet.init(&pCtx[k], pResInfo); @@ -950,14 +952,14 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return false; } -// if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { -// // return QUERY_IS_ASC_QUERY(pQueryAttr); -// } -// -// // denote the order type -// if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { -// // return pCtx->param[0].i == pQueryAttr->order.order; -// } + // if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + // // return QUERY_IS_ASC_QUERY(pQueryAttr); + // } + // + // // denote the order type + // if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + // // return pCtx->param[0].i == pQueryAttr->order.order; + // } // in the reverse table scan, only the following functions need to be executed // if (IS_REVERSE_SCAN(pRuntimeEnv) || @@ -1072,19 +1074,19 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu for (int32_t i = 0; i < numOfOutput; ++i) { if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0) { pValCtx[num++] = &pCtx[i]; - } else if (fmIsAggFunc(pCtx[i].functionId)) { + } else if (fmIsSelectFunc(pCtx[i].functionId)) { p = &pCtx[i]; } -// if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { -// tagLen += pCtx[i].resDataInfo.bytes; -// pTagCtx[num++] = &pCtx[i]; -// } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { -// // tag function may be the group by tag column -// // ts may be the required primary timestamp column -// continue; -// } else { -// // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ -// } + // if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + // tagLen += pCtx[i].resDataInfo.bytes; + // pTagCtx[num++] = &pCtx[i]; + // } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { + // // tag function may be the group by tag column + // // ts may be the required primary timestamp column + // continue; + // } else { + // // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ + // } } if (p != NULL) { @@ -1123,7 +1125,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SFuncExecEnv env = {0}; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; - if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) { + if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) { bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId); if (!isUdaf) { fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); @@ -1882,7 +1884,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { if (pFilterNode == NULL) { return; } @@ -2005,8 +2007,9 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ } } -int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { +int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, + int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; @@ -2055,11 +2058,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } else { // expand the result into multiple rows. E.g., _wstartts, top(k, 20) // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - for(int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); - } + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } } } @@ -2070,14 +2073,16 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } } - qDebug("%s result generated, rows:%d, groupId:%"PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); + qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, + pBlock->info.groupId); blockDataUpdateTsWindow(pBlock, 0); return 0; } -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { - SExprInfo* pExprInfo = pOperator->pExpr; - int32_t numOfExprs = pOperator->numOfExprs; +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf) { + SExprInfo* pExprInfo = pOperator->pExpr; + int32_t numOfExprs = pOperator->numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; @@ -2746,10 +2751,10 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; - idata.info.type = pSchema[i].type; + idata.info.type = pSchema[i].type; idata.info.bytes = pSchema[i].bytes; idata.info.colId = pSchema[i].colId; - idata.hasNull = true; + idata.hasNull = true; taosArrayPush(pBlock->pDataBlock, &idata); if (IS_VAR_DATA_TYPE(idata.info.type)) { @@ -3099,7 +3104,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -3212,7 +3217,7 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3 static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr, int32_t rowIndex) { for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index -// pCtx[j].startRow = rowIndex; + // pCtx[j].startRow = rowIndex; } for (int32_t j = 0; j < numOfExpr; ++j) { @@ -3263,7 +3268,7 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { -// pCtx[i].size = 1; + // pCtx[i].size = 1; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -3489,10 +3494,11 @@ _error: return NULL; } -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag) { +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation int32_t type = pOperator->operatorType; - if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { *order = TSDB_ORDER_ASC; *scanFlag = MAIN_SCAN; return TSDB_CODE_SUCCESS; @@ -3520,7 +3526,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; @@ -3859,7 +3865,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo); + code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, + pProjectInfo->pPseudoColInfo); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -4109,7 +4116,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) { SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i); for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) { - STableKeyInfo* pk = taosArrayGet(pa, j); + STableKeyInfo* pk = taosArrayGet(pa, j); STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; pTQueryInfo->lastKey = pk->lastKey; } @@ -4245,9 +4252,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; @@ -4266,15 +4273,15 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols, pTaskInfo); - pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols); + pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pExpr = pExprInfo; - pOperator->numOfExprs = num; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = num; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, destroyProjectOperatorInfo, NULL, NULL, NULL); @@ -4393,10 +4400,10 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa } pCol->slotId = slotId; - pCol->colId = colId; - pCol->bytes = pType->bytes; - pCol->type = pType->type; - pCol->scale = pType->scale; + pCol->colId = colId; + pCol->bytes = pType->bytes; + pCol->type = pType->type; + pCol->scale = pType->scale; pCol->precision = pType->precision; pCol->dataBlockId = blockId; @@ -4471,10 +4478,10 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) { pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error } else { - res->node.resType = (SDataType) {.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; nodesListAppend(pFuncNode->pParameterList, res); } } @@ -4544,7 +4551,7 @@ static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); -void extractTableSchemaVersion(SReadHandle *pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { +void extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, uid); @@ -4591,7 +4598,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t numOfCols = 0; @@ -4600,8 +4607,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); } else { - doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); } if (pDataReader == NULL && terrno != 0) { @@ -4612,15 +4618,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pDescNode); - SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID); + SArray* pCols = + extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo, - pScanPhyNode->node.pConditions, pOperatorDumy); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, + tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4632,7 +4640,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pDescNode); int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); + SArray* colList = + extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createSysTableScanOperatorInfo( pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); @@ -4654,8 +4663,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); int32_t numOfOutputCols = 0; - SArray* colList = - extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); + SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, + COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); @@ -4737,7 +4746,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; - SArray* pColList = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); + SArray* pColList = + extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { @@ -4769,7 +4779,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - SColumn col = extractColumnFromColumnNode(pColNode); + SColumn col = extractColumnFromColumnNode(pColNode); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; @@ -4837,11 +4847,11 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; return c; } @@ -5238,15 +5248,15 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, - const char* pKey, const char* pDir) { +int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, const char* pKey, + const char* pDir) { pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize); int32_t pageSize = rowSize * 32; int32_t bufSize = pageSize * 4096; createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);; + pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK); + ; return TSDB_CODE_SUCCESS; } - diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 4d45eb91ce..21d2776658 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -28,7 +28,7 @@ extern "C" { #define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0) #define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1) -#define FUNC_MGT_NONSTANDARD_SQL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2) +#define FUNC_MGT_INDEFINITE_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2) #define FUNC_MGT_STRING_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(3) #define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4) #define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3e71888bf9..2cec75c8d3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -14,8 +14,8 @@ */ #include "builtins.h" -#include "querynodes.h" #include "builtinsimpl.h" +#include "querynodes.h" #include "scalar.h" #include "taoserror.h" #include "tdatablock.h" @@ -185,6 +185,19 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + + SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 1); + if (nodeType(pParamNode) != QUERY_NODE_VALUE) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)pParamNode; + if (pValue->datum.i < 0 || pValue->datum.i > 100) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } + + pValue->notReserved = true; + if (3 == paraNum) { SNode* pPara3 = nodesListGetNode(pFunc->pParameterList, 2); if (QUERY_NODE_VALUE != nodeType(pPara3) || !validAperventileAlgo((SValueNode*)pPara3)) { @@ -215,7 +228,7 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - SValueNode* pValue = (SValueNode*) pParamNode; + SValueNode* pValue = (SValueNode*)pParamNode; if (pValue->node.resType.type != TSDB_DATA_TYPE_BIGINT) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -224,6 +237,8 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); } + pValue->notReserved = true; + SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; @@ -336,7 +351,7 @@ static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -361,7 +376,7 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -392,7 +407,7 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -434,7 +449,7 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - uint8_t colType = pCol->resType.type; + uint8_t colType = pCol->resType.type; if (IS_VAR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { @@ -463,7 +478,7 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - uint8_t colType = pCol->resType.type; + uint8_t colType = pCol->resType.type; if (IS_VAR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { @@ -500,8 +515,7 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The parameters of UNIQUE can only be columns"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE can only be columns"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; @@ -823,7 +837,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, .translateFunc = translateTop, .getEnvFunc = getTopBotFuncEnv, .initFunc = functionSetup, @@ -833,7 +847,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, .translateFunc = translateBottom, .getEnvFunc = getTopBotFuncEnv, .initFunc = functionSetup, @@ -915,7 +929,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -925,7 +939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "state_count", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -935,7 +949,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "state_duration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -945,7 +959,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -955,7 +969,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -965,7 +979,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -975,7 +989,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -985,7 +999,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 80e777196e..49b20ebc85 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -145,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId); } + bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } @@ -157,7 +159,7 @@ bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc( bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); } -bool fmIsNonstandardSQLFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); } +bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); } bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index fa80858253..83dcb6d7f0 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -138,6 +138,7 @@ static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); static int32_t udfdUvInit(); static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); +static void udfdConnectMnodeThreadFunc(void* args); void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); @@ -870,6 +871,23 @@ static int32_t udfdRun() { return 0; } +void udfdConnectMnodeThreadFunc(void* args) { + int32_t retryMnodeTimes = 0; + int32_t code = 0; + while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { + uv_sleep(100 * (1 << retryMnodeTimes)); + code = udfdConnectToMnode(); + if (code == 0) { + break; + } + fnError("udfd can not connect to mnode, code: %s. retry", tstrerror(code)); + } + + if (code != 0) { + fnError("udfd can not connect to mnode"); + } +} + int main(int argc, char *argv[]) { if (!taosCheckSystemIsSmallEnd()) { printf("failed to start since on non-small-end machines\n"); @@ -902,30 +920,18 @@ int main(int argc, char *argv[]) { return -3; } - int32_t retryMnodeTimes = 0; - int32_t code = 0; - while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { - uv_sleep(100 * (1 << retryMnodeTimes)); - code = udfdConnectToMnode(); - if (code == 0) { - break; - } - fnError("can not connect to mnode, code: %s. retry", tstrerror(code)); - } - - if (code != 0) { - fnError("failed to start since can not connect to mnode"); - return -4; - } - if (udfdUvInit() != 0) { fnError("uv init failure"); return -5; } + uv_thread_t mnodeConnectThread; + uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); + udfdRun(); removeListeningPipe(); + uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); return 0; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8019200e76..5774dcaa1d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -19,6 +19,21 @@ #include "taos.h" #include "taoserror.h" +#define COPY_SCALAR_FIELD(fldname) \ + do { \ + (pDst)->fldname = (pSrc)->fldname; \ + } while (0) + +#define COPY_CHAR_ARRAY_FIELD(fldname) \ + do { \ + strcpy((pDst)->fldname, (pSrc)->fldname); \ + } while (0) + +#define COPY_OBJECT_FIELD(fldname, size) \ + do { \ + memcpy(&((pDst)->fldname), &((pSrc)->fldname), size); \ + } while (0) + #define COPY_CHAR_POINT_FIELD(fldname) \ do { \ if (NULL == (pSrc)->fldname) { \ @@ -70,27 +85,61 @@ } \ } while (0) -static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {} - static SNode* exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { - dataTypeCopy(&pSrc->resType, &pDst->resType); - pDst->pAssociation = NULL; + COPY_OBJECT_FIELD(resType, sizeof(SDataType)); + COPY_CHAR_ARRAY_FIELD(aliasName); + COPY_CHAR_ARRAY_FIELD(userAlias); return (SNode*)pDst; } static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); - pDst->pProjectRef = NULL; + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(tableType); + COPY_SCALAR_FIELD(colId); + COPY_SCALAR_FIELD(colType); + COPY_CHAR_ARRAY_FIELD(dbName); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_CHAR_ARRAY_FIELD(tableAlias); + COPY_CHAR_ARRAY_FIELD(colName); + COPY_SCALAR_FIELD(dataBlockId); + COPY_SCALAR_FIELD(slotId); return (SNode*)pDst; } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); COPY_CHAR_POINT_FIELD(literal); + COPY_SCALAR_FIELD(isDuration); + COPY_SCALAR_FIELD(translate); + COPY_SCALAR_FIELD(notReserved); + COPY_SCALAR_FIELD(placeholderNo); + COPY_SCALAR_FIELD(typeData); + COPY_SCALAR_FIELD(unit); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + COPY_SCALAR_FIELD(datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + COPY_SCALAR_FIELD(datum.i); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + COPY_SCALAR_FIELD(datum.d); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + COPY_SCALAR_FIELD(datum.u); + break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -104,7 +153,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: - // todo + case TSDB_DATA_TYPE_MEDIUMBLOB: default: break; } @@ -113,6 +162,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_SCALAR_FIELD(opType); CLONE_NODE_FIELD(pLeft); CLONE_NODE_FIELD(pRight); return (SNode*)pDst; @@ -120,18 +170,27 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_SCALAR_FIELD(condType); CLONE_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_CHAR_ARRAY_FIELD(functionName); + COPY_SCALAR_FIELD(funcId); + COPY_SCALAR_FIELD(funcType); CLONE_NODE_LIST_FIELD(pParameterList); + COPY_SCALAR_FIELD(udfBufSize); return (SNode*)pDst; } static SNode* tableNodeCopy(const STableNode* pSrc, STableNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_CHAR_ARRAY_FIELD(dbName); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_CHAR_ARRAY_FIELD(tableAlias); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } @@ -159,6 +218,8 @@ static SNode* realTableNodeCopy(const SRealTableNode* pSrc, SRealTableNode* pDst COPY_BASE_OBJECT_FIELD(table, tableNodeCopy); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); + COPY_CHAR_ARRAY_FIELD(qualDbName); + COPY_SCALAR_FIELD(ratio); return (SNode*)pDst; } @@ -170,6 +231,7 @@ static SNode* tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDst static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) { COPY_BASE_OBJECT_FIELD(table, tableNodeCopy); + COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pLeft); CLONE_NODE_FIELD(pRight); CLONE_NODE_FIELD(pOnCond); @@ -177,21 +239,30 @@ static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst } static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { + COPY_SCALAR_FIELD(dataBlockId); + COPY_SCALAR_FIELD(slotId); CLONE_NODE_FIELD(pExpr); return (SNode*)pDst; } static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) { + COPY_SCALAR_FIELD(groupingSetType); CLONE_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) { CLONE_NODE_FIELD(pExpr); + COPY_SCALAR_FIELD(order); + COPY_SCALAR_FIELD(nullOrder); return (SNode*)pDst; } -static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { return (SNode*)pDst; } +static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { + COPY_SCALAR_FIELD(limit); + COPY_SCALAR_FIELD(offset); + return (SNode*)pDst; +} static SNode* stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNode* pDst) { CLONE_NODE_FIELD(pCol); @@ -215,13 +286,16 @@ static SNode* intervalWindowNodeCopy(const SIntervalWindowNode* pSrc, SIntervalW } static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { + COPY_OBJECT_FIELD(dataType, sizeof(SDataType)); CLONE_NODE_LIST_FIELD(pNodeList); return (SNode*)pDst; } static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { + COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); CLONE_NODE_FIELD(pWStartTs); + COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -229,7 +303,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_FIELD(pConditions); CLONE_NODE_LIST_FIELD(pChildren); - pDst->pParent = NULL; + COPY_SCALAR_FIELD(optimizedFlag); return (SNode*)pDst; } @@ -239,12 +313,25 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pScanPseudoCols); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); + COPY_SCALAR_FIELD(scanType); + COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); + COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); + COPY_OBJECT_FIELD(tableName, sizeof(SName)); + COPY_SCALAR_FIELD(showRewrite); + COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); + COPY_SCALAR_FIELD(dataRequired); + COPY_SCALAR_FIELD(interval); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); return (SNode*)pDst; } static SNode* logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pOnConditions); return (SNode*)pDst; } @@ -259,32 +346,50 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pProjections); + COPY_CHAR_ARRAY_FIELD(stmtName); + COPY_SCALAR_FIELD(limit); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(slimit); + COPY_SCALAR_FIELD(soffset); return (SNode*)pDst; } static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); - pDst->pDataBlocks = NULL; - pDst->pVgDataBlocks = NULL; + COPY_SCALAR_FIELD(msgType); return (SNode*)pDst; } static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(srcGroupId); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(winType); CLONE_NODE_LIST_FIELD(pFuncs); + COPY_SCALAR_FIELD(interval); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); + COPY_SCALAR_FIELD(sessionGap); CLONE_NODE_FIELD(pTspk); + CLONE_NODE_FIELD(pStateExpr); + COPY_SCALAR_FIELD(triggerType); + COPY_SCALAR_FIELD(watermark); return (SNode*)pDst; } static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pWStartTs); CLONE_NODE_FIELD(pValues); + COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -301,28 +406,41 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi } static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { + COPY_OBJECT_FIELD(id, sizeof(SSubplanId)); CLONE_NODE_FIELD(pNode); - pDst->pChildren = NULL; - pDst->pParents = NULL; - pDst->pVgroupList = NULL; + COPY_SCALAR_FIELD(subplanType); + COPY_SCALAR_FIELD(level); + COPY_SCALAR_FIELD(splitFlag); return (SNode*)pDst; } static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { + COPY_SCALAR_FIELD(dataBlockId); CLONE_NODE_LIST_FIELD(pSlots); + COPY_SCALAR_FIELD(totalRowSize); + COPY_SCALAR_FIELD(outputRowSize); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { - dataTypeCopy(&pSrc->dataType, &pDst->dataType); + COPY_SCALAR_FIELD(slotId); + COPY_OBJECT_FIELD(dataType, sizeof(SDataType)); + COPY_SCALAR_FIELD(reserve); + COPY_SCALAR_FIELD(output); + COPY_SCALAR_FIELD(tag); return (SNode*)pDst; } static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { + COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr)); + COPY_SCALAR_FIELD(taskId); + COPY_SCALAR_FIELD(schedId); return (SNode*)pDst; } static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { + COPY_SCALAR_FIELD(isDistinct); CLONE_NODE_LIST_FIELD(pProjectionList); CLONE_NODE_FIELD(pFromTable); CLONE_NODE_FIELD(pWhere); @@ -333,6 +451,12 @@ static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { CLONE_NODE_LIST_FIELD(pOrderByList); CLONE_NODE_FIELD(pLimit); CLONE_NODE_FIELD(pLimit); + COPY_CHAR_ARRAY_FIELD(stmtName); + COPY_SCALAR_FIELD(precision); + COPY_SCALAR_FIELD(isEmptyResult); + COPY_SCALAR_FIELD(isTimeOrderQuery); + COPY_SCALAR_FIELD(hasAggFuncs); + COPY_SCALAR_FIELD(hasRepeatScanFuncs); return (SNode*)pDst; } @@ -345,7 +469,6 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memcpy(pDst, pNode, nodesNodeSize(nodeType(pNode))); switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: return columnNodeCopy((const SColumnNode*)pNode, (SColumnNode*)pDst); @@ -387,6 +510,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst); case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst); + case QUERY_NODE_LEFT_VALUE: + return pDst; case QUERY_NODE_SELECT_STMT: return selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst); case QUERY_NODE_LOGIC_PLAN_SCAN: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0e8f530b0e..f28885aad5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -78,6 +78,8 @@ const char* nodesNodeName(ENodeType type) { return "TableOptions"; case QUERY_NODE_INDEX_OPTIONS: return "IndexOptions"; + case QUERY_NODE_LEFT_VALUE: + return "LeftValue"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -490,6 +492,7 @@ static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; +static const char* jkScanLogicPlanTagCond = "TagCond"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -507,6 +510,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); + } return code; } @@ -528,6 +534,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); + } return code; } @@ -1775,6 +1784,7 @@ static const char* jkSubplanDbFName = "DbFName"; static const char* jkSubplanNodeAddr = "NodeAddr"; static const char* jkSubplanRootNode = "RootNode"; static const char* jkSubplanDataSink = "DataSink"; +static const char* jkSubplanTagCond = "TagCond"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -1801,6 +1811,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSubplanDataSink, nodeToJson, pNode->pDataSink); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSubplanTagCond, nodeToJson, pNode->pTagCond); + } return code; } @@ -1831,6 +1844,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkSubplanDataSink, (SNode**)&pNode->pDataSink); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSubplanTagCond, (SNode**)&pNode->pTagCond); + } return code; } @@ -2175,7 +2191,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { code = TSDB_CODE_OUT_OF_MEMORY; break; } - varDataSetLen(pNode->datum.p, pNode->node.resType.bytes); + varDataSetLen(pNode->datum.p, pNode->node.resType.bytes - VARSTR_HEADER_SIZE); if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) { char* buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1); if (NULL == buf) { @@ -3019,6 +3035,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceNodeToJson(pObj, pJson); + case QUERY_NODE_LEFT_VALUE: + return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize. case QUERY_NODE_SET_OPERATOR: return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: @@ -3130,6 +3148,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToSlotDescNode(pJson, pObj); case QUERY_NODE_DOWNSTREAM_SOURCE: return jsonToDownstreamSourceNode(pJson, pObj); + case QUERY_NODE_LEFT_VALUE: + return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize. case QUERY_NODE_SET_OPERATOR: return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 9fb9d8e551..c1aaec1681 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -79,6 +79,8 @@ int32_t nodesNodeSize(ENodeType type) { return sizeof(SStreamOptions); case QUERY_NODE_TOPIC_OPTIONS: return sizeof(STopicOptions); + case QUERY_NODE_LEFT_VALUE: + return sizeof(SLeftValueNode); case QUERY_NODE_SET_OPERATOR: return sizeof(SSetOperator); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 80c4593d9b..f93f0218d4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -342,25 +342,19 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ CHECK_OUT_OF_MEM(cond); cond->condType = type; cond->pParameterList = nodesMakeList(); - if ((QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type != ((SLogicConditionNode*)pParam1)->condType) || - (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type != ((SLogicConditionNode*)pParam2)->condType)) { - nodesListAppend(cond->pParameterList, pParam1); - nodesListAppend(cond->pParameterList, pParam2); + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type == ((SLogicConditionNode*)pParam1)->condType) { + nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList); + ((SLogicConditionNode*)pParam1)->pParameterList = NULL; + nodesDestroyNode(pParam1); } else { - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1)) { - nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList); - ((SLogicConditionNode*)pParam1)->pParameterList = NULL; - nodesDestroyNode(pParam1); - } else { - nodesListAppend(cond->pParameterList, pParam1); - } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2)) { - nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList); - ((SLogicConditionNode*)pParam2)->pParameterList = NULL; - nodesDestroyNode(pParam2); - } else { - nodesListAppend(cond->pParameterList, pParam2); - } + nodesListAppend(cond->pParameterList, pParam1); + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type == ((SLogicConditionNode*)pParam2)->condType) { + nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList); + ((SLogicConditionNode*)pParam2)->pParameterList = NULL; + nodesDestroyNode(pParam2); + } else { + nodesListAppend(cond->pParameterList, pParam2); } return (SNode*)cond; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 11324e3f49..239bd21abc 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -647,7 +647,7 @@ static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, in if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { const char* rowEnd = tdRowEnd(rb->pBuf); STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 99e1135599..b6b1190748 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -292,8 +292,8 @@ static bool isScanPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsScanPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } -static bool isNonstandardSQLFunc(const SNode* pNode) { - return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsNonstandardSQLFunc(((SFunctionNode*)pNode)->funcId)); +static bool isIndefiniteRowsFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsIndefiniteRowsFunc(((SFunctionNode*)pNode)->funcId)); } static bool isDistinctOrderBy(STranslateContext* pCxt) { @@ -646,12 +646,13 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1); + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } - varDataSetLen(pVal->datum.p, targetDt.bytes); - strncpy(varDataVal(pVal->datum.p), pVal->literal, targetDt.bytes); + int32_t len = TMIN(targetDt.bytes - VARSTR_HEADER_SIZE, pVal->node.resType.bytes); + varDataSetLen(pVal->datum.p, len); + strncpy(varDataVal(pVal->datum.p), pVal->literal, len); break; } case TSDB_DATA_TYPE_TIMESTAMP: { @@ -662,19 +663,18 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; } case TSDB_DATA_TYPE_NCHAR: { - int32_t bytes = targetDt.bytes * TSDB_NCHAR_SIZE; - pVal->datum.p = taosMemoryCalloc(1, bytes + VARSTR_HEADER_SIZE + 1); + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); ; } - int32_t output = 0; - if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), bytes, - &output)) { + int32_t len = 0; + if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), + targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } - varDataSetLen(pVal->datum.p, output); + varDataSetLen(pVal->datum.p, len); break; } case TSDB_DATA_TYPE_JSON: @@ -690,8 +690,20 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD return DEAL_RES_CONTINUE; } +static int32_t calcTypeBytes(SDataType dt) { + if (TSDB_DATA_TYPE_BINARY == dt.type) { + return dt.bytes + VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { + return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else { + return dt.bytes; + } +} + static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { - return translateValueImpl(pCxt, pVal, pVal->node.resType); + SDataType dt = pVal->node.resType; + dt.bytes = calcTypeBytes(dt); + return translateValueImpl(pCxt, pVal, dt); } static bool isMultiResFunc(SNode* pNode) { @@ -806,7 +818,7 @@ static EDealRes haveAggOrNonstdFunction(SNode* pNode, void* pContext) { if (isAggFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; - } else if (isNonstandardSQLFunc(pNode)) { + } else if (isIndefiniteRowsFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -851,6 +863,15 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) { return hasInvalidFunc; } +static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { + SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, + .pRpc = pCxt->pParseCxt->pTransporter, + .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet, + .pErrBuf = pCxt->msgBuf.buf, + .errBufLen = pCxt->msgBuf.len}; + return fmGetFuncInfo(¶m, pFunc); +} + static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { SNode* pParam = NULL; FOREACH(pParam, pFunc->pParameterList) { @@ -859,12 +880,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) } } - SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, - .pRpc = pCxt->pParseCxt->pTransporter, - .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet, - .pErrBuf = pCxt->msgBuf.buf, - .errBufLen = pCxt->msgBuf.len}; - pCxt->errCode = fmGetFuncInfo(¶m, pFunc); + pCxt->errCode = getFuncInfo(pCxt, pFunc); if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) { if (beforeHaving(pCxt->currClause)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); @@ -872,7 +888,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (pCxt->pCurrStmt->hasNonstdSQLFunc) { + if (pCxt->pCurrStmt->hasIndefiniteRowsFunc) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -899,14 +915,15 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) } } } - if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsNonstandardSQLFunc(pFunc->funcId)) { - if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrStmt->hasNonstdSQLFunc || pCxt->pCurrStmt->hasAggFuncs) { + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsIndefiniteRowsFunc(pFunc->funcId)) { + if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrStmt->hasIndefiniteRowsFunc || + pCxt->pCurrStmt->hasAggFuncs) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - pCxt->pCurrStmt->hasNonstdSQLFunc = true; + pCxt->pCurrStmt->hasIndefiniteRowsFunc = true; } return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -990,7 +1007,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { - translateFunction(pCxt, pFunc); + pCxt->errCode == getFuncInfo(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; @@ -1060,7 +1077,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList } static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { - if (isAggFunc(*pNode)) { + if (isAggFunc(*pNode) || isIndefiniteRowsFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -1097,7 +1114,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isNonstandardSQLFunc(pNode)) { + if (isIndefiniteRowsFunc(pNode)) { pCxt->existNonstdFunc = true; return DEAL_RES_IGNORE_CHILD; } @@ -1939,7 +1956,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d nodesDestroyNode(pFunc); return TSDB_CODE_OUT_OF_MEMORY; } - if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) { + if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; nodesDestroyNode(pFunc); @@ -2343,16 +2360,6 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm return buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq); } -static int32_t calcTypeBytes(SDataType dt) { - if (TSDB_DATA_TYPE_BINARY == dt.type) { - return dt.bytes + VARSTR_HEADER_SIZE; - } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { - return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else { - return dt.bytes; - } -} - static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; @@ -4082,24 +4089,15 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS } static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) { - if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) { - return pCxt->errCode; + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); } - return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); -} - -static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) { - if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) { - return bytes - VARSTR_HEADER_SIZE; - } else if (TSDB_DATA_TYPE_NCHAR == type) { - return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } - return bytes; + return code; } static SDataType schemaToDataType(SSchema* pSchema) { SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; - dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes); return dt; } diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index f0bf32bf17..ad981073ca 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util nodes catalog cjson parser function qcom scalar + PRIVATE os util nodes catalog cjson parser function qcom scalar index PUBLIC transport ) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 6c567fd4ab..4e77ae5fba 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -418,7 +418,7 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) } static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { - if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { + if (!pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; } @@ -442,8 +442,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + if (TSDB_CODE_SUCCESS == code && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc)) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsVectorFunc, &pAgg->pAggFuncs); } // rewrite the expression in subsequent clauses diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8645225c04..4d489f68e7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -15,6 +15,7 @@ #include "filter.h" #include "functionMgt.h" +#include "index.h" #include "planInt.h" #define OPTIMIZE_FLAG_MASK(n) (1 << n) @@ -313,22 +314,53 @@ static EDealRes cpdIsPrimaryKeyCondImpl(SNode* pNode, void* pContext) { } static bool cpdIsPrimaryKeyCond(SNode* pNode) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + return false; + } bool isPrimaryKeyCond = false; nodesWalkExpr(pNode, cpdIsPrimaryKeyCondImpl, &isPrimaryKeyCond); return isPrimaryKeyCond; } -static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { +static EDealRes cpdIsTagCondImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *((bool*)pContext) = ((COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType) ? true : false); + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool cpdIsTagCond(SNode* pNode) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + return false; + } + bool isTagCond = false; + nodesWalkExpr(pNode, cpdIsTagCondImpl, &isTagCond); + return isTagCond; +} + +static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond, + SNode** pOtherCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pScan->node.pConditions; + if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { + *pPrimaryKeyCond = NULL; + *pOtherCond = pScan->node.pConditions; + pScan->node.pConditions = NULL; + return TSDB_CODE_SUCCESS; + } + int32_t code = TSDB_CODE_SUCCESS; SNodeList* pPrimaryKeyConds = NULL; + SNodeList* pTagConds = NULL; SNodeList* pOtherConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { if (cpdIsPrimaryKeyCond(pCond)) { code = nodesListMakeAppend(&pPrimaryKeyConds, nodesCloneNode(pCond)); + } else if (cpdIsTagCond(pScan->node.pConditions)) { + code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); } else { code = nodesListMakeAppend(&pOtherConds, nodesCloneNode(pCond)); } @@ -338,37 +370,46 @@ static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimary } SNode* pTempPrimaryKeyCond = NULL; + SNode* pTempTagCond = NULL; SNode* pTempOtherCond = NULL; if (TSDB_CODE_SUCCESS == code) { code = cpdMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); } + if (TSDB_CODE_SUCCESS == code) { + code = cpdMergeConds(&pTempTagCond, &pTagConds); + } if (TSDB_CODE_SUCCESS == code) { code = cpdMergeConds(&pTempOtherCond, &pOtherConds); } if (TSDB_CODE_SUCCESS == code) { *pPrimaryKeyCond = pTempPrimaryKeyCond; + *pTagCond = pTempTagCond; *pOtherCond = pTempOtherCond; nodesDestroyNode(pScan->node.pConditions); pScan->node.pConditions = NULL; } else { nodesDestroyList(pPrimaryKeyConds); + nodesDestroyList(pTagConds); nodesDestroyList(pOtherConds); nodesDestroyNode(pTempPrimaryKeyCond); + nodesDestroyNode(pTempTagCond); nodesDestroyNode(pTempOtherCond); } return code; } -static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions) && - LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pScan->node.pConditions)->condType) { - return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pOtherCond); +static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond, + SNode** pOtherCond) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions)) { + return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pTagCond, pOtherCond); } if (cpdIsPrimaryKeyCond(pScan->node.pConditions)) { *pPrimaryKeyCond = pScan->node.pConditions; + } else if (cpdIsTagCond(pScan->node.pConditions)) { + *pTagCond = pScan->node.pConditions; } else { *pOtherCond = pScan->node.pConditions; } @@ -391,6 +432,32 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, return code; } +static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** pOtherCond) { + int32_t code = TSDB_CODE_SUCCESS; + SIdxFltStatus idxStatus = idxGetFltStatus(*pTagCond); + switch (idxStatus) { + case SFLT_NOT_INDEX: + code = cpdCondAppend(pOtherCond, pTagCond); + break; + case SFLT_COARSE_INDEX: + pScan->pTagCond = nodesCloneNode(*pTagCond); + if (NULL == pScan->pTagCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + code = cpdCondAppend(pOtherCond, pTagCond); + break; + case SFLT_ACCURATE_INDEX: + pScan->pTagCond = *pTagCond; + *pTagCond = NULL; + break; + default: + code = TSDB_CODE_FAILED; + break; + } + return code; +} + static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) { @@ -398,11 +465,15 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* } SNode* pPrimaryKeyCond = NULL; + SNode* pTagCond = NULL; SNode* pOtherCond = NULL; - int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pOtherCond); + int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond); } + if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { + code = cpdApplyTagIndex(pScan, &pTagCond, &pOtherCond); + } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; } @@ -618,30 +689,6 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } } -// static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { -// if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { -// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); -// } -// return TSDB_CODE_SUCCESS; -// } - -// static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { -// if (LOGIC_COND_TYPE_AND != pOnCond->condType) { -// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); -// } -// bool hasPrimaryKeyEqualCond = false; -// SNode* pCond = NULL; -// FOREACH(pCond, pOnCond->pParameterList) { -// if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { -// hasPrimaryKeyEqualCond = true; -// } -// } -// if (!hasPrimaryKeyEqualCond) { -// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); -// } -// return TSDB_CODE_SUCCESS; -// } - static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); @@ -650,11 +697,6 @@ static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; - // if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { - // return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); - // } else { - // return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions); - // } } static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index affe9ef2f6..fcba2aa2d3 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -411,7 +411,7 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys return sortScanCols(pScanPhysiNode->pScanCols); } -static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, +static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); if (TSDB_CODE_SUCCESS == code) { @@ -438,6 +438,12 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); + if (NULL != pScanLogicNode->pTagCond) { + pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond); + if (NULL == pSubplan->pTagCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } } if (TSDB_CODE_SUCCESS == code) { @@ -463,7 +469,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla } vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); - return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, @@ -498,7 +504,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->intervalUnit = pScanLogicNode->intervalUnit; pTableScan->slidingUnit = pScanLogicNode->slidingUnit; - return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); } static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, @@ -522,7 +528,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); - return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, @@ -599,14 +605,17 @@ typedef struct SRewritePrecalcExprsCxt { static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { SNode* pExpr = nodesCloneNode(*pNode); if (NULL == pExpr) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; nodesDestroyNode(pExpr); return DEAL_RES_ERROR; } SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; nodesDestroyNode(pExpr); return DEAL_RES_ERROR; } @@ -624,16 +633,45 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) return DEAL_RES_IGNORE_CHILD; } +static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + if (NULL == pOper) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pOper->pLeft = nodesMakeNode(QUERY_NODE_LEFT_VALUE); + if (NULL == pOper->pLeft) { + nodesDestroyNode(pOper); + return TSDB_CODE_OUT_OF_MEMORY; + } + SValueNode* pVal = (SValueNode*)*pNode; + pOper->node.resType = pVal->node.resType; + strcpy(pOper->node.aliasName, pVal->node.aliasName); + pOper->opType = OP_TYPE_ASSIGN; + pOper->pRight = *pNode; + *pNode = (SNode*)pOper; + return TSDB_CODE_SUCCESS; +} + static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; switch (nodeType(*pNode)) { + case QUERY_NODE_VALUE: { + if (((SValueNode*)*pNode)->notReserved) { + break; + } + pCxt->errCode = rewriteValueToOperator(pCxt, pNode); + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + return DEAL_RES_ERROR; + } + return collectAndRewrite(pCxt, pNode); + } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: { - return collectAndRewrite(pContext, pNode); + return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_FUNCTION: { if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) { - return collectAndRewrite(pContext, pNode); + return collectAndRewrite(pCxt, pNode); } } default: @@ -677,9 +715,8 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } SRewritePrecalcExprsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs}; nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); - if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { - nodesDestroyList(cxt.pPrecalcExprs); - *pPrecalcExprs = NULL; + if (0 == LIST_LENGTH(cxt.pPrecalcExprs) || TSDB_CODE_SUCCESS != cxt.errCode) { + DESTORY_LIST(*pPrecalcExprs); } return cxt.errCode; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 8e6c04bb33..af62c52a89 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,6 +18,13 @@ #include "planInt.h" #include "scalar.h" +static void dumpQueryPlan(SQueryPlan* pPlan) { + char* pStr = NULL; + nodesNodeToString(pPlan, false, &pStr, NULL); + planDebugL("Query Plan: %s", pStr); + taosMemoryFree(pStr); +} + int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { SLogicNode* pLogicNode = NULL; SLogicSubplan* pLogicSubplan = NULL; @@ -36,6 +43,9 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); } + if (TSDB_CODE_SUCCESS == code) { + dumpQueryPlan(*pPlan); + } nodesDestroyNode(pLogicNode); nodesDestroyNode(pLogicSubplan); diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index a17d8cd850..4b84079f7b 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -50,4 +50,6 @@ TEST_F(PlanBasicTest, func) { run("SELECT DIFF(c1) FROM t1"); run("SELECT PERCENTILE(c1, 60) FROM t1"); + + run("SELECT TOP(c1, 60) FROM t1"); } diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 9ca1001f4c..cf51603470 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -49,6 +49,8 @@ TEST_F(PlanGroupByTest, aggFunc) { run("SELECT LAST(*), FIRST(*) FROM t1"); run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); + + run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } TEST_F(PlanGroupByTest, selectFunc) { diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 77f9b5846c..4234a1320a 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -32,6 +32,12 @@ TEST_F(PlanOptimizeTest, optimizeScanData) { run("SELECT PERCENTILE(c1, 40), COUNT(*) FROM t1"); } +TEST_F(PlanOptimizeTest, ConditionPushDown) { + useDb("root", "test"); + + run("SELECT ts, c1 FROM st1 WHERE tag1 > 4"); +} + TEST_F(PlanOptimizeTest, orderByPrimaryKey) { useDb("root", "test"); diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 86bacc9220..0847620888 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -233,45 +233,45 @@ class PlannerTestBaseImpl { if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (res_.prepareAst_.empty()) { - cout << "syntax tree : " << endl; + cout << "+++++++++++++++++++++syntax tree : " << endl; cout << res_.ast_ << endl; } else { - cout << "prepare syntax tree : " << endl; + cout << "+++++++++++++++++++++prepare syntax tree : " << endl; cout << res_.prepareAst_ << endl; - cout << "bound syntax tree : " << endl; + cout << "+++++++++++++++++++++bound syntax tree : " << endl; cout << res_.boundAst_ << endl; - cout << "syntax tree : " << endl; + cout << "+++++++++++++++++++++syntax tree : " << endl; cout << res_.ast_ << endl; } } if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { - cout << "raw logic plan : " << endl; + cout << "+++++++++++++++++++++raw logic plan : " << endl; cout << res_.rawLogicPlan_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) { - cout << "optimized logic plan : " << endl; + cout << "+++++++++++++++++++++optimized logic plan : " << endl; cout << res_.optimizedLogicPlan_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) { - cout << "split logic plan : " << endl; + cout << "+++++++++++++++++++++split logic plan : " << endl; cout << res_.splitLogicPlan_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) { - cout << "scaled logic plan : " << endl; + cout << "+++++++++++++++++++++scaled logic plan : " << endl; cout << res_.scaledLogicPlan_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) { - cout << "physical plan : " << endl; + cout << "+++++++++++++++++++++physical plan : " << endl; cout << res_.physiPlan_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) { - cout << "physical subplan : " << endl; + cout << "+++++++++++++++++++++physical subplan : " << endl; for (const auto& subplan : res_.physiSubplans_) { cout << subplan << endl; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 49ed3ab48b..fb03eaefa4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -182,6 +182,11 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { + case QUERY_NODE_LEFT_VALUE: { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0); + param->numOfRows = pb->info.rows; + break; + } case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; @@ -845,7 +850,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { } EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)|| QUERY_NODE_LEFT_VALUE == nodeType(pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 45742189d5..12496eec55 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -824,7 +824,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } //for constant conversion, need to set proper length of pOutput description if (len < outputLen) { - pOutput->columnData->info.bytes = len; + pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE; } break; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 145ed69a77..0fb3712c30 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1333,6 +1333,22 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO doReleaseVec(pLeftCol, leftConvert); } +void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + pOut->numOfRows = pLeft->numOfRows; + + if (colDataIsNull_s(pRight->columnData, 0)) { + for (int32_t i = 0; i < pOut->numOfRows; ++i) { + colDataAppend(pOutputCol, i, NULL, true); + } + } else { + for (int32_t i = 0; i < pOut->numOfRows; ++i) { + colDataAppend(pOutputCol, i, colDataGetData(pRight->columnData, 0), false); + } + } +} + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { #if 0 int32_t len = pLeft->bytes + pRight->bytes; @@ -1691,6 +1707,8 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMathRemainder; case OP_TYPE_MINUS: return vectorMathMinus; + case OP_TYPE_ASSIGN: + return vectorAssign; case OP_TYPE_GREATER_THAN: return vectorGreater; case OP_TYPE_GREATER_EQUAL: diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index ada1f599f2..71cd6de73f 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -14,17 +14,17 @@ */ #define _DEFAULT_SOURCE -#include "tcompare.h" #include "os.h" #include "taoserror.h" +#include "tcompare.h" #include "tref.h" #include "walInt.h" typedef struct { - int8_t stop; - int8_t inited; - uint32_t seq; - int32_t refSetId; + int8_t stop; + int8_t inited; + uint32_t seq; + int32_t refSetId; TdThread thread; } SWalMgmt; @@ -36,30 +36,42 @@ static void walFreeObj(void *pWal); int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); } int32_t walInit() { - int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); - if (old == 1) return 0; - - tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - - int32_t code = walCreateThread(); - if (code != 0) { - wError("failed to init wal module since %s", tstrerror(code)); - atomic_store_8(&tsWal.inited, 0); - return code; + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 2); + if (old != 2) break; + } + + if (old == 0) { + tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); + + int32_t code = walCreateThread(); + if (code != 0) { + wError("failed to init wal module since %s", tstrerror(code)); + atomic_store_8(&tsWal.inited, 0); + return code; + } + + wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId); + atomic_store_8(&tsWal.inited, 1); } - wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId); return 0; } void walCleanUp() { - int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); - if (old == 0) { - return; + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 2); + if (old != 2) break; + } + + if (old == 1) { + walStopThread(); + taosCloseRef(tsWal.refSetId); + wInfo("wal module is cleaned up"); + atomic_store_8(&tsWal.inited, 0); } - walStopThread(); - taosCloseRef(tsWal.refSetId); - wInfo("wal module is cleaned up"); } SWal *walOpen(const char *path, SWalCfg *pCfg) { @@ -126,7 +138,6 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } if (walCheckAndRepairIdx(pWal) < 0) { - } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3890a55ff1..7c4f0fa2dd 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exist TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists")