xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/sync-mnode-integration

This commit is contained in:
Li Minghao 2022-05-23 22:11:28 +08:00 committed by GitHub
parent 43f2a51e4b a1b97b7a24
commit 43a62936e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 752 additions and 1566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/ttypes.h
View File

@ -30,7 +30,7 @@ typedef uint64_t TDRowVerT;
typedef int16_t col_id_t;
typedef int8_t col_type_t;
typedef int32_t col_bytes_t;
typedef uint16_t schema_ver_t;
typedef int32_t schema_ver_t;
typedef int32_t func_id_t;
#pragma pack(push, 1)

3
include/libs/function/functionMgt.h
View File

@ -146,7 +146,8 @@ bool fmIsBuiltinFunc(const char* pFunc);
bool fmIsAggFunc(int32_t funcId);
bool fmIsScalarFunc(int32_t funcId);
bool fmIsNonstandardSQLFunc(int32_t funcId);
bool fmIsVectorFunc(int32_t funcId);
bool fmIsIndefiniteRowsFunc(int32_t funcId);
bool fmIsStringFunc(int32_t funcId);
bool fmIsDatetimeFunc(int32_t funcId);
bool fmIsSelectFunc(int32_t funcId);

9
include/libs/nodes/nodes.h
View File

@ -59,10 +59,10 @@ extern "C" {
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
(NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
#define DESTORY_LIST(list) \
do { \
nodesDestroyList(list); \
list = NULL; \
#define DESTORY_LIST(list) \
do { \
nodesDestroyList((list)); \
(list) = NULL; \
} while (0)
typedef enum ENodeType {
@ -96,6 +96,7 @@ typedef enum ENodeType {
QUERY_NODE_EXPLAIN_OPTIONS,
QUERY_NODE_STREAM_OPTIONS,
QUERY_NODE_TOPIC_OPTIONS,
QUERY_NODE_LEFT_VALUE,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR,

2
include/libs/nodes/plannodes.h
View File

@ -54,6 +54,7 @@ typedef struct SScanLogicNode {
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SNode* pTagCond;
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -343,6 +344,7 @@ typedef struct SSubplan {
SNodeList* pParents; // the data destination subplan, get data from current subplan
SPhysiNode* pNode; // physical plan of current subplan
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
SNode* pTagCond;
} SSubplan;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;

7
include/libs/nodes/querynodes.h
View File

@ -81,6 +81,7 @@ typedef struct SValueNode {
char* literal;
bool isDuration;
bool translate;
bool notReserved;
int16_t placeholderNo;
union {
bool b;
@ -93,6 +94,10 @@ typedef struct SValueNode {
char unit;
} SValueNode;
typedef struct SLeftValueNode {
ENodeType type;
} SLeftValueNode;
typedef struct SOperatorNode {
SExprNode node; // QUERY_NODE_OPERATOR
EOperatorType opType;
@ -236,7 +241,7 @@ typedef struct SSelectStmt {
bool isTimeOrderQuery;
bool hasAggFuncs;
bool hasRepeatScanFuncs;
bool hasNonstdSQLFunc;
bool hasIndefiniteRowsFunc;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;

1
include/util/taoserror.h
View File

@ -253,6 +253,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2)
#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3)
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4)
#define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D5)
// mnode-mq
#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0)

1
include/util/tdef.h
View File

@ -132,6 +132,7 @@ typedef enum EOperatorType {
OP_TYPE_MOD,
// unary arithmetic operator
OP_TYPE_MINUS,
OP_TYPE_ASSIGN,
// bit operator
OP_TYPE_BIT_AND,

2
include/util/tencode.h
View File

@ -82,7 +82,7 @@ typedef struct {
do { \
SEncoder coder = {0}; \
tEncoderInit(&coder, NULL, 0); \
if ((E)(&coder, S) == 0) { \
if ((E)(&coder, S) >= 0) { \
SIZE = coder.pos; \
RET = 0; \
} else { \

2
source/common/src/tdatablock.c
View File

@ -1538,7 +1538,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
int32_t msgLen = sizeof(SSubmitReq);
int32_t numOfBlks = 0;
SRowBuilder rb = {0};
tdSRowInit(&rb, 0); // TODO: use the latest version
tdSRowInit(&rb, pTSchema->version); // TODO: use the latest version
for (int32_t i = 0; i < sz; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);

2
source/common/src/tmsg.c
View File

@ -3817,7 +3817,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) {
return NULL;
}

2
source/common/src/trow.c
View File

@ -924,7 +924,7 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) {
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pSchema);
tdSTSRowIterReset(&iter, row);
printf("%s >>>", tag);
printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row));
for (int i = 0; i < pSchema->numOfCols; ++i) {
STColumn *stCol = pSchema->columns + i;
SCellVal sVal = {255, NULL};

2
source/dnode/mnode/impl/inc/mndTopic.h
View File

@ -35,7 +35,7 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
#ifdef __cplusplus
}

2
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -419,7 +419,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SMqTopicObj topicObj = {0};
memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
mndReleaseTopic(pMnode, pTopic);
}

16
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -448,13 +448,13 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
}
mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -524,13 +524,13 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) {
}
mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());

3
source/dnode/mnode/impl/src/mndOffset.c
View File

@ -153,6 +153,7 @@ int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicNa
return -1;
}
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
// commit log or redo log?
if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
return -1;
}
@ -188,7 +189,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
pOffsetObj->offset = pOffset->offset;
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pOffsetRaw);
mndTransAppendCommitlog(pTrans, pOffsetRaw);
if (create) {
taosMemoryFree(pOffsetObj);
} else {

4
source/dnode/mnode/impl/src/mndStb.c
View File

@ -743,9 +743,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) {
goto _OVER;
}
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;

8
source/dnode/mnode/impl/src/mndStream.c
View File

@ -279,13 +279,13 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
}
mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name);
SSdbRaw *pRedoRaw = mndStreamActionEncode(pStream);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
return 0;
}

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -479,7 +479,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
SMqTopicObj topicObj = {0};
memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum;
if (mndSetTopicRedoLogs(pMnode, pTrans, &topicObj) != 0) goto REB_FAIL;
if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto REB_FAIL;
}
}

26
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -386,14 +386,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
}
mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name);
SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
taosMemoryFreeClear(topicObj.physicalPlan);
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -473,13 +473,13 @@ CREATE_TOPIC_OVER:
}
static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -627,11 +627,11 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
return numOfRows;
}
int32_t mndSetTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}

6
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -761,6 +761,12 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
return -1;
}
if (taosArrayGetSize(pTrans->commitLogs) <= 0) {
terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL;
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
return -1;
}
mDebug("trans:%d, prepare transaction", pTrans->id);
if (mndTransSync(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());

24
source/dnode/mnode/impl/src/mndUser.c
View File

@ -272,13 +272,13 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
}
mDebug("trans:%d, used to create user:%s", pTrans->id, pCreate->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -352,13 +352,13 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc
}
mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pNew);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndUserActionEncode(pNew);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -559,13 +559,13 @@ static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) {
}
mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pUser);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
SSdbRaw *pCommitRaw = mndUserActionEncode(pUser);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());

1
source/dnode/mnode/impl/test/trans/CMakeLists.txt
View File

@ -35,3 +35,4 @@ target_include_directories(
# NAME transTest2
# COMMAND transTest2
#)

1
source/dnode/vnode/CMakeLists.txt
View File

@ -48,7 +48,6 @@ target_sources(
# tq
"src/tq/tq.c"
"src/tq/tqCommit.c"
"src/tq/tqMetaStore.c"
"src/tq/tqOffset.c"
"src/tq/tqPush.c"
"src/tq/tqRead.c"

172
source/dnode/vnode/src/inc/tq.h
View File

@ -20,9 +20,9 @@
#include "executor.h"
#include "os.h"
#include "tcache.h"
#include "thash.h"
#include "tmsg.h"
#include "tqueue.h"
#include "trpc.h"
#include "ttimer.h"
#include "wal.h"
@ -41,45 +41,6 @@ extern "C" {
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
#define TQ_BUFFER_SIZE 4
#define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096
// key + offset + size
#define TQ_IDX_SIZE 24
// 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170
// 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080
// 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3
#define TQ_SVER 0
// TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
typedef struct STqOffsetCfg STqOffsetCfg;
typedef struct STqOffsetStore STqOffsetStore;
@ -98,53 +59,6 @@ struct STqReadHandle {
STSchema* pSchema;
};
typedef struct {
int16_t ver;
int16_t action;
int32_t checksum;
int64_t ssize;
char content[];
} STqSerializedHead;
typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj);
typedef void (*FTqDelete)(void*);
typedef struct {
int64_t key;
int64_t offset;
int64_t serializedSize;
void* valueInUse;
void* valueInTxn;
} STqMetaHandle;
typedef struct STqMetaList {
STqMetaHandle handle;
struct STqMetaList* next;
// struct STqMetaList* inTxnPrev;
// struct STqMetaList* inTxnNext;
struct STqMetaList* unpersistPrev;
struct STqMetaList* unpersistNext;
} STqMetaList;
typedef struct {
STQ* pTq;
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// topics that are not connectted
STqMetaList* unconnectTopic;
TdFilePtr pFile;
TdFilePtr pIdxFile;
char* dirPath;
int32_t tqConfigFlag;
FTqSerialize pSerializer;
FTqDeserialize pDeserializer;
FTqDelete pDeleter;
} STqMetaStore;
typedef struct {
int64_t consumerId;
int32_t epoch;
@ -172,6 +86,9 @@ typedef struct {
qTaskInfo_t task[5];
} STqExec;
int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec);
int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec);
struct STQ {
char* path;
SHashObj* pushMgr; // consumerId -> STqExec*
@ -179,7 +96,7 @@ struct STQ {
SHashObj* pStreamTasks;
SVnode* pVnode;
SWal* pWal;
// TDB* pTdb;
TDB* pTdb;
};
typedef struct {
@ -187,89 +104,12 @@ typedef struct {
tmr_h timer;
} STqMgmt;
static STqMgmt tqMgmt;
typedef struct {
int8_t status;
int64_t offset;
qTaskInfo_t task;
STqReadHandle* pReadHandle;
} STqTaskItem;
// new version
typedef struct {
int64_t firstOffset;
int64_t lastOffset;
STqTaskItem output[TQ_BUFFER_SIZE];
} STqBuffer;
typedef struct {
char topicName[TSDB_TOPIC_FNAME_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
char* qmsg;
STqBuffer buffer;
SWalReadHandle* pReadhandle;
} STqTopic;
typedef struct {
int64_t consumerId;
int32_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqTopic>
} STqConsumer;
typedef struct {
int8_t type;
int8_t nodeType;
int8_t reserved[6];
int64_t streamId;
qTaskInfo_t task;
// TODO sync function
} STqStreamPusher;
typedef struct {
int8_t inited;
tmr_h timer;
} STqPushMgmt;
static STqPushMgmt tqPushMgmt;
static STqMgmt tqMgmt = {0};
// init once
int tqInit();
void tqCleanUp();
// open in each vnode
// required by vnode
int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);
static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }
// tqMetaStore.h
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
FTqDelete pDeleter, int32_t tqConfigFlag);
int32_t tqStoreClose(STqMetaStore*);
// int32_t tqStoreDelete(TqMetaStore*);
// int32_t tqStoreCommitAll(TqMetaStore*);
int32_t tqStorePersist(STqMetaStore*);
// clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore*);
void* tqHandleGet(STqMetaStore*, int64_t key);
// make it unpersist
void* tqHandleTouchGet(STqMetaStore*, int64_t key);
int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
// delete committed kv pair
// notice that a delete action still needs to be committed
int32_t tqHandleDel(STqMetaStore*, int64_t key);
int32_t tqHandlePurge(STqMetaStore*, int64_t key);
int32_t tqHandleCommit(STqMetaStore*, int64_t key);
int32_t tqHandleAbort(STqMetaStore*, int64_t key);
// tqOffset
STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
void STqOffsetClose(STqOffsetStore*);

3
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -278,12 +278,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
pSW = metaGetTableSchema(pMeta, quid, sver, 0);
if (!pSW) return NULL;
tdInitTSchemaBuilder(&sb, 0);
tdInitTSchemaBuilder(&sb, sver);
for (int i = 0; i < pSW->nCols; i++) {
pSchema = pSW->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
taosMemoryFree(pSW->pSchema);

199
source/dnode/vnode/src/tq/tq.c
View File

@ -14,14 +14,37 @@
*/
#include "tq.h"
#include "tqueue.h"
int32_t tqInit() {
//
int8_t old;
while (1) {
old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 2);
if (old != 2) break;
}
if (old == 0) {
tqMgmt.timer = taosTmrInit(10000, 100, 10000, "TQ");
if (tqMgmt.timer == NULL) {
atomic_store_8(&tqMgmt.inited, 0);
return -1;
}
atomic_store_8(&tqMgmt.inited, 1);
}
return 0;
}
void tqCleanUp() {}
void tqCleanUp() {
int8_t old;
while (1) {
old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 2);
if (old != 2) break;
}
if (old == 1) {
taosTmrCleanUp(tqMgmt.timer);
atomic_store_8(&tqMgmt.inited, 0);
}
}
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
STQ* pTq = taosMemoryMalloc(sizeof(STQ));
@ -32,9 +55,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
pTq->path = strdup(path);
pTq->pVnode = pVnode;
pTq->pWal = pWal;
/*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/
/*ASSERT(0);*/
/*}*/
if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {
ASSERT(0);
}
pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
@ -51,11 +74,45 @@ void tqClose(STQ* pTq) {
taosHashCleanup(pTq->execs);
taosHashCleanup(pTq->pStreamTasks);
taosHashCleanup(pTq->pushMgr);
tdbClose(pTq->pTdb);
taosMemoryFree(pTq);
}
// TODO
}
int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeCStr(pEncoder, pExec->subKey) < 0) return -1;
if (tEncodeI64(pEncoder, pExec->consumerId) < 0) return -1;
if (tEncodeI32(pEncoder, pExec->epoch) < 0) return -1;
if (tEncodeI8(pEncoder, pExec->subType) < 0) return -1;
if (tEncodeI8(pEncoder, pExec->withTbName) < 0) return -1;
if (tEncodeI8(pEncoder, pExec->withSchema) < 0) return -1;
if (tEncodeI8(pEncoder, pExec->withTag) < 0) return -1;
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
if (tEncodeCStr(pEncoder, pExec->qmsg) < 0) return -1;
// TODO encode modified exec
}
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pExec->subKey) < 0) return -1;
if (tDecodeI64(pDecoder, &pExec->consumerId) < 0) return -1;
if (tDecodeI32(pDecoder, &pExec->epoch) < 0) return -1;
if (tDecodeI8(pDecoder, &pExec->subType) < 0) return -1;
if (tDecodeI8(pDecoder, &pExec->withTbName) < 0) return -1;
if (tDecodeI8(pDecoder, &pExec->withSchema) < 0) return -1;
if (tDecodeI8(pDecoder, &pExec->withTag) < 0) return -1;
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
if (tDecodeCStrAlloc(pDecoder, &pExec->qmsg) < 0) return -1;
// TODO decode modified exec
}
tEndDecode(pDecoder);
return 0;
}
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
void* pIter = NULL;
while (1) {
@ -214,7 +271,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
// TODO error handle
// TODO handle sma error
}
void* data = taosMemoryMalloc(msgLen);
if (data == NULL) {
@ -230,134 +287,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
int tqCommit(STQ* pTq) {
// do nothing
/*return tqStorePersist(pTq->tqMeta);*/
return 0;
}
int32_t tqGetTopicHandleSize(const STqTopic* pTopic) {
return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) +
sizeof(int64_t) * 3;
}
int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) {
int num = taosArrayGetSize(pConsumer->topics);
int32_t sz = 0;
for (int i = 0; i < num; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
sz += tqGetTopicHandleSize(pTopic);
}
return sz;
}
static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopic->topicName);
/*tlen += taosEncodeString(buf, pTopic->sql);*/
/*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/
tlen += taosEncodeString(buf, pTopic->qmsg);
/*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/
/*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/
/*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/
return tlen;
}
static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) {
buf = taosDecodeStringTo(buf, pTopic->topicName);
/*buf = taosDecodeString(buf, &pTopic->sql);*/
/*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/
buf = taosDecodeString(buf, &pTopic->qmsg);
/*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/
/*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/
/*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/
return buf;
}
static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) {
int32_t sz;
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
tlen += taosEncodeString(buf, pConsumer->cgroup);
sz = taosArrayGetSize(pConsumer->topics);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
tlen += tEncodeSTqTopic(buf, pTopic);
}
return tlen;
}
static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
buf = taosDecodeStringTo(buf, pConsumer->cgroup);
buf = taosDecodeFixedI32(buf, &sz);
pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic));
if (pConsumer->topics == NULL) return NULL;
for (int32_t i = 0; i < sz; i++) {
STqTopic pTopic;
buf = tDecodeSTqTopic(buf, &pTopic);
taosArrayPush(pConsumer->topics, &pTopic);
}
return buf;
}
int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) {
int32_t sz = tEncodeSTqConsumer(NULL, pConsumer);
if (sz > (*ppHead)->ssize) {
void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz);
if (tmpPtr == NULL) {
taosMemoryFree(*ppHead);
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
*ppHead = tmpPtr;
(*ppHead)->ssize = sz;
}
void* ptr = (*ppHead)->content;
void* abuf = ptr;
tEncodeSTqConsumer(&abuf, pConsumer);
return 0;
}
int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) {
const void* str = pHead->content;
*ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
if (*ppConsumer == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
STqConsumer* pConsumer = *ppConsumer;
int32_t sz = taosArrayGetSize(pConsumer->topics);
for (int32_t i = 0; i < sz; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
pTopic->pReadhandle = walOpenReadHandle(pTq->pWal);
if (pTopic->pReadhandle == NULL) {
ASSERT(false);
}
for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
pTopic->buffer.output[j].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnode->pMeta,
.pMsgCb = &pTq->pVnode->msgCb,
};
pTopic->buffer.output[j].pReadHandle = pReadHandle;
pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
}
}
return 0;
}

622
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -1,622 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tq.h"
// #include <fcntl.h>
// #include <string.h>
// #include <unistd.h>
// #include "osDir.h"
#define TQ_META_NAME "tq.meta"
#define TQ_IDX_NAME "tq.idx"
static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key);
static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
if (pNode->unpersistNext == NULL) {
pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pNode->unpersistPrev = pMeta->unpersistHead;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
pMeta->unpersistHead->unpersistNext = pNode;
}
}
static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
int offset = taosLSeekFile(pFile, 0, SEEK_END);
int pageNo = offset / TQ_PAGE_SIZE;
int curPageOffset = pageNo * TQ_PAGE_SIZE;
return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
}
// TODO: the struct is tightly coupled with index entry
typedef struct STqIdxPageHead {
int16_t writeOffset;
int8_t unused[14];
} STqIdxPageHead;
typedef struct STqIdxPageBuf {
STqIdxPageHead head;
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf;
static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(pFile);
int nBytes;
if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (nBytes == 0) {
memset(pBuf, 0, TQ_PAGE_SIZE);
pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
}
ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
return taosLSeekFile(pFile, offset, SEEK_SET);
}
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
FTqDelete deleter, int32_t tqConfigFlag) {
STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore));
if (pMeta == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
}
pMeta->pTq = pTq;
// concat data file name and index file name
size_t pathLen = strlen(path);
pMeta->dirPath = taosMemoryMalloc(pathLen + 1);
if (pMeta->dirPath == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
taosMemoryFree(pMeta);
return NULL;
}
strcpy(pMeta->dirPath, path);
char* name = taosMemoryMalloc(pathLen + 10);
strcpy(name, path);
if (!taosDirExist(name) && taosMkDir(name) != 0) {
terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
tqError("failed to create dir:%s since %s ", name, terrstr());
}
strcat(name, "/" TQ_IDX_NAME);
TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
taosMemoryFree(name);
return NULL;
}
pMeta->pIdxFile = pIdxFile;
pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
taosMemoryFree(name);
return NULL;
}
pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
strcpy(name, path);
strcat(name, "/" TQ_META_NAME);
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
taosMemoryFree(name);
return NULL;
}
taosMemoryFree(name);
pMeta->pFile = pFile;
pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer;
pMeta->pDeleter = deleter;
pMeta->tqConfigFlag = tqConfigFlag;
// read idx file and load into memory
STqIdxPageBuf idxBuf;
STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE);
if (serializedObj == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
}
int idxRead;
int allocated = TQ_PAGE_SIZE;
bool readEnd = false;
while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to read tq index file since %s", terrstr());
}
ASSERT(idxBuf.head.writeOffset == idxRead);
// loop read every entry
for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList));
if (pNode == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
}
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
if (allocated < pNode->handle.serializedSize) {
void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
}
serializedObj = ptr;
allocated = pNode->handle.serializedSize;
}
serializedObj->ssize = pNode->handle.serializedSize;
if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error
}
if (serializedObj->action == TQ_ACTION_INUSE) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse);
} else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN;
}
} else if (serializedObj->action == TQ_ACTION_INTXN) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn);
} else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
}
} else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse);
} else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN;
}
STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
if (ptr->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn);
} else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
}
} else {
ASSERT(0);
}
// put into list
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
if (pBucketNode == NULL) {
pMeta->bucket[bucketKey] = pNode;
} else if (pBucketNode->handle.key == pNode->handle.key) {
pNode->next = pBucketNode->next;
pMeta->bucket[bucketKey] = pNode;
} else {
while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
pBucketNode = pBucketNode->next;
}
if (pBucketNode->next) {
ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
STqMetaList* pNodeFound = pBucketNode->next;
pNode->next = pNodeFound->next;
pBucketNode->next = pNode;
pBucketNode = pNodeFound;
} else {
pNode->next = pMeta->bucket[bucketKey];
pMeta->bucket[bucketKey] = pNode;
pBucketNode = NULL;
}
}
if (pBucketNode) {
if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInUse);
}
if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInTxn);
}
taosMemoryFree(pBucketNode);
}
}
}
taosMemoryFree(serializedObj);
return pMeta;
}
int32_t tqStoreClose(STqMetaStore* pMeta) {
// commit data and idx
tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
while (pNode) {
ASSERT(pNode->unpersistNext == NULL);
ASSERT(pNode->unpersistPrev == NULL);
if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
STqMetaList* next = pNode->next;
taosMemoryFree(pNode);
pNode = next;
}
}
taosMemoryFree(pMeta->dirPath);
taosMemoryFree(pMeta->unpersistHead);
taosMemoryFree(pMeta);
return 0;
}
int32_t tqStoreDelete(STqMetaStore* pMeta) {
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
pMeta->bucket[i] = NULL;
while (pNode) {
if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
STqMetaList* next = pNode->next;
taosMemoryFree(pNode);
pNode = next;
}
}
taosMemoryFree(pMeta->unpersistHead);
taosRemoveDir(pMeta->dirPath);
taosMemoryFree(pMeta->dirPath);
taosMemoryFree(pMeta);
return 0;
}
int32_t tqStorePersist(STqMetaStore* pMeta) {
STqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer;
STqMetaList* pHead = pMeta->unpersistHead;
STqMetaList* pNode = pHead->unpersistNext;
STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead));
if (pSHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
pSHead->ver = TQ_SVER;
pSHead->checksum = 0;
pSHead->ssize = sizeof(STqSerializedHead);
/*int allocatedSize = sizeof(STqSerializedHead);*/
int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
tqReadLastPage(pMeta->pIdxFile, &idxBuf);
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} else {
bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
}
while (pHead != pNode) {
int nBytes = 0;
if (pNode->handle.valueInUse) {
if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INUSE_CONT;
} else {
pSHead->action = TQ_ACTION_INUSE;
}
if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(STqSerializedHead);
} else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
}
nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytes == pSHead->ssize);
}
if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INTXN;
if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(STqSerializedHead);
} else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
}
int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytesTxn == pSHead->ssize);
nBytes += nBytesTxn;
}
pNode->handle.offset = offset;
offset += nBytes;
// write idx file
// TODO: endian check and convert
*(bufPtr++) = pNode->handle.key;
*(bufPtr++) = pNode->handle.offset;
*(bufPtr++) = (int64_t)nBytes;
idxBuf.head.writeOffset += TQ_IDX_SIZE;
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
// TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
bufPtr = (int64_t*)&idxBuf.buffer;
}
// remove from unpersist list
pHead->unpersistNext = pNode->unpersistNext;
pHead->unpersistNext->unpersistPrev = pHead;
pNode->unpersistPrev = pNode->unpersistNext = NULL;
pNode = pHead->unpersistNext;
// remove from bucket
if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
if (pBucketHead == pNode) {
pMeta->bucket[bucketKey] = pNode->next;
} else {
STqMetaList* pBucketNode = pBucketHead;
while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
pBucketNode = pBucketNode->next;
}
// impossible for pBucket->next == NULL
ASSERT(pBucketNode->next == pNode);
pBucketNode->next = pNode->next;
}
taosMemoryFree(pNode);
}
}
// write left bytes
taosMemoryFree(pSHead);
// TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
// TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset);
}
// TODO: using fsync in tfile
taosFsyncFile(pMeta->pIdxFile);
taosFsyncFile(pMeta->pFile);
return 0;
}
static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
// change pointer ownership
pNode->handle.valueInUse = value;
return 0;
} else {
pNode = pNode->next;
}
}
STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList));
if (pNewNode == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
pNewNode->handle.key = key;
pNewNode->handle.valueInUse = value;
pNewNode->next = pMeta->bucket[bucketKey];
// put into unpersist list
pNewNode->unpersistPrev = pMeta->unpersistHead;
pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
pMeta->unpersistHead->unpersistNext = pNewNode;
return 0;
}
void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
return pNode->handle.valueInUse;
} else {
return NULL;
}
} else {
pNode = pNode->next;
}
}
return NULL;
}
void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
tqLinkUnpersist(pMeta, pNode);
return pNode->handle.valueInUse;
} else {
return NULL;
}
} else {
pNode = pNode->next;
}
}
return NULL;
}
static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn) {
if (tqDupIntxnReject(pMeta->tqConfigFlag)) {
terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN;
return -1;
}
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
}
pNode->handle.valueInTxn = value;
tqLinkUnpersist(pMeta, pNode);
return 0;
} else {
pNode = pNode->next;
}
}
STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList));
if (pNewNode == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
pNewNode->handle.key = key;
pNewNode->handle.valueInTxn = value;
pNewNode->next = pMeta->bucket[bucketKey];
pMeta->bucket[bucketKey] = pNewNode;
tqLinkUnpersist(pMeta, pNewNode);
return 0;
}
int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void* vmem = taosMemoryMalloc(vsize);
if (vmem == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
memcpy(vmem, value, vsize);
return tqHandlePutImpl(pMeta, key, vmem);
}
static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
return pNode->handle.valueInTxn;
} else {
return NULL;
}
} else {
pNode = pNode->next;
}
}
return NULL;
}
int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn == NULL) {
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1;
}
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
pNode->handle.valueInUse = pNode->handle.valueInTxn;
pNode->handle.valueInTxn = NULL;
tqLinkUnpersist(pMeta, pNode);
return 0;
} else {
pNode = pNode->next;
}
}
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn) {
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
pNode->handle.valueInTxn = NULL;
tqLinkUnpersist(pMeta, pNode);
return 0;
}
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1;
} else {
pNode = pNode->next;
}
}
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInTxn) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
tqLinkUnpersist(pMeta, pNode);
return 0;
}
} else {
pNode = pNode->next;
}
}
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
pNode->handle.valueInUse = TQ_DELETE_TOKEN;
tqLinkUnpersist(pMeta, pNode);
return 0;
} else {
pNode = pNode->next;
}
}
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
// TODO: clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }

8
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1638,9 +1638,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
int32_t numOfColsOfRow1 = 0;
if (pSchema1 == NULL) {
// pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
// TODO: use the real schemaVersion
pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
}
#ifdef TD_DEBUG_PRINT_ROW
@ -1657,9 +1655,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
if (row2) {
isRow2DataRow = TD_IS_TP_ROW(row2);
if (pSchema2 == NULL) {
// pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
// TODO: use the real schemaVersion
pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1);
pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
}
if (isRow2DataRow) {
numOfColsOfRow2 = schemaNCols(pSchema2);

279
source/dnode/vnode/test/tqMetaTest.cpp
View File

@ -1,279 +0,0 @@
#include <gtest/gtest.h>
#include <cstring>
#include <iostream>
#include <queue>
#include "tqMetaStore.h"
struct Foo {
int32_t a;
};
int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*)pObj;
if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
}
*(int32_t*)(*ppHead)->content = foo->a;
return (*ppHead)->ssize;
}
const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if (*ppObj == NULL) {
*ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t));
}
Foo* pFoo = *(Foo**)ppObj;
pFoo->a = *(int32_t*)pHead->content;
return NULL;
}
void FooDeleter(void* pObj) { taosMemoryFree(pObj); }
class TqMetaUpdateAppendTest : public ::testing::Test {
protected:
void SetUp() override {
taosRemoveDir(pathName);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
}
void TearDown() override { tqStoreClose(pMeta); }
STqMetaStore* pMeta;
const char* pathName = TD_TMP_DIR_PATH "tq_test";
};
TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
Foo foo;
foo.a = 3;
tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo));
Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo->a, 3);
}
TEST_F(TqMetaUpdateAppendTest, persistTest) {
Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
pFoo->a = 2;
tqHandleMovePut(pMeta, 1, pFoo);
Foo* pBar = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pBar == NULL, true);
tqHandleCommit(pMeta, 1);
pBar = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pBar->a, pFoo->a);
pBar = (Foo*)tqHandleGet(pMeta, 2);
EXPECT_EQ(pBar == NULL, true);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pBar = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pBar != NULL, true);
EXPECT_EQ(pBar->a, 2);
pBar = (Foo*)tqHandleGet(pMeta, 2);
EXPECT_EQ(pBar == NULL, true);
}
TEST_F(TqMetaUpdateAppendTest, uncommittedTest) {
Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
TEST_F(TqMetaUpdateAppendTest, abortTest) {
Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleAbort(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
TEST_F(TqMetaUpdateAppendTest, deleteTest) {
Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3);
tqHandleDel(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
tqHandleCommit(pMeta, 1);
Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo));
pBar->a = 4;
tqHandleMovePut(pMeta, 1, pBar);
Foo* pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo1->a, 3);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo1->a, 3);
tqHandleCommit(pMeta, 1);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo1->a, 4);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo1->a, 4);
}
TEST_F(TqMetaUpdateAppendTest, multiplePage) {
taosSeedRand(0);
std::vector<int> v;
for (int i = 0; i < 1000; i++) {
v.push_back(taosRand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
for (int i = 500; i < 1000; i++) {
tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}
TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
taosSeedRand(0);
std::vector<int> v;
for (int i = 0; i < 1000; i++) {
v.push_back(taosRand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i);
v[i] = taosRand();
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for (int i = 500; i < 1000; i++) {
v[i] = taosRand();
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for (int i = 0; i < 1000; i++) {
tqHandleCommit(pMeta, i);
}
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
for (int i = 500; i < 1000; i++) {
v[i] = taosRand();
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
tqHandleCommit(pMeta, i);
}
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}
TEST_F(TqMetaUpdateAppendTest, dupCommit) {
taosSeedRand(0);
std::vector<int> v;
for (int i = 0; i < 1000; i++) {
v.push_back(taosRand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, 0);
ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1);
}
for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1);
}
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}

202
source/libs/executor/src/executorimpl.c
View File

@ -106,7 +106,7 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo);
static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols);
static void releaseQueryBuf(size_t numOfTables);
static void releaseQueryBuf(size_t numOfTables);
static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr);
@ -154,8 +154,9 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn,
void operatorDummyCloseFn(void* param, int32_t numOfCols) {}
static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs);
static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo,
SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset,
SqlFunctionCtx* pCtx, int32_t numOfExprs);
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo);
@ -182,10 +183,10 @@ static int compareRowData(const void* a, const void* b, const void* userData) {
int16_t offset = supporter->dataOffset;
return 0;
// char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset);
// char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
// char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset);
// char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
// return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
// return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
}
// setup the output buffer for each operator
@ -582,8 +583,9 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
colDataAppendInt64(pColData, 4, &pQueryWindow->ekey);
}
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin,
SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol,
int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
for (int32_t k = 0; k < numOfOutput; ++k) {
// keep it temporarily
bool hasAgg = pCtx[k].input.colDataAggIsSet;
@ -665,8 +667,8 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC
}
}
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol) {
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order,
int32_t scanFlag, bool createDummyCol) {
if (pBlock->pBlockAgg != NULL) {
doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order);
} else {
@ -717,7 +719,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
}
static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order,
int32_t scanFlag, bool createDummyCol) {
int32_t scanFlag, bool createDummyCol) {
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
@ -725,7 +727,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
pCtx[i].input.numOfRows = pBlock->info.rows;
pCtx[i].pSrcBlock = pBlock;
pCtx[i].scanFlag = scanFlag;
pCtx[i].scanFlag = scanFlag;
SInputColumnInfoData* pInput = &pCtx[i].input;
pInput->uid = pBlock->info.uid;
@ -834,7 +836,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
@ -852,7 +854,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
// _rowts/_c0, not tbname column
if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
// do nothing
} else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) {
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]);
pfCtx->fpSet.init(&pCtx[k], pResInfo);
@ -950,14 +952,14 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
return false;
}
// if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) {
// // return QUERY_IS_ASC_QUERY(pQueryAttr);
// }
//
// // denote the order type
// if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) {
// // return pCtx->param[0].i == pQueryAttr->order.order;
// }
// if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) {
// // return QUERY_IS_ASC_QUERY(pQueryAttr);
// }
//
// // denote the order type
// if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) {
// // return pCtx->param[0].i == pQueryAttr->order.order;
// }
// in the reverse table scan, only the following functions need to be executed
// if (IS_REVERSE_SCAN(pRuntimeEnv) ||
@ -1072,19 +1074,19 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
for (int32_t i = 0; i < numOfOutput; ++i) {
if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
pValCtx[num++] = &pCtx[i];
} else if (fmIsAggFunc(pCtx[i].functionId)) {
} else if (fmIsSelectFunc(pCtx[i].functionId)) {
p = &pCtx[i];
}
// if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
// tagLen += pCtx[i].resDataInfo.bytes;
// pTagCtx[num++] = &pCtx[i];
// } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) {
// // tag function may be the group by tag column
// // ts may be the required primary timestamp column
// continue;
// } else {
// // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ
// }
// if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
// tagLen += pCtx[i].resDataInfo.bytes;
// pTagCtx[num++] = &pCtx[i];
// } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) {
// // tag function may be the group by tag column
// // ts may be the required primary timestamp column
// continue;
// } else {
// // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ
// }
}
if (p != NULL) {
@ -1123,7 +1125,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
SFuncExecEnv env = {0};
pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) {
if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) {
bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId);
if (!isUdaf) {
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
@ -1882,7 +1884,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
}
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) {
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) {
if (pFilterNode == NULL) {
return;
}
@ -2005,8 +2007,9 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_
}
}
int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) {
int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx,
int32_t numOfExprs) {
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
int32_t start = pGroupResInfo->index;
@ -2055,11 +2058,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
} else {
// expand the result into multiple rows. E.g., _wstartts, top(k, 20)
// the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
for(int32_t k = 0; k < pRow->numOfRows; ++k) {
colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
}
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
for (int32_t k = 0; k < pRow->numOfRows; ++k) {
colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
}
}
}
@ -2070,14 +2073,16 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
}
}
qDebug("%s result generated, rows:%d, groupId:%"PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId);
qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.groupId);
blockDataUpdateTsWindow(pBlock, 0);
return 0;
}
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) {
SExprInfo* pExprInfo = pOperator->pExpr;
int32_t numOfExprs = pOperator->numOfExprs;
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf) {
SExprInfo* pExprInfo = pOperator->pExpr;
int32_t numOfExprs = pOperator->numOfExprs;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t* rowCellOffset = pbInfo->rowCellInfoOffset;
@ -2746,10 +2751,10 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0};
idata.info.type = pSchema[i].type;
idata.info.type = pSchema[i].type;
idata.info.bytes = pSchema[i].bytes;
idata.info.colId = pSchema[i].colId;
idata.hasNull = true;
idata.hasNull = true;
taosArrayPush(pBlock->pDataBlock, &idata);
if (IS_VAR_DATA_TYPE(idata.info.type)) {
@ -3099,7 +3104,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) {
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock,
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock,
SExecTaskInfo* pTaskInfo) {
SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -3212,7 +3217,7 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3
static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr,
int32_t rowIndex) {
for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index
// pCtx[j].startRow = rowIndex;
// pCtx[j].startRow = rowIndex;
}
for (int32_t j = 0; j < numOfExpr; ++j) {
@ -3263,7 +3268,7 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock
SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
// pCtx[i].size = 1;
// pCtx[i].size = 1;
}
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@ -3489,10 +3494,11 @@ _error:
return NULL;
}
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag) {
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) {
// todo add more information about exchange operation
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
*order = TSDB_ORDER_ASC;
*scanFlag = MAIN_SCAN;
return TSDB_CODE_SUCCESS;
@ -3520,7 +3526,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
SAggOperatorInfo* pAggInfo = pOperator->info;
SOptrBasicInfo* pInfo = &pAggInfo->binfo;
SOperatorInfo* downstream = pOperator->pDownstream[0];
SOperatorInfo* downstream = pOperator->pDownstream[0];
int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
@ -3859,7 +3865,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo);
code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs,
pProjectInfo->pPseudoColInfo);
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
@ -4109,7 +4116,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf
for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) {
SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i);
for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) {
STableKeyInfo* pk = taosArrayGet(pa, j);
STableKeyInfo* pk = taosArrayGet(pa, j);
STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++];
pTQueryInfo->lastKey = pk->lastKey;
}
@ -4245,9 +4252,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
goto _error;
}
pInfo->limit = *pLimit;
pInfo->slimit = *pSlimit;
pInfo->curOffset = pLimit->offset;
pInfo->limit = *pLimit;
pInfo->slimit = *pSlimit;
pInfo->curOffset = pLimit->offset;
pInfo->curSOffset = pSlimit->offset;
pInfo->binfo.pRes = pResBlock;
@ -4266,15 +4273,15 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols, pTaskInfo);
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols);
pOperator->name = "ProjectOperator";
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols);
pOperator->name = "ProjectOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pExpr = pExprInfo;
pOperator->numOfExprs = num;
pOperator->pTaskInfo = pTaskInfo;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pExpr = pExprInfo;
pOperator->numOfExprs = num;
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL,
destroyProjectOperatorInfo, NULL, NULL, NULL);
@ -4393,10 +4400,10 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa
}
pCol->slotId = slotId;
pCol->colId = colId;
pCol->bytes = pType->bytes;
pCol->type = pType->type;
pCol->scale = pType->scale;
pCol->colId = colId;
pCol->bytes = pType->bytes;
pCol->type = pType->type;
pCol->scale = pType->scale;
pCol->precision = pType->precision;
pCol->dataBlockId = blockId;
@ -4471,10 +4478,10 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) {
pFuncNode->pParameterList = nodesMakeList();
ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) { // todo handle error
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) { // todo handle error
} else {
res->node.resType = (SDataType) {.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
nodesListAppend(pFuncNode->pParameterList, res);
}
}
@ -4544,7 +4551,7 @@ static SArray* extractColumnInfo(SNodeList* pNodeList);
static SArray* createSortInfo(SNodeList* pNodeList);
static SArray* extractPartitionColInfo(SNodeList* pNodeList);
void extractTableSchemaVersion(SReadHandle *pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
void extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
SMetaReader mr = {0};
metaReaderInit(&mr, pHandle->meta, 0);
metaGetTableEntryByUid(&mr, uid);
@ -4591,7 +4598,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
@ -4600,8 +4607,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (pHandle->vnode) {
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
} else {
doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
queryId, taskId);
doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
}
if (pDataReader == NULL && terrno != 0) {
@ -4612,15 +4618,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
}
SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
SArray* tableIdList = extractTableIdList(pTableGroupInfo);
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
SArray* pCols =
extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo,
pScanPhyNode->node.pConditions, pOperatorDumy);
SOperatorInfo* pOperator =
createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols,
tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy);
taosArrayDestroy(tableIdList);
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@ -4632,7 +4640,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
int32_t numOfOutputCols = 0;
SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
SArray* colList =
extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
SOperatorInfo* pOperator = createSysTableScanOperatorInfo(
pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList,
pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
@ -4654,8 +4663,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num);
int32_t numOfOutputCols = 0;
SArray* colList =
extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo,
COL_MATCH_FROM_COL_ID);
SOperatorInfo* pOperator =
createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo);
@ -4737,7 +4746,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
int32_t numOfOutputCols = 0;
SArray* pColList = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID);
SArray* pColList =
extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID);
pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) {
@ -4769,7 +4779,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
SColumn col = extractColumnFromColumnNode(pColNode);
SColumn col = extractColumnFromColumnNode(pColNode);
pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) {
SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode;
@ -4837,11 +4847,11 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
SColumn c = {0};
c.slotId = pColNode->slotId;
c.colId = pColNode->colId;
c.type = pColNode->node.resType.type;
c.bytes = pColNode->node.resType.bytes;
c.scale = pColNode->node.resType.scale;
c.slotId = pColNode->slotId;
c.colId = pColNode->colId;
c.type = pColNode->node.resType.type;
c.bytes = pColNode->node.resType.bytes;
c.scale = pColNode->node.resType.scale;
c.precision = pColNode->node.resType.precision;
return c;
}
@ -5238,15 +5248,15 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
return TSDB_CODE_SUCCESS;
}
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize,
const char* pKey, const char* pDir) {
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, const char* pKey,
const char* pDir) {
pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize);
int32_t pageSize = rowSize * 32;
int32_t bufSize = pageSize * 4096;
createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);;
pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);
;
return TSDB_CODE_SUCCESS;
}

2
source/libs/function/inc/functionMgtInt.h
View File

@ -28,7 +28,7 @@ extern "C" {
#define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0)
#define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1)
#define FUNC_MGT_NONSTANDARD_SQL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
#define FUNC_MGT_INDEFINITE_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
#define FUNC_MGT_STRING_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
#define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
#define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)

52
source/libs/function/src/builtins.c
View File

@ -14,8 +14,8 @@
*/
#include "builtins.h"
#include "querynodes.h"
#include "builtinsimpl.h"
#include "querynodes.h"
#include "scalar.h"
#include "taoserror.h"
#include "tdatablock.h"
@ -185,6 +185,19 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 1);
if (nodeType(pParamNode) != QUERY_NODE_VALUE) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SValueNode* pValue = (SValueNode*)pParamNode;
if (pValue->datum.i < 0 || pValue->datum.i > 100) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
if (3 == paraNum) {
SNode* pPara3 = nodesListGetNode(pFunc->pParameterList, 2);
if (QUERY_NODE_VALUE != nodeType(pPara3) || !validAperventileAlgo((SValueNode*)pPara3)) {
@ -215,7 +228,7 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SValueNode* pValue = (SValueNode*) pParamNode;
SValueNode* pValue = (SValueNode*)pParamNode;
if (pValue->node.resType.type != TSDB_DATA_TYPE_BIGINT) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@ -224,6 +237,8 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type};
return TSDB_CODE_SUCCESS;
@ -336,7 +351,7 @@ static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT };
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
@ -361,7 +376,7 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT };
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
@ -392,7 +407,7 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
}
}
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType};
return TSDB_CODE_SUCCESS;
}
@ -434,7 +449,7 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
}
SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
uint8_t colType = pCol->resType.type;
uint8_t colType = pCol->resType.type;
if (IS_VAR_DATA_TYPE(colType)) {
pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType};
} else {
@ -463,7 +478,7 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
}
SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
uint8_t colType = pCol->resType.type;
uint8_t colType = pCol->resType.type;
if (IS_VAR_DATA_TYPE(colType)) {
pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType};
} else {
@ -500,8 +515,7 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
if (QUERY_NODE_COLUMN != nodeType(pPara)) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
"The parameters of UNIQUE can only be columns");
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE can only be columns");
}
pFunc->node.resType = ((SExprNode*)pPara)->resType;
@ -823,7 +837,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "top",
.type = FUNCTION_TYPE_TOP,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC,
.translateFunc = translateTop,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = functionSetup,
@ -833,7 +847,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "bottom",
.type = FUNCTION_TYPE_BOTTOM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC,
.translateFunc = translateBottom,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = functionSetup,
@ -915,7 +929,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "diff",
.type = FUNCTION_TYPE_DIFF,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateDiff,
.getEnvFunc = getDiffFuncEnv,
.initFunc = diffFunctionSetup,
@ -925,7 +939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "state_count",
.type = FUNCTION_TYPE_STATE_COUNT,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC,
.translateFunc = translateStateCount,
.getEnvFunc = getStateFuncEnv,
.initFunc = functionSetup,
@ -935,7 +949,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "state_duration",
.type = FUNCTION_TYPE_STATE_DURATION,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateStateDuration,
.getEnvFunc = getStateFuncEnv,
.initFunc = functionSetup,
@ -945,7 +959,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "csum",
.type = FUNCTION_TYPE_CSUM,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateCsum,
.getEnvFunc = getCsumFuncEnv,
.initFunc = functionSetup,
@ -955,7 +969,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "mavg",
.type = FUNCTION_TYPE_MAVG,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateMavg,
.getEnvFunc = getMavgFuncEnv,
.initFunc = mavgFunctionSetup,
@ -965,7 +979,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "sample",
.type = FUNCTION_TYPE_SAMPLE,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateSample,
.getEnvFunc = getSampleFuncEnv,
.initFunc = sampleFunctionSetup,
@ -975,7 +989,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "tail",
.type = FUNCTION_TYPE_TAIL,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateTail,
.getEnvFunc = getTailFuncEnv,
.initFunc = tailFunctionSetup,
@ -985,7 +999,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "unique",
.type = FUNCTION_TYPE_UNIQUE,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateUnique,
.getEnvFunc = getUniqueFuncEnv,
.initFunc = uniqueFunctionSetup,

4
source/libs/function/src/functionMgt.c
View File

@ -145,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG
bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); }
bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId); }
bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); }
bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); }
@ -157,7 +159,7 @@ bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(
bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId); }
bool fmIsNonstandardSQLFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); }
bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); }
bool fmIsSpecialDataRequiredFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED);

38
source/libs/function/src/udfd.c
View File

@ -138,6 +138,7 @@ static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
static int32_t udfdUvInit();
static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
static int32_t udfdRun();
static void udfdConnectMnodeThreadFunc(void* args);
void udfdProcessRequest(uv_work_t *req) {
SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
@ -870,6 +871,23 @@ static int32_t udfdRun() {
return 0;
}
void udfdConnectMnodeThreadFunc(void* args) {
int32_t retryMnodeTimes = 0;
int32_t code = 0;
while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) {
uv_sleep(100 * (1 << retryMnodeTimes));
code = udfdConnectToMnode();
if (code == 0) {
break;
}
fnError("udfd can not connect to mnode, code: %s. retry", tstrerror(code));
}
if (code != 0) {
fnError("udfd can not connect to mnode");
}
}
int main(int argc, char *argv[]) {
if (!taosCheckSystemIsSmallEnd()) {
printf("failed to start since on non-small-end machines\n");
@ -902,30 +920,18 @@ int main(int argc, char *argv[]) {
return -3;
}
int32_t retryMnodeTimes = 0;
int32_t code = 0;
while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) {
uv_sleep(100 * (1 << retryMnodeTimes));
code = udfdConnectToMnode();
if (code == 0) {
break;
}
fnError("can not connect to mnode, code: %s. retry", tstrerror(code));
}
if (code != 0) {
fnError("failed to start since can not connect to mnode");
return -4;
}
if (udfdUvInit() != 0) {
fnError("uv init failure");
return -5;
}
uv_thread_t mnodeConnectThread;
uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL);
udfdRun();
removeListeningPipe();
uv_thread_join(&mnodeConnectThread);
udfdCloseClientRpc();
return 0;

155
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -19,6 +19,21 @@
#include "taos.h"
#include "taoserror.h"
#define COPY_SCALAR_FIELD(fldname) \
do { \
(pDst)->fldname = (pSrc)->fldname; \
} while (0)
#define COPY_CHAR_ARRAY_FIELD(fldname) \
do { \
strcpy((pDst)->fldname, (pSrc)->fldname); \
} while (0)
#define COPY_OBJECT_FIELD(fldname, size) \
do { \
memcpy(&((pDst)->fldname), &((pSrc)->fldname), size); \
} while (0)
#define COPY_CHAR_POINT_FIELD(fldname) \
do { \
if (NULL == (pSrc)->fldname) { \
@ -70,27 +85,61 @@
} \
} while (0)
static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {}
static SNode* exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
dataTypeCopy(&pSrc->resType, &pDst->resType);
pDst->pAssociation = NULL;
COPY_OBJECT_FIELD(resType, sizeof(SDataType));
COPY_CHAR_ARRAY_FIELD(aliasName);
COPY_CHAR_ARRAY_FIELD(userAlias);
return (SNode*)pDst;
}
static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
pDst->pProjectRef = NULL;
COPY_SCALAR_FIELD(tableId);
COPY_SCALAR_FIELD(tableType);
COPY_SCALAR_FIELD(colId);
COPY_SCALAR_FIELD(colType);
COPY_CHAR_ARRAY_FIELD(dbName);
COPY_CHAR_ARRAY_FIELD(tableName);
COPY_CHAR_ARRAY_FIELD(tableAlias);
COPY_CHAR_ARRAY_FIELD(colName);
COPY_SCALAR_FIELD(dataBlockId);
COPY_SCALAR_FIELD(slotId);
return (SNode*)pDst;
}
static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_CHAR_POINT_FIELD(literal);
COPY_SCALAR_FIELD(isDuration);
COPY_SCALAR_FIELD(translate);
COPY_SCALAR_FIELD(notReserved);
COPY_SCALAR_FIELD(placeholderNo);
COPY_SCALAR_FIELD(typeData);
COPY_SCALAR_FIELD(unit);
if (!pSrc->translate) {
return (SNode*)pDst;
}
switch (pSrc->node.resType.type) {
case TSDB_DATA_TYPE_BOOL:
COPY_SCALAR_FIELD(datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
COPY_SCALAR_FIELD(datum.i);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
COPY_SCALAR_FIELD(datum.d);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
COPY_SCALAR_FIELD(datum.u);
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
@ -104,7 +153,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
case TSDB_DATA_TYPE_MEDIUMBLOB:
default:
break;
}
@ -113,6 +162,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_SCALAR_FIELD(opType);
CLONE_NODE_FIELD(pLeft);
CLONE_NODE_FIELD(pRight);
return (SNode*)pDst;
@ -120,18 +170,27 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_SCALAR_FIELD(condType);
CLONE_NODE_LIST_FIELD(pParameterList);
return (SNode*)pDst;
}
static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_CHAR_ARRAY_FIELD(functionName);
COPY_SCALAR_FIELD(funcId);
COPY_SCALAR_FIELD(funcType);
CLONE_NODE_LIST_FIELD(pParameterList);
COPY_SCALAR_FIELD(udfBufSize);
return (SNode*)pDst;
}
static SNode* tableNodeCopy(const STableNode* pSrc, STableNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_CHAR_ARRAY_FIELD(dbName);
COPY_CHAR_ARRAY_FIELD(tableName);
COPY_CHAR_ARRAY_FIELD(tableAlias);
COPY_SCALAR_FIELD(precision);
return (SNode*)pDst;
}
@ -159,6 +218,8 @@ static SNode* realTableNodeCopy(const SRealTableNode* pSrc, SRealTableNode* pDst
COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
COPY_CHAR_ARRAY_FIELD(qualDbName);
COPY_SCALAR_FIELD(ratio);
return (SNode*)pDst;
}
@ -170,6 +231,7 @@ static SNode* tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDst
static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) {
COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
COPY_SCALAR_FIELD(joinType);
CLONE_NODE_FIELD(pLeft);
CLONE_NODE_FIELD(pRight);
CLONE_NODE_FIELD(pOnCond);
@ -177,21 +239,30 @@ static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst
}
static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId);
COPY_SCALAR_FIELD(slotId);
CLONE_NODE_FIELD(pExpr);
return (SNode*)pDst;
}
static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) {
COPY_SCALAR_FIELD(groupingSetType);
CLONE_NODE_LIST_FIELD(pParameterList);
return (SNode*)pDst;
}
static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) {
CLONE_NODE_FIELD(pExpr);
COPY_SCALAR_FIELD(order);
COPY_SCALAR_FIELD(nullOrder);
return (SNode*)pDst;
}
static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { return (SNode*)pDst; }
static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) {
COPY_SCALAR_FIELD(limit);
COPY_SCALAR_FIELD(offset);
return (SNode*)pDst;
}
static SNode* stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNode* pDst) {
CLONE_NODE_FIELD(pCol);
@ -215,13 +286,16 @@ static SNode* intervalWindowNodeCopy(const SIntervalWindowNode* pSrc, SIntervalW
}
static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) {
COPY_OBJECT_FIELD(dataType, sizeof(SDataType));
CLONE_NODE_LIST_FIELD(pNodeList);
return (SNode*)pDst;
}
static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
COPY_SCALAR_FIELD(mode);
CLONE_NODE_FIELD(pValues);
CLONE_NODE_FIELD(pWStartTs);
COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
return (SNode*)pDst;
}
@ -229,7 +303,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions);
CLONE_NODE_LIST_FIELD(pChildren);
pDst->pParent = NULL;
COPY_SCALAR_FIELD(optimizedFlag);
return (SNode*)pDst;
}
@ -239,12 +313,25 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
CLONE_NODE_LIST_FIELD(pScanPseudoCols);
CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
COPY_SCALAR_FIELD(scanType);
COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2);
COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow));
COPY_OBJECT_FIELD(tableName, sizeof(SName));
COPY_SCALAR_FIELD(showRewrite);
COPY_SCALAR_FIELD(ratio);
CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
COPY_SCALAR_FIELD(dataRequired);
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(sliding);
COPY_SCALAR_FIELD(intervalUnit);
COPY_SCALAR_FIELD(slidingUnit);
return (SNode*)pDst;
}
static SNode* logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(joinType);
CLONE_NODE_FIELD(pOnConditions);
return (SNode*)pDst;
}
@ -259,32 +346,50 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pProjections);
COPY_CHAR_ARRAY_FIELD(stmtName);
COPY_SCALAR_FIELD(limit);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(slimit);
COPY_SCALAR_FIELD(soffset);
return (SNode*)pDst;
}
static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
pDst->pDataBlocks = NULL;
pDst->pVgDataBlocks = NULL;
COPY_SCALAR_FIELD(msgType);
return (SNode*)pDst;
}
static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(srcGroupId);
COPY_SCALAR_FIELD(precision);
return (SNode*)pDst;
}
static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(winType);
CLONE_NODE_LIST_FIELD(pFuncs);
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(sliding);
COPY_SCALAR_FIELD(intervalUnit);
COPY_SCALAR_FIELD(slidingUnit);
COPY_SCALAR_FIELD(sessionGap);
CLONE_NODE_FIELD(pTspk);
CLONE_NODE_FIELD(pStateExpr);
COPY_SCALAR_FIELD(triggerType);
COPY_SCALAR_FIELD(watermark);
return (SNode*)pDst;
}
static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(mode);
CLONE_NODE_FIELD(pWStartTs);
CLONE_NODE_FIELD(pValues);
COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
return (SNode*)pDst;
}
@ -301,28 +406,41 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi
}
static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
COPY_OBJECT_FIELD(id, sizeof(SSubplanId));
CLONE_NODE_FIELD(pNode);
pDst->pChildren = NULL;
pDst->pParents = NULL;
pDst->pVgroupList = NULL;
COPY_SCALAR_FIELD(subplanType);
COPY_SCALAR_FIELD(level);
COPY_SCALAR_FIELD(splitFlag);
return (SNode*)pDst;
}
static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId);
CLONE_NODE_LIST_FIELD(pSlots);
COPY_SCALAR_FIELD(totalRowSize);
COPY_SCALAR_FIELD(outputRowSize);
COPY_SCALAR_FIELD(precision);
return (SNode*)pDst;
}
static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) {
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(slotId);
COPY_OBJECT_FIELD(dataType, sizeof(SDataType));
COPY_SCALAR_FIELD(reserve);
COPY_SCALAR_FIELD(output);
COPY_SCALAR_FIELD(tag);
return (SNode*)pDst;
}
static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) {
COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr));
COPY_SCALAR_FIELD(taskId);
COPY_SCALAR_FIELD(schedId);
return (SNode*)pDst;
}
static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
COPY_SCALAR_FIELD(isDistinct);
CLONE_NODE_LIST_FIELD(pProjectionList);
CLONE_NODE_FIELD(pFromTable);
CLONE_NODE_FIELD(pWhere);
@ -333,6 +451,12 @@ static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
CLONE_NODE_LIST_FIELD(pOrderByList);
CLONE_NODE_FIELD(pLimit);
CLONE_NODE_FIELD(pLimit);
COPY_CHAR_ARRAY_FIELD(stmtName);
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(isEmptyResult);
COPY_SCALAR_FIELD(isTimeOrderQuery);
COPY_SCALAR_FIELD(hasAggFuncs);
COPY_SCALAR_FIELD(hasRepeatScanFuncs);
return (SNode*)pDst;
}
@ -345,7 +469,6 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memcpy(pDst, pNode, nodesNodeSize(nodeType(pNode)));
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN:
return columnNodeCopy((const SColumnNode*)pNode, (SColumnNode*)pDst);
@ -387,6 +510,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst);
case QUERY_NODE_DOWNSTREAM_SOURCE:
return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst);
case QUERY_NODE_LEFT_VALUE:
return pDst;
case QUERY_NODE_SELECT_STMT:
return selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst);
case QUERY_NODE_LOGIC_PLAN_SCAN:

22
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -78,6 +78,8 @@ const char* nodesNodeName(ENodeType type) {
return "TableOptions";
case QUERY_NODE_INDEX_OPTIONS:
return "IndexOptions";
case QUERY_NODE_LEFT_VALUE:
return "LeftValue";
case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT:
@ -490,6 +492,7 @@ static const char* jkScanLogicPlanScanCols = "ScanCols";
static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize";
static const char* jkScanLogicPlanTableMeta = "TableMeta";
static const char* jkScanLogicPlanTagCond = "TagCond";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@ -507,6 +510,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond);
}
return code;
}
@ -528,6 +534,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond);
}
return code;
}
@ -1775,6 +1784,7 @@ static const char* jkSubplanDbFName = "DbFName";
static const char* jkSubplanNodeAddr = "NodeAddr";
static const char* jkSubplanRootNode = "RootNode";
static const char* jkSubplanDataSink = "DataSink";
static const char* jkSubplanTagCond = "TagCond";
static int32_t subplanToJson(const void* pObj, SJson* pJson) {
const SSubplan* pNode = (const SSubplan*)pObj;
@ -1801,6 +1811,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSubplanDataSink, nodeToJson, pNode->pDataSink);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSubplanTagCond, nodeToJson, pNode->pTagCond);
}
return code;
}
@ -1831,6 +1844,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSubplanDataSink, (SNode**)&pNode->pDataSink);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSubplanTagCond, (SNode**)&pNode->pTagCond);
}
return code;
}
@ -2175,7 +2191,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
varDataSetLen(pNode->datum.p, pNode->node.resType.bytes);
varDataSetLen(pNode->datum.p, pNode->node.resType.bytes - VARSTR_HEADER_SIZE);
if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) {
char* buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1);
if (NULL == buf) {
@ -3019,6 +3035,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
break;
case QUERY_NODE_DOWNSTREAM_SOURCE:
return downstreamSourceNodeToJson(pObj, pJson);
case QUERY_NODE_LEFT_VALUE:
return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize.
case QUERY_NODE_SET_OPERATOR:
return setOperatorToJson(pObj, pJson);
case QUERY_NODE_SELECT_STMT:
@ -3130,6 +3148,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToSlotDescNode(pJson, pObj);
case QUERY_NODE_DOWNSTREAM_SOURCE:
return jsonToDownstreamSourceNode(pJson, pObj);
case QUERY_NODE_LEFT_VALUE:
return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize.
case QUERY_NODE_SET_OPERATOR:
return jsonToSetOperator(pJson, pObj);
case QUERY_NODE_SELECT_STMT:

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -79,6 +79,8 @@ int32_t nodesNodeSize(ENodeType type) {
return sizeof(SStreamOptions);
case QUERY_NODE_TOPIC_OPTIONS:
return sizeof(STopicOptions);
case QUERY_NODE_LEFT_VALUE:
return sizeof(SLeftValueNode);
case QUERY_NODE_SET_OPERATOR:
return sizeof(SSetOperator);
case QUERY_NODE_SELECT_STMT:

30
source/libs/parser/src/parAstCreater.c
View File

@ -342,25 +342,19 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ
CHECK_OUT_OF_MEM(cond);
cond->condType = type;
cond->pParameterList = nodesMakeList();
if ((QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type != ((SLogicConditionNode*)pParam1)->condType) ||
(QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type != ((SLogicConditionNode*)pParam2)->condType)) {
nodesListAppend(cond->pParameterList, pParam1);
nodesListAppend(cond->pParameterList, pParam2);
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type == ((SLogicConditionNode*)pParam1)->condType) {
nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList);
((SLogicConditionNode*)pParam1)->pParameterList = NULL;
nodesDestroyNode(pParam1);
} else {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1)) {
nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList);
((SLogicConditionNode*)pParam1)->pParameterList = NULL;
nodesDestroyNode(pParam1);
} else {
nodesListAppend(cond->pParameterList, pParam1);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2)) {
nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList);
((SLogicConditionNode*)pParam2)->pParameterList = NULL;
nodesDestroyNode(pParam2);
} else {
nodesListAppend(cond->pParameterList, pParam2);
}
nodesListAppend(cond->pParameterList, pParam1);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type == ((SLogicConditionNode*)pParam2)->condType) {
nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList);
((SLogicConditionNode*)pParam2)->pParameterList = NULL;
nodesDestroyNode(pParam2);
} else {
nodesListAppend(cond->pParameterList, pParam2);
}
return (SNode*)cond;
}

2
source/libs/parser/src/parInsert.c
View File

@ -647,7 +647,7 @@ static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, in
if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
const char* rowEnd = tdRowEnd(rb->pBuf);
STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len);
tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx);
tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx);
} else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;

98
source/libs/parser/src/parTranslater.c
View File

@ -292,8 +292,8 @@ static bool isScanPseudoColumnFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsScanPseudoColumnFunc(((SFunctionNode*)pNode)->funcId));
}
static bool isNonstandardSQLFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsNonstandardSQLFunc(((SFunctionNode*)pNode)->funcId));
static bool isIndefiniteRowsFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsIndefiniteRowsFunc(((SFunctionNode*)pNode)->funcId));
}
static bool isDistinctOrderBy(STranslateContext* pCxt) {
@ -646,12 +646,13 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
}
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY: {
pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1);
pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1);
if (NULL == pVal->datum.p) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
}
varDataSetLen(pVal->datum.p, targetDt.bytes);
strncpy(varDataVal(pVal->datum.p), pVal->literal, targetDt.bytes);
int32_t len = TMIN(targetDt.bytes - VARSTR_HEADER_SIZE, pVal->node.resType.bytes);
varDataSetLen(pVal->datum.p, len);
strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
@ -662,19 +663,18 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
break;
}
case TSDB_DATA_TYPE_NCHAR: {
int32_t bytes = targetDt.bytes * TSDB_NCHAR_SIZE;
pVal->datum.p = taosMemoryCalloc(1, bytes + VARSTR_HEADER_SIZE + 1);
pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1);
if (NULL == pVal->datum.p) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
;
}
int32_t output = 0;
if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), bytes,
&output)) {
int32_t len = 0;
if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p),
targetDt.bytes - VARSTR_HEADER_SIZE, &len)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
varDataSetLen(pVal->datum.p, output);
varDataSetLen(pVal->datum.p, len);
break;
}
case TSDB_DATA_TYPE_JSON:
@ -690,8 +690,20 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
return DEAL_RES_CONTINUE;
}
static int32_t calcTypeBytes(SDataType dt) {
if (TSDB_DATA_TYPE_BINARY == dt.type) {
return dt.bytes + VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
} else {
return dt.bytes;
}
}
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
return translateValueImpl(pCxt, pVal, pVal->node.resType);
SDataType dt = pVal->node.resType;
dt.bytes = calcTypeBytes(dt);
return translateValueImpl(pCxt, pVal, dt);
}
static bool isMultiResFunc(SNode* pNode) {
@ -806,7 +818,7 @@ static EDealRes haveAggOrNonstdFunction(SNode* pNode, void* pContext) {
if (isAggFunc(pNode)) {
*((bool*)pContext) = true;
return DEAL_RES_END;
} else if (isNonstandardSQLFunc(pNode)) {
} else if (isIndefiniteRowsFunc(pNode)) {
*((bool*)pContext) = true;
return DEAL_RES_END;
}
@ -851,6 +863,15 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) {
return hasInvalidFunc;
}
static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
.pRpc = pCxt->pParseCxt->pTransporter,
.pMgmtEps = &pCxt->pParseCxt->mgmtEpSet,
.pErrBuf = pCxt->msgBuf.buf,
.errBufLen = pCxt->msgBuf.len};
return fmGetFuncInfo(&param, pFunc);
}
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
SNode* pParam = NULL;
FOREACH(pParam, pFunc->pParameterList) {
@ -859,12 +880,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc)
}
}
SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
.pRpc = pCxt->pParseCxt->pTransporter,
.pMgmtEps = &pCxt->pParseCxt->mgmtEpSet,
.pErrBuf = pCxt->msgBuf.buf,
.errBufLen = pCxt->msgBuf.len};
pCxt->errCode = fmGetFuncInfo(&param, pFunc);
pCxt->errCode = getFuncInfo(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) {
if (beforeHaving(pCxt->currClause)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION);
@ -872,7 +888,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc)
if (hasInvalidFuncNesting(pFunc->pParameterList)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING);
}
if (pCxt->pCurrStmt->hasNonstdSQLFunc) {
if (pCxt->pCurrStmt->hasIndefiniteRowsFunc) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
@ -899,14 +915,15 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc)
}
}
}
if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsNonstandardSQLFunc(pFunc->funcId)) {
if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrStmt->hasNonstdSQLFunc || pCxt->pCurrStmt->hasAggFuncs) {
if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsIndefiniteRowsFunc(pFunc->funcId)) {
if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrStmt->hasIndefiniteRowsFunc ||
pCxt->pCurrStmt->hasAggFuncs) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
if (hasInvalidFuncNesting(pFunc->pParameterList)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING);
}
pCxt->pCurrStmt->hasNonstdSQLFunc = true;
pCxt->pCurrStmt->hasIndefiniteRowsFunc = true;
}
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
@ -990,7 +1007,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode
strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName);
pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
translateFunction(pCxt, pFunc);
pCxt->errCode == getFuncInfo(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
*pNode = (SNode*)pFunc;
@ -1060,7 +1077,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList
}
static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) {
if (isAggFunc(*pNode)) {
if (isAggFunc(*pNode) || isIndefiniteRowsFunc(*pNode)) {
return DEAL_RES_IGNORE_CHILD;
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
@ -1097,7 +1114,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) {
pCxt->existAggFunc = true;
return DEAL_RES_IGNORE_CHILD;
}
if (isNonstandardSQLFunc(pNode)) {
if (isIndefiniteRowsFunc(pNode)) {
pCxt->existNonstdFunc = true;
return DEAL_RES_IGNORE_CHILD;
}
@ -1939,7 +1956,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d
nodesDestroyNode(pFunc);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) {
if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) {
nodesClearList(pFunc->pParameterList);
pFunc->pParameterList = NULL;
nodesDestroyNode(pFunc);
@ -2343,16 +2360,6 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
return buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq);
}
static int32_t calcTypeBytes(SDataType dt) {
if (TSDB_DATA_TYPE_BINARY == dt.type) {
return dt.bytes + VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
} else {
return dt.bytes;
}
}
static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode;
@ -4082,24 +4089,15 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS
}
static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) {
if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) {
return pCxt->errCode;
int32_t code = getFuncInfo(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == code) {
code = scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal);
}
return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal);
}
static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) {
if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) {
return bytes - VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == type) {
return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
}
return bytes;
return code;
}
static SDataType schemaToDataType(SSchema* pSchema) {
SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0};
dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes);
return dt;
}

2
source/libs/planner/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
planner
PRIVATE os util nodes catalog cjson parser function qcom scalar
PRIVATE os util nodes catalog cjson parser function qcom scalar index
PUBLIC transport
)

6
source/libs/planner/src/planLogicCreater.c
View File

@ -418,7 +418,7 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr)
}
static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) {
if (!pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && NULL == pSelect->pGroupByList) {
return TSDB_CODE_SUCCESS;
}
@ -442,8 +442,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) {
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs);
if (TSDB_CODE_SUCCESS == code && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc)) {
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsVectorFunc, &pAgg->pAggFuncs);
}
// rewrite the expression in subsequent clauses

112
source/libs/planner/src/planOptimizer.c
View File

@ -15,6 +15,7 @@
#include "filter.h"
#include "functionMgt.h"
#include "index.h"
#include "planInt.h"
#define OPTIMIZE_FLAG_MASK(n) (1 << n)
@ -313,22 +314,53 @@ static EDealRes cpdIsPrimaryKeyCondImpl(SNode* pNode, void* pContext) {
}
static bool cpdIsPrimaryKeyCond(SNode* pNode) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
return false;
}
bool isPrimaryKeyCond = false;
nodesWalkExpr(pNode, cpdIsPrimaryKeyCondImpl, &isPrimaryKeyCond);
return isPrimaryKeyCond;
}
static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) {
static EDealRes cpdIsTagCondImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
*((bool*)pContext) = ((COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType) ? true : false);
return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END;
}
return DEAL_RES_CONTINUE;
}
static bool cpdIsTagCond(SNode* pNode) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
return false;
}
bool isTagCond = false;
nodesWalkExpr(pNode, cpdIsTagCondImpl, &isTagCond);
return isTagCond;
}
static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond,
SNode** pOtherCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pScan->node.pConditions;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
*pPrimaryKeyCond = NULL;
*pOtherCond = pScan->node.pConditions;
pScan->node.pConditions = NULL;
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pPrimaryKeyConds = NULL;
SNodeList* pTagConds = NULL;
SNodeList* pOtherConds = NULL;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
if (cpdIsPrimaryKeyCond(pCond)) {
code = nodesListMakeAppend(&pPrimaryKeyConds, nodesCloneNode(pCond));
} else if (cpdIsTagCond(pScan->node.pConditions)) {
code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond));
} else {
code = nodesListMakeAppend(&pOtherConds, nodesCloneNode(pCond));
}
@ -338,37 +370,46 @@ static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimary
}
SNode* pTempPrimaryKeyCond = NULL;
SNode* pTempTagCond = NULL;
SNode* pTempOtherCond = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = cpdMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds);
}
if (TSDB_CODE_SUCCESS == code) {
code = cpdMergeConds(&pTempTagCond, &pTagConds);
}
if (TSDB_CODE_SUCCESS == code) {
code = cpdMergeConds(&pTempOtherCond, &pOtherConds);
}
if (TSDB_CODE_SUCCESS == code) {
*pPrimaryKeyCond = pTempPrimaryKeyCond;
*pTagCond = pTempTagCond;
*pOtherCond = pTempOtherCond;
nodesDestroyNode(pScan->node.pConditions);
pScan->node.pConditions = NULL;
} else {
nodesDestroyList(pPrimaryKeyConds);
nodesDestroyList(pTagConds);
nodesDestroyList(pOtherConds);
nodesDestroyNode(pTempPrimaryKeyCond);
nodesDestroyNode(pTempTagCond);
nodesDestroyNode(pTempOtherCond);
}
return code;
}
static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pScan->node.pConditions)->condType) {
return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pOtherCond);
static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond,
SNode** pOtherCond) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions)) {
return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pTagCond, pOtherCond);
}
if (cpdIsPrimaryKeyCond(pScan->node.pConditions)) {
*pPrimaryKeyCond = pScan->node.pConditions;
} else if (cpdIsTagCond(pScan->node.pConditions)) {
*pTagCond = pScan->node.pConditions;
} else {
*pOtherCond = pScan->node.pConditions;
}
@ -391,6 +432,32 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
return code;
}
static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** pOtherCond) {
int32_t code = TSDB_CODE_SUCCESS;
SIdxFltStatus idxStatus = idxGetFltStatus(*pTagCond);
switch (idxStatus) {
case SFLT_NOT_INDEX:
code = cpdCondAppend(pOtherCond, pTagCond);
break;
case SFLT_COARSE_INDEX:
pScan->pTagCond = nodesCloneNode(*pTagCond);
if (NULL == pScan->pTagCond) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
code = cpdCondAppend(pOtherCond, pTagCond);
break;
case SFLT_ACCURATE_INDEX:
pScan->pTagCond = *pTagCond;
*pTagCond = NULL;
break;
default:
code = TSDB_CODE_FAILED;
break;
}
return code;
}
static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) {
@ -398,11 +465,15 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
}
SNode* pPrimaryKeyCond = NULL;
SNode* pTagCond = NULL;
SNode* pOtherCond = NULL;
int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pOtherCond);
int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pTagCond, &pOtherCond);
if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) {
code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond);
}
if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) {
code = cpdApplyTagIndex(pScan, &pTagCond, &pOtherCond);
}
if (TSDB_CODE_SUCCESS == code) {
pScan->node.pConditions = pOtherCond;
}
@ -618,30 +689,6 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
}
}
// static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) {
// if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// return TSDB_CODE_SUCCESS;
// }
// static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) {
// if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// bool hasPrimaryKeyEqualCond = false;
// SNode* pCond = NULL;
// FOREACH(pCond, pOnCond->pParameterList) {
// if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) {
// hasPrimaryKeyEqualCond = true;
// }
// }
// if (!hasPrimaryKeyEqualCond) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// return TSDB_CODE_SUCCESS;
// }
static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN);
@ -650,11 +697,6 @@ static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin)
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
}
return TSDB_CODE_SUCCESS;
// if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) {
// return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions);
// } else {
// return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions);
// }
}
static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {

55
source/libs/planner/src/planPhysiCreater.c
View File

@ -411,7 +411,7 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys
return sortScanCols(pScanPhysiNode->pScanCols);
}
static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode,
static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) {
int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols);
if (TSDB_CODE_SUCCESS == code) {
@ -438,6 +438,12 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName));
if (NULL != pScanLogicNode->pTagCond) {
pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond);
if (NULL == pSubplan->pTagCond) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
}
if (TSDB_CODE_SUCCESS == code) {
@ -463,7 +469,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla
}
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode);
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode);
}
static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
@ -498,7 +504,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
pTableScan->intervalUnit = pScanLogicNode->intervalUnit;
pTableScan->slidingUnit = pScanLogicNode->slidingUnit;
return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode);
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode);
}
static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
@ -522,7 +528,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet;
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
}
static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
@ -599,14 +605,17 @@ typedef struct SRewritePrecalcExprsCxt {
static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
SNode* pExpr = nodesCloneNode(*pNode);
if (NULL == pExpr) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
@ -624,16 +633,45 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode)
return DEAL_RES_IGNORE_CHILD;
}
static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
if (NULL == pOper) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pOper->pLeft = nodesMakeNode(QUERY_NODE_LEFT_VALUE);
if (NULL == pOper->pLeft) {
nodesDestroyNode(pOper);
return TSDB_CODE_OUT_OF_MEMORY;
}
SValueNode* pVal = (SValueNode*)*pNode;
pOper->node.resType = pVal->node.resType;
strcpy(pOper->node.aliasName, pVal->node.aliasName);
pOper->opType = OP_TYPE_ASSIGN;
pOper->pRight = *pNode;
*pNode = (SNode*)pOper;
return TSDB_CODE_SUCCESS;
}
static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext;
switch (nodeType(*pNode)) {
case QUERY_NODE_VALUE: {
if (((SValueNode*)*pNode)->notReserved) {
break;
}
pCxt->errCode = rewriteValueToOperator(pCxt, pNode);
if (TSDB_CODE_SUCCESS != pCxt->errCode) {
return DEAL_RES_ERROR;
}
return collectAndRewrite(pCxt, pNode);
}
case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION: {
return collectAndRewrite(pContext, pNode);
return collectAndRewrite(pCxt, pNode);
}
case QUERY_NODE_FUNCTION: {
if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) {
return collectAndRewrite(pContext, pNode);
return collectAndRewrite(pCxt, pNode);
}
}
default:
@ -677,9 +715,8 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN
}
SRewritePrecalcExprsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs};
nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt);
if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
nodesDestroyList(cxt.pPrecalcExprs);
*pPrecalcExprs = NULL;
if (0 == LIST_LENGTH(cxt.pPrecalcExprs) || TSDB_CODE_SUCCESS != cxt.errCode) {
DESTORY_LIST(*pPrecalcExprs);
}
return cxt.errCode;
}

10
source/libs/planner/src/planner.c
View File

@ -18,6 +18,13 @@
#include "planInt.h"
#include "scalar.h"
static void dumpQueryPlan(SQueryPlan* pPlan) {
char* pStr = NULL;
nodesNodeToString(pPlan, false, &pStr, NULL);
planDebugL("Query Plan: %s", pStr);
taosMemoryFree(pStr);
}
int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) {
SLogicNode* pLogicNode = NULL;
SLogicSubplan* pLogicSubplan = NULL;
@ -36,6 +43,9 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
if (TSDB_CODE_SUCCESS == code) {
code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList);
}
if (TSDB_CODE_SUCCESS == code) {
dumpQueryPlan(*pPlan);
}
nodesDestroyNode(pLogicNode);
nodesDestroyNode(pLogicSubplan);

2
source/libs/planner/test/planBasicTest.cpp
View File

@ -50,4 +50,6 @@ TEST_F(PlanBasicTest, func) {
run("SELECT DIFF(c1) FROM t1");
run("SELECT PERCENTILE(c1, 60) FROM t1");
run("SELECT TOP(c1, 60) FROM t1");
}

2
source/libs/planner/test/planGroupByTest.cpp
View File

@ -49,6 +49,8 @@ TEST_F(PlanGroupByTest, aggFunc) {
run("SELECT LAST(*), FIRST(*) FROM t1");
run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1");
run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2");
}
TEST_F(PlanGroupByTest, selectFunc) {

6
source/libs/planner/test/planOptimizeTest.cpp
View File

@ -32,6 +32,12 @@ TEST_F(PlanOptimizeTest, optimizeScanData) {
run("SELECT PERCENTILE(c1, 40), COUNT(*) FROM t1");
}
TEST_F(PlanOptimizeTest, ConditionPushDown) {
useDb("root", "test");
run("SELECT ts, c1 FROM st1 WHERE tag1 > 4");
}
TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
useDb("root", "test");

20
source/libs/planner/test/planTestUtil.cpp
View File

@ -233,45 +233,45 @@ class PlannerTestBaseImpl {
if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) {
if (res_.prepareAst_.empty()) {
cout << "syntax tree : " << endl;
cout << "+++++++++++++++++++++syntax tree : " << endl;
cout << res_.ast_ << endl;
} else {
cout << "prepare syntax tree : " << endl;
cout << "+++++++++++++++++++++prepare syntax tree : " << endl;
cout << res_.prepareAst_ << endl;
cout << "bound syntax tree : " << endl;
cout << "+++++++++++++++++++++bound syntax tree : " << endl;
cout << res_.boundAst_ << endl;
cout << "syntax tree : " << endl;
cout << "+++++++++++++++++++++syntax tree : " << endl;
cout << res_.ast_ << endl;
}
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) {
cout << "raw logic plan : " << endl;
cout << "+++++++++++++++++++++raw logic plan : " << endl;
cout << res_.rawLogicPlan_ << endl;
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) {
cout << "optimized logic plan : " << endl;
cout << "+++++++++++++++++++++optimized logic plan : " << endl;
cout << res_.optimizedLogicPlan_ << endl;
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) {
cout << "split logic plan : " << endl;
cout << "+++++++++++++++++++++split logic plan : " << endl;
cout << res_.splitLogicPlan_ << endl;
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) {
cout << "scaled logic plan : " << endl;
cout << "+++++++++++++++++++++scaled logic plan : " << endl;
cout << res_.scaledLogicPlan_ << endl;
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) {
cout << "physical plan : " << endl;
cout << "+++++++++++++++++++++physical plan : " << endl;
cout << res_.physiPlan_ << endl;
}
if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) {
cout << "physical subplan : " << endl;
cout << "+++++++++++++++++++++physical subplan : " << endl;
for (const auto& subplan : res_.physiSubplans_) {
cout << subplan << endl;
}

7
source/libs/scalar/src/scalar.c
View File

@ -182,6 +182,11 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) {
int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) {
switch (nodeType(node)) {
case QUERY_NODE_LEFT_VALUE: {
SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0);
param->numOfRows = pb->info.rows;
break;
}
case QUERY_NODE_VALUE: {
SValueNode *valueNode = (SValueNode *)node;
@ -845,7 +850,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
}
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)|| QUERY_NODE_LEFT_VALUE == nodeType(pNode)) {
return DEAL_RES_CONTINUE;
}

2
source/libs/scalar/src/sclfunc.c
View File

@ -824,7 +824,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
}
//for constant conversion, need to set proper length of pOutput description
if (len < outputLen) {
pOutput->columnData->info.bytes = len;
pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE;
}
break;
}

18
source/libs/scalar/src/sclvector.c
View File

@ -1333,6 +1333,22 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
doReleaseVec(pLeftCol, leftConvert);
}
void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData;
pOut->numOfRows = pLeft->numOfRows;
if (colDataIsNull_s(pRight->columnData, 0)) {
for (int32_t i = 0; i < pOut->numOfRows; ++i) {
colDataAppend(pOutputCol, i, NULL, true);
}
} else {
for (int32_t i = 0; i < pOut->numOfRows; ++i) {
colDataAppend(pOutputCol, i, colDataGetData(pRight->columnData, 0), false);
}
}
}
void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
#if 0
int32_t len = pLeft->bytes + pRight->bytes;
@ -1691,6 +1707,8 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorMathRemainder;
case OP_TYPE_MINUS:
return vectorMathMinus;
case OP_TYPE_ASSIGN:
return vectorAssign;
case OP_TYPE_GREATER_THAN:
return vectorGreater;
case OP_TYPE_GREATER_EQUAL:

57
source/libs/wal/src/walMgmt.c
View File

@ -14,17 +14,17 @@
*/
#define _DEFAULT_SOURCE
#include "tcompare.h"
#include "os.h"
#include "taoserror.h"
#include "tcompare.h"
#include "tref.h"
#include "walInt.h"
typedef struct {
int8_t stop;
int8_t inited;
uint32_t seq;
int32_t refSetId;
int8_t stop;
int8_t inited;
uint32_t seq;
int32_t refSetId;
TdThread thread;
} SWalMgmt;
@ -36,30 +36,42 @@ static void walFreeObj(void *pWal);
int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); }
int32_t walInit() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1);
if (old == 1) return 0;
tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
int32_t code = walCreateThread();
if (code != 0) {
wError("failed to init wal module since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0);
return code;
int8_t old;
while (1) {
old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 2);
if (old != 2) break;
}
if (old == 0) {
tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
int32_t code = walCreateThread();
if (code != 0) {
wError("failed to init wal module since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0);
return code;
}
wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId);
atomic_store_8(&tsWal.inited, 1);
}
wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId);
return 0;
}
void walCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0);
if (old == 0) {
return;
int8_t old;
while (1) {
old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 2);
if (old != 2) break;
}
if (old == 1) {
walStopThread();
taosCloseRef(tsWal.refSetId);
wInfo("wal module is cleaned up");
atomic_store_8(&tsWal.inited, 0);
}
walStopThread();
taosCloseRef(tsWal.refSetId);
wInfo("wal module is cleaned up");
}
SWal *walOpen(const char *path, SWalCfg *pCfg) {
@ -126,7 +138,6 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
}
if (walCheckAndRepairIdx(pWal) < 0) {
}
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,

1
source/util/src/terror.c
View File

@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exist
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null")
// mnode-mq
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists")