xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-01-27 10:28:17 +08:00
parent 78e49d6b08 7cc364bcba
commit 3fdbfae4c1
100 changed files with 4511 additions and 1429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -86,6 +86,7 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/
tests/examples/JDBC/JDBCDemo/.classpath
tests/examples/JDBC/JDBCDemo/.project
tests/examples/JDBC/JDBCDemo/.settings/
source/libs/parser/inc/new_sql.*
# Emacs
# -*- mode: gitignore; -*-
@ -101,4 +102,4 @@ TAGS
contrib/*
!contrib/CMakeLists.txt
!contrib/test
!contrib/test

6
cmake/cmake.options
View File

@ -50,6 +50,12 @@ option(
OFF
)
option(
BUILD_WITH_UV_TRANS
"If build with libuv_trans "
OFF
)
option(
BUILD_WITH_CRAFT
"If build with canonical-raft"

74
include/common/common.h
View File

@ -80,6 +80,80 @@ typedef struct SColumnInfoData {
char *pData; // the corresponding block data in memory
} SColumnInfoData;
static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid;
int32_t numOfCols = pBlock->info.numOfCols;
int32_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, tbUid);
tlen += taosEncodeFixedI32(buf, numOfCols);
tlen += taosEncodeFixedI32(buf, rows);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
tlen += taosEncodeFixedI16(buf, pColData->info.colId);
tlen += taosEncodeFixedI16(buf, pColData->info.type);
tlen += taosEncodeFixedI16(buf, pColData->info.bytes);
int32_t colSz = rows * pColData->info.bytes;
tlen += taosEncodeBinary(buf, pColData->pData, colSz);
}
return tlen;
}
static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI32(buf, &pBlock->info.numOfCols);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz);
pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
for (int32_t i = 0; i < sz; i++) {
SColumnInfoData data;
buf = taosDecodeFixedI16(buf, &data.info.colId);
buf = taosDecodeFixedI16(buf, &data.info.type);
buf = taosDecodeFixedI16(buf, &data.info.bytes);
int32_t colSz = pBlock->info.rows * data.info.bytes;
buf = taosDecodeBinary(buf, (void**)&data.pData, colSz);
taosArrayPush(pBlock->pDataBlock, &data);
}
return buf;
}
static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) {
int32_t tlen = 0;
int32_t sz = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas);
if (pRsp->pBlockData) {
sz = taosArrayGetSize(pRsp->pBlockData);
}
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pBlock = (SSDataBlock*) taosArrayGet(pRsp->pBlockData, i);
tlen += tEncodeDataBlock(buf, pBlock);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper));
if (pRsp->schemas == NULL) return NULL;
buf = tDecodeSSchemaWrapper(buf, pRsp->schemas);
buf = taosDecodeFixedI32(buf, &sz);
pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
for (int32_t i = 0; i < sz; i++) {
SSDataBlock block;
tDecodeDataBlock(buf, &block);
taosArrayPush(pRsp->pBlockData, &block);
}
return buf;
}
//======================================================================================================================
// the following structure shared by parser and executor
typedef struct SColumn {

14
include/common/tep.h
View File

@ -1,6 +1,10 @@
#ifndef TDENGINE_TEP_H
#define TDENGINE_TEP_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tmsg.h"
@ -9,10 +13,16 @@ typedef struct SCorEpSet {
SEpSet epSet;
} SCorEpSet;
int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
int taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
SEpSet getEpSet_s(SCorEpSet *pEpSet);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TEP_H

118
include/common/tmsg.h
View File

@ -161,10 +161,10 @@ typedef struct {
#pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
typedef struct {
typedef struct SEp {
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
} SEpAddr;
} SEp;
typedef struct {
int32_t contLen;
@ -273,8 +273,7 @@ typedef struct {
typedef struct SEpSet {
int8_t inUse;
int8_t numOfEps;
uint16_t port[TSDB_MAX_REPLICA];
char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
SEp eps[TSDB_MAX_REPLICA];
} SEpSet;
static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) {
@ -282,8 +281,8 @@ static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) {
tlen += taosEncodeFixedI8(buf, pEp->inUse);
tlen += taosEncodeFixedI8(buf, pEp->numOfEps);
for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
tlen += taosEncodeFixedU16(buf, pEp->port[i]);
tlen += taosEncodeString(buf, pEp->fqdn[i]);
tlen += taosEncodeFixedU16(buf, pEp->eps[i].port);
tlen += taosEncodeString(buf, pEp->eps[i].fqdn);
}
return tlen;
}
@ -292,8 +291,8 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
buf = taosDecodeFixedI8(buf, &pEp->inUse);
buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
buf = taosDecodeFixedU16(buf, &pEp->port[i]);
buf = taosDecodeStringTo(buf, pEp->fqdn[i]);
buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
}
return buf;
}
@ -624,8 +623,7 @@ typedef struct {
int32_t id;
int8_t isMnode;
int8_t align;
uint16_t port;
char fqdn[TSDB_FQDN_LEN];
SEp ep;
} SDnodeEp;
typedef struct {
@ -698,24 +696,17 @@ typedef struct {
char tableNames[];
} SMultiTableInfoReq;
// todo refactor
typedef struct SVgroupInfo {
int32_t vgId;
uint32_t hashBegin;
uint32_t hashEnd;
int8_t inUse;
int8_t numOfEps;
SEpAddr epAddr[TSDB_MAX_REPLICA];
SEpSet epset;
} SVgroupInfo;
typedef struct {
int32_t vgId;
int8_t numOfEps;
SEpAddr epAddr[TSDB_MAX_REPLICA];
} SVgroupMsg;
typedef struct {
int32_t numOfVgroups;
SVgroupMsg vgroups[];
int32_t numOfVgroups;
SVgroupInfo vgroups[];
} SVgroupsInfo;
typedef struct {
@ -1572,9 +1563,7 @@ typedef struct SMqSetCVgReq {
char* sql;
char* logicalPlan;
char* physicalPlan;
uint32_t qmsgLen;
void* qmsg;
//SSubQueryMsg msg;
char* qmsg;
} SMqSetCVgReq;
static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) {
@ -1606,7 +1595,6 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->logicalPlan);
tlen += taosEncodeString(buf, pReq->physicalPlan);
tlen += taosEncodeFixedU32(buf, pReq->qmsgLen);
tlen += taosEncodeString(buf, (char*)pReq->qmsg);
//tlen += tEncodeSSubQueryMsg(buf, &pReq->msg);
return tlen;
@ -1621,7 +1609,6 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeString(buf, &pReq->sql);
buf = taosDecodeString(buf, &pReq->logicalPlan);
buf = taosDecodeString(buf, &pReq->physicalPlan);
buf = taosDecodeFixedU32(buf, &pReq->qmsgLen);
buf = taosDecodeString(buf, (char**)&pReq->qmsg);
//buf = tDecodeSSubQueryMsg(buf, &pReq->msg);
return buf;
@ -1635,41 +1622,76 @@ typedef struct SMqSetCVgRsp {
char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqSetCVgRsp;
typedef struct SMqColData {
int16_t colId;
int16_t type;
int16_t bytes;
char data[];
} SMqColData;
typedef struct {
uint32_t nCols;
SSchema *pSchema;
} SSchemaWrapper;
static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pSchema->type);
tlen += taosEncodeFixedI32(buf, pSchema->bytes);
tlen += taosEncodeFixedI32(buf, pSchema->colId);
tlen += taosEncodeString(buf, pSchema->name);
return tlen;
}
static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) {
buf = taosDecodeFixedI8(buf, &pSchema->type);
buf = taosDecodeFixedI32(buf, &pSchema->bytes);
buf = taosDecodeFixedI32(buf, &pSchema->colId);
buf = taosDecodeStringTo(buf, pSchema->name);
return buf;
}
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
int32_t tlen = 0;
tlen += taosEncodeFixedU32(buf, pSW->nCols);
for (int32_t i = 0; i < pSW->nCols; i ++) {
tlen += tEncodeSSchema(buf, &pSW->pSchema[i]);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->nCols);
pSW->pSchema = (SSchema*) calloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) {
return NULL;
}
for (int32_t i = 0; i < pSW->nCols; i ++) {
buf = tDecodeSSchema(buf, &pSW->pSchema[i]);
}
return buf;
}
typedef struct SMqTbData {
int64_t uid;
int32_t numOfCols;
int32_t numOfRows;
SMqColData colData[];
char* colData;
} SMqTbData;
typedef struct SMqTopicBlk {
char topicName[TSDB_TOPIC_FNAME_LEN];
int64_t committedOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t bodyLen;
int32_t numOfTb;
SMqTbData tbData[];
char topicName[TSDB_TOPIC_FNAME_LEN];
int64_t committedOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t bodyLen;
int32_t numOfTb;
SMqTbData* tbData;
} SMqTopicData;
typedef struct SMqConsumeRsp {
int64_t reqId;
int64_t consumerId;
int32_t bodyLen;
int32_t numOfTopics;
SMqTopicData data[];
int64_t consumerId;
SSchemaWrapper* schemas;
int32_t numOfTopics;
SArray* pBlockData; //SArray<SSDataBlock>
} SMqConsumeRsp;
// one req for one vg+topic
typedef struct SMqConsumeReq {
SMsgHead head;
//0: commit only, current offset
//1: consume only, poll next offset
//2: commit current and consume next offset
@ -1702,7 +1724,7 @@ typedef struct SMqCMGetSubEpRsp {
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI16(buf, pVgEp->vgId);
tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
return tlen;
}

30
include/common/ttokendef.h
View File

@ -207,6 +207,36 @@
#define TK_INTO 189
#define TK_VALUES 190
#define NEW_TK_UNION 1
#define NEW_TK_ALL 2
#define NEW_TK_MINUS 3
#define NEW_TK_EXCEPT 4
#define NEW_TK_INTERSECT 5
#define NEW_TK_NK_PLUS 6
#define NEW_TK_NK_MINUS 7
#define NEW_TK_NK_STAR 8
#define NEW_TK_NK_SLASH 9
#define NEW_TK_SHOW 10
#define NEW_TK_DATABASES 11
#define NEW_TK_NK_ID 12
#define NEW_TK_NK_LP 13
#define NEW_TK_NK_RP 14
#define NEW_TK_NK_COMMA 15
#define NEW_TK_NK_LITERAL 16
#define NEW_TK_NK_DOT 17
#define NEW_TK_SELECT 18
#define NEW_TK_DISTINCT 19
#define NEW_TK_AS 20
#define NEW_TK_FROM 21
#define NEW_TK_WITH 22
#define NEW_TK_RECURSIVE 23
#define NEW_TK_ORDER 24
#define NEW_TK_BY 25
#define NEW_TK_ASC 26
#define NEW_TK_DESC 27
#define NEW_TK_NULLS 28
#define NEW_TK_FIRST 29
#define NEW_TK_LAST 30
#define TK_SPACE 300
#define TK_COMMENT 301

3
include/libs/executor/executor.h
View File

@ -24,6 +24,7 @@ extern "C" {
typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
struct SRpcMsg;
struct SSubplan;
/**
@ -208,6 +209,8 @@ void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
*/
void** qDeregisterQInfo(void* pMgmt, void* pQInfo);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
#ifdef __cplusplus
}
#endif

2
include/libs/parser/parsenodes.h
View File

@ -35,7 +35,7 @@ typedef struct SQueryNode {
int16_t type;
} SQueryNode;
#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type)
#define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type)
typedef struct SField {
char name[TSDB_COL_NAME_LEN];

13
include/libs/qcom/query.h
View File

@ -128,20 +128,9 @@ typedef struct SMsgSendInfo {
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
int8_t inUse;
int8_t numOfEps;
SEpAddr epAddr[TSDB_MAX_REPLICA];
SEpSet epset;
} SQueryNodeAddr;
static FORCE_INLINE void tConvertQueryAddrToEpSet(SEpSet* pEpSet, const SQueryNodeAddr* pAddr) {
pEpSet->inUse = pAddr->inUse;
pEpSet->numOfEps = pAddr->numOfEps;
for (int j = 0; j < TSDB_MAX_REPLICA; j++) {
pEpSet->port[j] = pAddr->epAddr[j].port;
memcpy(pEpSet->fqdn[j], pAddr->epAddr[j].fqdn, TSDB_FQDN_LEN);
}
}
int32_t initTaskQueue();
int32_t cleanupTaskQueue();

7
include/libs/qworker/qworker.h
View File

@ -49,9 +49,10 @@ typedef struct {
} SQWorkerStat;
typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *);
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp);
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
putReqToQueryQFp fp1, sendReqToDnodeFp fp2);
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
@ -65,6 +66,8 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);

15
include/libs/transport/transport.h
View File

@ -20,21 +20,6 @@
extern "C" {
#endif
//typedef struct SEpAddr {
// char fqdn[TSDB_FQDN_LEN];
// uint16_t port;
//} SEpAddr;
//
//typedef struct SVgroup {
// int32_t vgId;
// int8_t numOfEps;
// SEpAddr epAddr[TSDB_MAX_REPLICA];
//} SVgroup;
//
//typedef struct SVgroupsInfo {
// int32_t numOfVgroups;
// SVgroup vgroups[];
//} SVgroupsInfo;
#ifdef __cplusplus
}

3
include/libs/wal/wal.h
View File

@ -19,6 +19,7 @@
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
@ -159,7 +160,7 @@ int32_t walAlter(SWal *, SWalCfg *pCfg);
void walClose(SWal *);
// write
int64_t walWrite(SWal *, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen);
int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
void walFsync(SWal *, bool force);
// apis for lifecycle management

81
include/nodes/nodes.h
View File

@ -20,7 +20,6 @@
extern "C" {
#endif
#include "tarray.h"
#include "tdef.h"
typedef enum ENodeType {
@ -35,12 +34,14 @@ typedef enum ENodeType {
QUERY_NODE_JOIN_TABLE,
QUERY_NODE_GROUPING_SET,
QUERY_NODE_ORDER_BY_EXPR,
QUERY_NODE_LIMIT,
QUERY_NODE_STATE_WINDOW,
QUERY_NODE_SESSION_WINDOW,
QUERY_NODE_INTERVAL_WINDOW,
QUERY_NODE_SET_OPERATOR,
QUERY_NODE_SELECT_STMT
QUERY_NODE_SELECT_STMT,
QUERY_NODE_SHOW_STMT
} ENodeType;
/**
@ -52,6 +53,27 @@ typedef struct SNode {
} SNode;
#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type)
#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type))
typedef struct SListCell {
SNode* pNode;
struct SListCell* pNext;
} SListCell;
typedef struct SNodeList {
int16_t length;
SListCell* pHeader;
} SNodeList;
#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0)
#define FOREACH(node, list) \
for (SListCell* cell = (NULL != (list) ? (list)->pHeader : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
#define FORBOTH(node1, list1, node2, list2) \
for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHeader : NULL), *cell2 = (NULL != (list2) ? (list2)->pHeader : NULL); \
(NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
cell1 = cell1->pNext, cell2 = cell2->pNext)
typedef struct SDataType {
uint8_t type;
@ -128,7 +150,7 @@ typedef enum ELogicConditionType {
typedef struct SLogicConditionNode {
ENodeType type; // QUERY_NODE_LOGIC_CONDITION
ELogicConditionType condType;
SArray* pParameterList;
SNodeList* pParameterList;
} SLogicConditionNode;
typedef struct SIsNullCondNode {
@ -141,7 +163,7 @@ typedef struct SFunctionNode {
SExprNode type; // QUERY_NODE_FUNCTION
char functionName[TSDB_FUNC_NAME_LEN];
int32_t funcId;
SArray* pParameterList; // SNode
SNodeList* pParameterList; // SNode
} SFunctionNode;
typedef struct STableNode {
@ -151,12 +173,12 @@ typedef struct STableNode {
} STableNode;
typedef struct SRealTableNode {
STableNode type; // QUERY_NODE_REAL_TABLE
STableNode table; // QUERY_NODE_REAL_TABLE
char dbName[TSDB_DB_NAME_LEN];
} SRealTableNode;
typedef struct STempTableNode {
STableNode type; // QUERY_NODE_TEMP_TABLE
STableNode table; // QUERY_NODE_TEMP_TABLE
SNode* pSubquery;
} STempTableNode;
@ -165,7 +187,7 @@ typedef enum EJoinType {
} EJoinType;
typedef struct SJoinTableNode {
STableNode type; // QUERY_NODE_JOIN_TABLE
STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType;
SNode* pLeft;
SNode* pRight;
@ -179,7 +201,7 @@ typedef enum EGroupingSetType {
typedef struct SGroupingSetNode {
ENodeType type; // QUERY_NODE_GROUPING_SET
EGroupingSetType groupingSetType;
SArray* pParameterList;
SNodeList* pParameterList;
} SGroupingSetNode;
typedef enum EOrder {
@ -188,7 +210,8 @@ typedef enum EOrder {
} EOrder;
typedef enum ENullOrder {
NULL_ORDER_FIRST = 1,
NULL_ORDER_DEFAULT = 1,
NULL_ORDER_FIRST,
NULL_ORDER_LAST
} ENullOrder;
@ -199,10 +222,11 @@ typedef struct SOrderByExprNode {
ENullOrder nullOrder;
} SOrderByExprNode;
typedef struct SLimitInfo {
typedef struct SLimitNode {
ENodeType type; // QUERY_NODE_LIMIT
uint64_t limit;
uint64_t offset;
} SLimitInfo;
} SLimitNode;
typedef struct SStateWindowNode {
ENodeType type; // QUERY_NODE_STATE_WINDOW
@ -225,15 +249,16 @@ typedef struct SIntervalWindowNode {
typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct;
SArray* pProjectionList; // SNode
bool isStar;
SNodeList* pProjectionList; // SNode
SNode* pFromTable;
SNode* pWhereCond;
SArray* pPartitionByList; // SNode
SNodeList* pPartitionByList; // SNode
SNode* pWindowClause;
SArray* pGroupByList; // SGroupingSetNode
SArray* pOrderByList; // SOrderByExprNode
SLimitInfo limit;
SLimitInfo slimit;
SNodeList* pGroupByList; // SGroupingSetNode
SNodeList* pOrderByList; // SOrderByExprNode
SLimitNode limit;
SLimitNode slimit;
} SSelectStmt;
typedef enum ESetOperatorType {
@ -249,20 +274,24 @@ typedef struct SSetOperator {
typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext);
bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext);
bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext);
bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext);
bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext);
bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext);
bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext);
bool nodeEqual(const SNode* a, const SNode* b);
bool nodesEqualNode(const SNode* a, const SNode* b);
void cloneNode(const SNode* pNode);
void nodesCloneNode(const SNode* pNode);
int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen);
int32_t stringToNode(const char* pStr, SNode** pNode);
int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen);
int32_t nodesStringToNode(const char* pStr, SNode** pNode);
bool isTimeorderQuery(const SNode* pQuery);
bool isTimelineQuery(const SNode* pQuery);
bool nodesIsTimeorderQuery(const SNode* pQuery);
bool nodesIsTimelineQuery(const SNode* pQuery);
SNode* nodesMakeNode(ENodeType type);
void nodesDestroyNode(SNode* pNode);
void nodesDestroyNodeList(SNodeList* pList);
#ifdef __cplusplus
}

38
include/nodes/nodesShowStmts.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_NODES_SHOW_STMTS_H_
#define _TD_NODES_SHOW_STMTS_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "nodes.h"
typedef enum EShowStmtType {
SHOW_TYPE_DATABASE = 1
} EShowStmtType;
typedef struct SShowStmt {
ENodeType type; // QUERY_NODE_SHOW_STMT
EShowStmtType showType;
} SShowStmt;
#ifdef __cplusplus
}
#endif
#endif /*_TD_NODES_SHOW_STMTS_H_*/

1
include/util/taoserror.h
View File

@ -253,6 +253,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4)
#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5)
#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6)
#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7)
// dnode
#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)

4
source/client/src/clientEnv.c
View File

@ -127,6 +127,8 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
void destroyTscObj(void *pObj) {
STscObj *pTscObj = pObj;
SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType};
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
pthread_mutex_destroy(&pTscObj->mutex);
@ -517,4 +519,4 @@ setConfRet taos_set_config(const char *config){
pthread_mutex_unlock(&setConfMutex);
return ret;
}
#endif
#endif

37
source/client/src/clientHb.c
View File

@ -300,11 +300,13 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) {
static void* hbThreadFunc(void* param) {
setThreadName("hb");
while (1) {
int8_t threadStop = atomic_load_8(&clientHbMgr.threadStop);
if(threadStop) {
int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2);
if(1 == threadStop) {
break;
}
pthread_mutex_lock(&clientHbMgr.lock);
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for(int i = 0; i < sz; i++) {
SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
@ -352,6 +354,9 @@ static void* hbThreadFunc(void* param) {
atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
}
pthread_mutex_unlock(&clientHbMgr.lock);
taosMsleep(HEARTBEAT_INTERVAL);
}
return NULL;
@ -371,7 +376,13 @@ static int32_t hbCreateThread() {
}
static void hbStopThread() {
atomic_store_8(&clientHbMgr.threadStop, 1);
if (atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 0, 1)) {
return;
}
while (2 != atomic_load_8(&clientHbMgr.threadStop)) {
usleep(10);
}
}
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) {
@ -409,11 +420,18 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) {
return NULL;
}
pthread_mutex_lock(&clientHbMgr.lock);
taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
pthread_mutex_unlock(&clientHbMgr.lock);
return pAppHbMgr;
}
void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) {
if (NULL == pAppHbMgr) {
return;
}
pthread_mutex_lock(&clientHbMgr.lock);
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
@ -421,7 +439,9 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) {
SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (pAppHbMgr == pTarget) {
taosHashCleanup(pTarget->activeInfo);
pTarget->activeInfo = NULL;
taosHashCleanup(pTarget->connInfo);
pTarget->connInfo = NULL;
}
}
@ -446,11 +466,17 @@ int hbMgrInit() {
}
void hbMgrCleanUp() {
hbStopThread();
// destroy all appHbMgr
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);
if (old == 0) return;
taosArrayDestroy(clientHbMgr.appHbMgrs);
pthread_mutex_lock(&clientHbMgr.lock);
taosArrayDestroy(clientHbMgr.appHbMgrs);
pthread_mutex_unlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL;
}
int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
@ -502,6 +528,9 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) {
taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
if (atomic_load_32(&pAppHbMgr->connKeyCnt) <= 0) {
appHbMgrCleanup(pAppHbMgr);
}
}
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) {

164
source/client/src/clientImpl.c
View File

@ -101,7 +101,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
}
if (port) {
epSet.epSet.port[0] = port;
epSet.epSet.eps[0].port = port;
}
} else {
if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
@ -328,6 +328,7 @@ struct tmq_t {
char clientId[256];
int64_t consumerId;
int64_t status;
tsem_t rspSem;
STscObj* pTscObj;
tmq_commit_cb* commit_cb;
int32_t nextTopicIdx;
@ -344,6 +345,7 @@ tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t err
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
pTmq->commit_cb = conf->commit_cb;
tsem_init(&pTmq->rspSem, 0, 0);
pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
return pTmq;
@ -372,11 +374,27 @@ int32_t tmq_list_append(tmq_list_t* ptr, char* src) {
}
int32_t tmq_null_cb(void* param, const SDataBuf* pMsg, int32_t code) {
if (code == 0) {
//
}
//
return 0;
}
TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
SRequestObj *pRequest = NULL;
tmq->status = 1;
int32_t sz = topic_list->cnt;
tmq->clientTopics = taosArrayInit(sz, sizeof(void*));
//destroy ex
taosArrayDestroy(tmq->clientTopics);
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
SCMSubscribeReq req;
req.topicNum = sz;
req.consumerId = tmq->consumerId;
req.consumerGroup = strdup(tmq->groupId);
req.topicNames = taosArrayInit(sz, sizeof(void*));
for (int i = 0; i < sz; i++) {
char* topicName = topic_list->elems[i];
@ -391,16 +409,21 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
}
tNameExtractFullName(&name, topicFname);
tscDebug("subscribe topic: %s", topicFname);
taosArrayPush(tmq->clientTopics, &topicFname);
SMqClientTopic topic = {
.nextVgIdx = 0,
.sql = NULL,
.sqlLen = 0,
.topicId = 0,
.topicName = topicFname,
.vgs = NULL
};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
/*SMqClientTopic topic = {*/
/*.*/
/*};*/
taosArrayPush(req.topicNames, &topicFname);
}
SCMSubscribeReq req;
req.topicNum = taosArrayGetSize(tmq->clientTopics);
req.consumerId = tmq->consumerId;
req.consumerGroup = strdup(tmq->groupId);
req.topicNames = tmq->clientTopics;
int tlen = tSerializeSCMSubscribeReq(NULL, &req);
void* buf = malloc(tlen);
@ -419,18 +442,19 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
/*sendInfo->fp*/
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
_return:
if (body != NULL) {
destroySendMsgInfo(body);
}
/*if (sendInfo != NULL) {*/
/*destroySendMsgInfo(sendInfo);*/
/*}*/
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
@ -564,19 +588,19 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql,
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
_return:
qDestroyQuery(pQueryNode);
if (body != NULL) {
destroySendMsgInfo(body);
}
/*if (sendInfo != NULL) {*/
/*destroySendMsgInfo(sendInfo);*/
/*}*/
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
@ -592,11 +616,38 @@ struct tmq_message_t {
};
int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) {
SMqConsumeRsp rsp;
tDecodeSMqConsumeRsp(pMsg->pData, &rsp);
int32_t colNum = rsp.schemas->nCols;
for (int32_t i = 0; i < colNum; i++) {
printf("| %s |", rsp.schemas->pSchema[i].name);
}
printf("\n");
int32_t sz = taosArrayGetSize(rsp.pBlockData);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i);
int32_t rows = pDataBlock->info.rows;
for (int32_t j = 0; j < colNum; j++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, j);
for (int32_t k = 0; k < rows; k++) {
void* var = POINTER_SHIFT(pColInfoData->pData, k * pColInfoData->info.bytes);
if (j == 0) printf(" %ld ", *(int64_t*)var);
if (j == 1) printf(" %d ", *(int32_t*)var);
}
}
/*pDataBlock->*/
}
return 0;
}
int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = (tmq_t*)param;
if (code != 0) {
tsem_post(&tmq->rspSem);
return 0;
}
tscDebug("tmq ask ep cb called");
bool set = false;
SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp);
int32_t sz = taosArrayGetSize(rsp.topics);
@ -611,38 +662,35 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) {
for (int32_t j = 0; j < vgSz; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
SMqClientVg clientVg = {
.pollCnt = 0,
.committedOffset = -1,
.currentOffset = -1,
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet
};
taosArrayPush(topic.vgs, &clientVg);
set = true;
}
taosArrayPush(tmq->clientTopics, &topic);
}
if(set) tmq->status = 1;
// unlock
tsem_post(&tmq->rspSem);
return 0;
}
tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) {
return NULL;
}
SRequestObj *pRequest = NULL;
SMqConsumeReq req = {0};
req.reqType = 1;
req.blockingTime = blocking_time;
req.consumerId = tmq->consumerId;
tmq_message_t* tmq_message = NULL;
strcpy(req.cgroup, tmq->groupId);
if (taosArrayGetSize(tmq->clientTopics) == 0) {
if (taosArrayGetSize(tmq->clientTopics) == 0 || tmq->status == 0) {
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen);
if (buf == NULL) {
tscError("failed to malloc get subscribe ep buf");
}
buf->consumerId = htobe64(buf->consumerId);
buf->consumerId = htobe64(tmq->consumerId);
strcpy(buf->cgroup, tmq->groupId);
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
if (pRequest == NULL) {
tscError("failed to malloc subscribe ep request");
}
@ -659,23 +707,38 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
tsem_wait(&tmq->rspSem);
}
SMqClientTopic* pTopic = taosArrayGetP(tmq->clientTopics, tmq->nextTopicIdx);
if (taosArrayGetSize(tmq->clientTopics) == 0) {
tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
return NULL;
}
SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
pReq->reqType = 1;
pReq->blockingTime = blocking_time;
pReq->consumerId = tmq->consumerId;
tmq_message_t* tmq_message = NULL;
strcpy(pReq->cgroup, tmq->groupId);
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
strcpy(req.topic, pTopic->topicName);
strcpy(pReq->topic, pTopic->topicName);
int32_t nextVgIdx = pTopic->nextVgIdx;
pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx);
req.offset = pVg->currentOffset;
pReq->offset = pVg->currentOffset;
pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) };
pRequest->type = TDMT_VND_CONSUME;
pReq->head.vgId = htonl(pVg->vgId);
pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) };
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = &tmq_message;
/*sendInfo->param = &tmq_message;*/
sendInfo->fp = tmq_poll_cb_inner;
int64_t transporterId = 0;
@ -729,7 +792,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
if (qIsDdlQuery(pQueryNode)) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
pRequest->code = terrno;
@ -759,7 +821,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe
return -1;
}
taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0]));
taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
mgmtEpSet->numOfEps++;
}
@ -769,7 +831,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe
return -1;
}
taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
}
@ -974,14 +1036,7 @@ void* doFetchRow(SRequestObj* pRequest) {
SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex);
epSet.numOfEps = pVgroupInfo->numOfEps;
epSet.inUse = pVgroupInfo->inUse;
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i]));
epSet.port[i] = pVgroupInfo->epAddr[i].port;
}
epSet = pVgroupInfo->epset;
} else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) {
pRequest->type = TDMT_VND_SHOW_TABLES;
SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
@ -998,14 +1053,7 @@ void* doFetchRow(SRequestObj* pRequest) {
pRequest->body.requestMsg.pData = pShowReq;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
epSet.numOfEps = pVgroupInfo->numOfEps;
epSet.inUse = pVgroupInfo->inUse;
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i]));
epSet.port[i] = pVgroupInfo->epAddr[i].port;
}
epSet = pVgroupInfo->epset;
int64_t transporterId = 0;
STscObj *pTscObj = pRequest->pTscObj;

2
source/client/src/clientMain.c
View File

@ -46,6 +46,8 @@ void taos_cleanup(void) {
clientConnRefPool = -1;
taosCloseRef(id);
hbMgrCleanUp();
rpcCleanup();
catalogDestroy();
taosCloseLog();

7
source/client/src/clientMsgHandler.c
View File

@ -54,7 +54,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
assert(pConnect->epSet.numOfEps > 0);
for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) {
pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]);
pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port);
}
if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) {
@ -62,7 +62,8 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
}
for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id);
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn,
pConnect->epSet.eps[i].port, pTscObj->id);
}
pTscObj->connId = pConnect->connId;
@ -72,6 +73,8 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pTscObj->pAppInfo->clusterId = pConnect->clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
pTscObj->connType = HEARTBEAT_TYPE_QUERY;
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);
// pRequest->body.resInfo.pRspMsg = pMsg->pData;

1175
source/client/test/clientTests.cpp
View File

File diff suppressed because it is too large Load Diff

29
source/common/src/tep.c
View File

@ -2,32 +2,43 @@
#include "tglobal.h"
#include "tlockfree.h"
int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
*port = 0;
strcpy(fqdn, ep);
int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
pEp->port = 0;
strcpy(pEp->fqdn, ep);
char *temp = strchr(fqdn, ':');
char *temp = strchr(pEp->fqdn, ':');
if (temp) {
*temp = 0;
*port = atoi(temp+1);
pEp->port = atoi(temp+1);
}
if (*port == 0) {
*port = tsServerPort;
if (pEp->port == 0) {
pEp->port = tsServerPort;
return -1;
}
return 0;
}
void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return;
}
int32_t index = pEpSet->numOfEps;
tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn));
pEpSet->eps[index].port = port;
pEpSet->numOfEps += 1;
}
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) {
if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) {
return false;
}
for (int32_t i = 0; i < s1->numOfEps; i++) {
if (s1->port[i] != s2->port[i]
|| strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0)
if (s1->eps[i].port != s2->eps[i].port
|| strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0)
return false;
}
return true;

12
source/common/src/tglobal.c
View File

@ -1080,9 +1080,7 @@ static void doInitGlobalConfig(void) {
void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); }
int32_t taosCheckAndPrintCfg() {
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
SEp ep = {0};
if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
taosSetAllDebugFlag();
}
@ -1097,15 +1095,15 @@ int32_t taosCheckAndPrintCfg() {
if (tsFirst[0] == 0) {
strcpy(tsFirst, tsLocalEp);
} else {
taosGetFqdnPortFromEp(tsFirst, fqdn, &port);
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", fqdn, port);
taosGetFqdnPortFromEp(tsFirst, &ep);
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port);
}
if (tsSecond[0] == 0) {
strcpy(tsSecond, tsLocalEp);
} else {
taosGetFqdnPortFromEp(tsSecond, fqdn, &port);
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
taosGetFqdnPortFromEp(tsSecond, &ep);
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port);
}
taosCheckDataDirCfg();

1
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -289,6 +289,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
.charset = pCfg->charset,
.nthreads = pCfg->numOfCommitThreads,
.putReqToVQueryQFp = dndPutReqToVQueryQ,
.sendReqToDnodeFp = dndSendReqToDnode
};
if (vnodeInit(&vnodeOpt) != 0) {

45
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -57,13 +57,13 @@ void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint
SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
if (pDnodeEp != NULL) {
if (pPort != NULL) {
*pPort = pDnodeEp->port;
*pPort = pDnodeEp->ep.port;
}
if (pFqdn != NULL) {
tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN);
tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
}
if (pEp != NULL) {
snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
}
}
@ -85,12 +85,12 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) {
dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]);
if (strcmp(epSet.fqdn[i], pDnode->cfg.localFqdn) == 0 && epSet.port[i] == pDnode->cfg.serverPort) {
dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) {
epSet.inUse = (i + 1) % epSet.numOfEps;
}
epSet.port[i] = htons(epSet.port[i]);
epSet.eps[i].port = htons(epSet.eps[i].port);
}
rpcSendRedirectRsp(pReq->handle, &epSet);
@ -104,7 +104,7 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
pMgmt->mnodeEpSet = *pEpSet;
for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
taosWUnLockLatch(&pMgmt->latch);
@ -116,7 +116,7 @@ static void dndPrintDnodes(SDnode *pDnode) {
dDebug("print dnode ep list, num:%d", pMgmt->dnodeEps->num);
for (int32_t i = 0; i < pMgmt->dnodeEps->num; i++) {
SDnodeEp *pEp = &pMgmt->dnodeEps->eps[i];
dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode);
dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
}
}
@ -145,8 +145,8 @@ static void dndResetDnodes(SDnode *pDnode, SDnodeEps *pDnodeEps) {
if (!pDnodeEp->isMnode) continue;
if (mIndex >= TSDB_MAX_REPLICA) continue;
pMgmt->mnodeEpSet.numOfEps++;
strcpy(pMgmt->mnodeEpSet.fqdn[mIndex], pDnodeEp->fqdn);
pMgmt->mnodeEpSet.port[mIndex] = pDnodeEp->port;
pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep;
mIndex++;
}
@ -167,7 +167,7 @@ static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) {
SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
if (pDnodeEp != NULL) {
char epstr[TSDB_EP_LEN + 1];
snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
changed = strcmp(pEp, epstr) != 0;
}
@ -251,11 +251,12 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
cJSON *dnodeId = cJSON_GetObjectItem(node, "id");
if (!dnodeId || dnodeId->type != cJSON_Number) {
cJSON *did = cJSON_GetObjectItem(node, "id");
if (!did || did->type != cJSON_Number) {
dError("failed to read %s since dnodeId not found", pMgmt->file);
goto PRASE_DNODE_OVER;
}
pDnodeEp->id = dnodeId->valueint;
cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
@ -263,14 +264,15 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
dError("failed to read %s since dnodeFqdn not found", pMgmt->file);
goto PRASE_DNODE_OVER;
}
tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
tstrncpy(pDnodeEp->ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
if (!dnodePort || dnodePort->type != cJSON_Number) {
dError("failed to read %s since dnodePort not found", pMgmt->file);
goto PRASE_DNODE_OVER;
}
pDnodeEp->port = dnodePort->valueint;
pDnodeEp->ep.port = dnodePort->valueint;
cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
if (!isMnode || isMnode->type != cJSON_Number) {
@ -298,7 +300,8 @@ PRASE_DNODE_OVER:
pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
pMgmt->dnodeEps->num = 1;
pMgmt->dnodeEps->eps[0].isMnode = 1;
taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port);
taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &(pMgmt->dnodeEps->eps[0].ep));
}
dndResetDnodes(pDnode, pMgmt->dnodeEps);
@ -329,8 +332,8 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
for (int32_t i = 0; i < pMgmt->dnodeEps->num; ++i) {
SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id);
len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->fqdn);
len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->port);
len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port);
len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode);
if (i < pMgmt->dnodeEps->num - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
@ -395,7 +398,7 @@ void dndSendStatusReq(SDnode *pDnode) {
static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
if (pMgmt->dnodeId == 0) {
dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
dInfo("set dnodeId:%d clusterId:0x%" PRId64, pCfg->dnodeId, pCfg->clusterId);
taosWLockLatch(&pMgmt->latch);
pMgmt->dnodeId = pCfg->dnodeId;
pMgmt->clusterId = pCfg->clusterId;
@ -450,7 +453,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) {
pDnodeEps->num = htonl(pDnodeEps->num);
for (int32_t i = 0; i < pDnodeEps->num; ++i) {
pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id);
pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port);
pDnodeEps->eps[i].ep.port = htons(pDnodeEps->eps[i].ep.port);
}
dndUpdateDnodeEps(pDnode, pDnodeEps);
@ -529,7 +532,7 @@ int32_t dndInitMgmt(SDnode *pDnode) {
}
if (pMgmt->dropped) {
dError("dnode will not start for its already dropped");
dError("dnode not start since its already dropped");
return -1;
}

2
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -122,6 +122,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg;
@ -148,6 +149,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
}
static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {

8
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "dndVnodes.h"
#include "dndTransport.h"
#include "dndMgmt.h"
typedef struct {
int32_t vgId;
@ -527,7 +528,6 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->vgId = pCreate->vgId;
pCfg->wsize = pCreate->cacheBlockSize;
pCfg->ssize = pCreate->cacheBlockSize;
pCfg->wsize = pCreate->cacheBlockSize;
pCfg->lsize = pCreate->cacheBlockSize;
pCfg->isHeapAllocator = true;
pCfg->ttl = 4;
@ -578,6 +578,12 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
SWrapperCfg wrapperCfg = {0};
dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg);
if (pCreate->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION;
dDebug("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr());
return -1;
}
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist", pCreate->vgId);

7
source/dnode/mgmt/impl/test/sut/src/client.cpp
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tep.h"
#include "sut.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
@ -61,11 +62,7 @@ void TestClient::Cleanup() {
SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) {
SEpSet epSet = {0};
epSet.inUse = 0;
epSet.numOfEps = 1;
epSet.port[0] = port;
memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN);
addEpIntoEpSet(&epSet, fqdn, port);
rpcSendRequest(clientRpc, &epSet, pReq, NULL);
tsem_wait(&sem);

38
source/dnode/mgmt/impl/test/vnode/vnode.cpp
View File

@ -68,6 +68,44 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED);
}
}
{
int32_t contLen = sizeof(SCreateVnodeReq);
SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen);
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(3);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->vgVersion = htonl(1);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9527);
}
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION);
}
}
TEST_F(DndTestVnode, 02_ALTER_Vnode) {

79
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -363,9 +363,7 @@ typedef struct SMqConsumerEp {
int64_t consumerId; // -1 for unassigned
int64_t lastConsumerHbTs;
int64_t lastVgHbTs;
uint32_t qmsgLen;
char* qmsg;
//SSubQueryMsg qExec;
} SMqConsumerEp;
static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
@ -374,9 +372,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon
tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
tlen += taosEncodeFixedI64(buf, pConsumerEp->lastConsumerHbTs);
tlen += taosEncodeFixedI64(buf, pConsumerEp->lastVgHbTs);
//tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen);
tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen);
tlen += taosEncodeString(buf, pConsumerEp->qmsg);
return tlen;
}
@ -385,9 +384,10 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu
buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
buf = taosDecodeFixedI64(buf, &pConsumerEp->lastConsumerHbTs);
buf = taosDecodeFixedI64(buf, &pConsumerEp->lastVgHbTs);
//buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen);
buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen);
buf = taosDecodeString(buf, &pConsumerEp->qmsg);
return buf;
}
@ -423,18 +423,27 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
free(pSub);
return NULL;
}
pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
pSub->lostConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->lostConsumer == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->idleConsumer);
taosArrayDestroy(pSub->assigned);
free(pSub);
return NULL;
}
pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->idleConsumer == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->lostConsumer);
free(pSub);
return NULL;
}
pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
if (pSub->unassignedVg == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->lostConsumer);
taosArrayDestroy(pSub->idleConsumer);
taosArrayDestroy(pSub->unassignedVg);
free(pSub);
return NULL;
}
@ -461,6 +470,13 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb
tlen += tEncodeSMqConsumerEp(buf, pCEp);
}
sz = taosArrayGetSize(pSub->lostConsumer);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp* pCEp = taosArrayGet(pSub->lostConsumer, i);
tlen += tEncodeSMqConsumerEp(buf, pCEp);
}
sz = taosArrayGetSize(pSub->idleConsumer);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
@ -485,20 +501,47 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pSub->assigned = taosArrayInit(sz, sizeof(int64_t));
if (pSub->assigned == NULL) {
pSub->availConsumer = taosArrayInit(sz, sizeof(int64_t));
if (pSub->availConsumer == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
int64_t consumerId;
buf = taosDecodeFixedI64(buf, &consumerId);
taosArrayPush(pSub->assigned, &consumerId);
taosArrayPush(pSub->availConsumer, &consumerId);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->assigned = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer);
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp cEp;
buf = tDecodeSMqConsumerEp(buf, &cEp);
taosArrayPush(pSub->assigned, &cEp);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->lostConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->lostConsumer == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->assigned);
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp cEp;
buf = tDecodeSMqConsumerEp(buf, &cEp);
taosArrayPush(pSub->lostConsumer, &cEp);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->idleConsumer == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->lostConsumer);
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
@ -507,10 +550,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
taosArrayPush(pSub->idleConsumer, &cEp);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedVg == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->lostConsumer);
taosArrayDestroy(pSub->idleConsumer);
return NULL;
}
@ -580,7 +626,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic
int32_t tlen = 0;
tlen += taosEncodeString(buf, pConsumerTopic->name);
tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch);
int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
int32_t sz = 0;
if (pConsumerTopic->pVgInfo != NULL) {
sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
}
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i);

20
source/dnode/mnode/impl/src/mndDb.c
View File

@ -226,10 +226,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1;
@ -498,7 +498,7 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbReq *pAlter) {
static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
@ -649,7 +649,7 @@ static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) {
SDbObj dbObj = {0};
memcpy(&dbObj, pDb, sizeof(SDbObj));
int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter);
int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter);
if (code != 0) {
mndReleaseDb(pMnode, pDb);
mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code));
@ -816,22 +816,22 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis
if (pIter == NULL) break;
if (pVgroup->dbUid == pDb->uid) {
SVgroupInfo *pInfo = &vgList[vindex];
SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex];
pInfo->vgId = htonl(pVgroup->vgId);
pInfo->hashBegin = htonl(pVgroup->hashBegin);
pInfo->hashEnd = htonl(pVgroup->hashEnd);
pInfo->numOfEps = pVgroup->replica;
pInfo->epset.numOfEps = pVgroup->replica;
for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
SEpAddr *pEpArrr = &pInfo->epAddr[gid];
SEp * pEp = &pInfo->epset.eps[gid];
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode != NULL) {
memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
pEpArrr->port = htons(pDnode->port);
memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
pEp->port = htons(pDnode->port);
}
mndReleaseDnode(pMnode, pDnode);
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
pInfo->inUse = gid;
pInfo->epset.inUse = gid;
}
}
vindex++;
@ -1223,7 +1223,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3
prec = TSDB_TIME_PRECISION_NANO_STR;
break;
default:
assert(false);
prec = "none";
break;
}
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);

8
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -203,8 +203,8 @@ void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) {
}
SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) {
SEpSet epSet = {.inUse = 0, .numOfEps = 1, .port[0] = pDnode->port};
memcpy(epSet.fqdn[0], pDnode->fqdn, TSDB_FQDN_LEN);
SEpSet epSet = {0};
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
return epSet;
}
@ -261,8 +261,8 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) {
SDnodeEp *pEp = &pEps->eps[numOfEps];
pEp->id = htonl(pDnode->id);
pEp->port = htons(pDnode->port);
memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
pEp->ep.port = htons(pDnode->port);
memcpy(pEp->ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
pEp->isMnode = 0;
if (mndIsMnode(pMnode, pDnode->id)) {
pEp->isMnode = 1;

5
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -20,6 +20,7 @@
#include "mndTrans.h"
#define SDB_FUNC_VER 1
#define SDB_FUNC_RESERVE_SIZE 64
static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc);
static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw);
@ -60,7 +61,7 @@ void mndCleanupFunc(SMnode *pMnode) {}
static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj);
int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj) + SDB_FUNC_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size);
if (pRaw == NULL) goto FUNC_ENCODE_OVER;
@ -78,6 +79,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) {
SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_ENCODE_OVER)
SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_ENCODE_OVER)
SDB_SET_DATALEN(pRaw, dataPos, FUNC_ENCODE_OVER);
terrno = 0;
@ -131,6 +133,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER)
SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_DECODE_OVER)
terrno = 0;

4
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -237,8 +237,8 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pIter == NULL) break;
if (pObj->pDnode == NULL) break;
pEpSet->port[pEpSet->numOfEps] = htons(pObj->pDnode->port);
memcpy(pEpSet->fqdn[pEpSet->numOfEps], pObj->pDnode->fqdn, TSDB_FQDN_LEN);
pEpSet->eps[pEpSet->numOfEps].port = htons(pObj->pDnode->port);
memcpy(pEpSet->eps[pEpSet->numOfEps].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
if (pObj->role == TAOS_SYNC_STATE_LEADER) {
pEpSet->inUse = pEpSet->numOfEps;
}

48
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -66,13 +66,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->pCont;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
SMqCMGetSubEpRsp rsp;
int64_t consumerId = be64toh(pReq->consumerId);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId);
if (pConsumer == NULL) {
/*terrno = */
terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
return -1;
}
ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
@ -91,9 +91,13 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
int32_t assignedSz = taosArrayGetSize(pSub->assigned);
topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp));
for (int32_t j = 0; j < assignedSz; j++) {
SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j);
if (pCEp->consumerId == consumerId) {
taosArrayPush(pSub->assigned, pCEp);
SMqSubVgEp vgEp = {
.epSet = pCEp->epSet,
.vgId = pCEp->vgId
};
taosArrayPush(topicEp.vgs, &vgEp);
}
}
if (taosArrayGetSize(topicEp.vgs) != 0) {
@ -101,7 +105,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
}
}
int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
void *buf = malloc(tlen);
void *buf = rpcMallocCont(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -161,8 +165,6 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
pReq->sql = strdup(pTopic->sql);
pReq->logicalPlan = strdup(pTopic->logicalPlan);
pReq->physicalPlan = strdup(pTopic->physicalPlan);
pReq->qmsgLen = pCEp->qmsgLen;
/*memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen);*/
pReq->qmsg = strdup(pCEp->qmsg);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq);
void *reqStr = malloc(tlen);
@ -192,7 +194,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
}
mndReleaseTopic(pMnode, pTopic);
/*mndReleaseTopic(pMnode, pTopic);*/
mndTransDrop(pTrans);
}
pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub);
@ -206,12 +208,12 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas
SArray *pArray;
SArray *inner = taosArrayGet(pDag->pSubplans, 0);
SSubplan *plan = taosArrayGetP(inner, 0);
plan->execNode.inUse = 0;
strcpy(plan->execNode.epAddr[0].fqdn, "localhost");
plan->execNode.epAddr[0].port = 6030;
plan->execNode.nodeId = 2;
plan->execNode.numOfEps = 1;
plan->execNode.nodeId = 2;
SEpSet* pEpSet = &plan->execNode.epset;
pEpSet->inUse = 0;
addEpIntoEpSet(pEpSet, "localhost", 6030);
if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) {
return -1;
}
@ -220,20 +222,15 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp CEp;
CEp.status = 0;
CEp.consumerId = -1;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
STaskInfo *pTaskInfo = taosArrayGet(pArray, i);
tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr);
CEp.epSet = pTaskInfo->addr.epset;
/*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1],
* CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/
CEp.vgId = pTaskInfo->addr.nodeId;
CEp.qmsg = strdup(pTaskInfo->msg->msg);
CEp.qmsgLen = strlen(CEp.qmsg) + 1;
printf("abc:\n%s\n", CEp.qmsg);
/*CEp.qmsg = malloc(CEp.qmsgLen);*/
/*if (CEp.qmsg == NULL) {*/
/*return -1;*/
/*}*/
/*memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen);*/
taosArrayPush(unassignedVg, &CEp);
}
@ -257,8 +254,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume
req.sql = pTopic->sql;
req.logicalPlan = pTopic->logicalPlan;
req.physicalPlan = pTopic->physicalPlan;
req.qmsg = strdup(pCEp->qmsg);
req.qmsgLen = strlen(req.qmsg);
req.qmsg = pCEp->qmsg;
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
void *buf = malloc(sizeof(SMsgHead) + tlen);
if (buf == NULL) {
@ -631,14 +627,14 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
if (newSub) taosArrayDestroy(newSub);
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
/*mndReleaseConsumer(pMnode, pConsumer);*/
return -1;
}
if (newSub) taosArrayDestroy(newSub);
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
return 0;
/*mndReleaseConsumer(pMnode, pConsumer);*/
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) {

2
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -237,7 +237,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime;
topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
topicObj.dbUid = pDb->uid;
topicObj.version = 1;
topicObj.sql = strdup(pCreate->sql);

4
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -921,7 +921,7 @@ static int32_t mndProcessTransMsg(SMnodeMsg *pMsg) {
void mndTransPullup(SMnode *pMnode) {
STrans *pTrans = NULL;
void *pIter = NULL;
void * pIter = NULL;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@ -930,4 +930,6 @@ void mndTransPullup(SMnode *pMnode) {
mndTransExecute(pMnode, pTrans);
sdbRelease(pMnode->pSdb, pTrans);
}
sdbWriteFile(pMnode->pSdb);
}

6
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -178,7 +178,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId);
if (pVgroup == NULL) {
if (pVgroup == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
return pVgroup;
@ -424,9 +424,7 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) {
epset.inUse = epset.numOfEps;
}
epset.port[epset.numOfEps] = pDnode->port;
memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN);
epset.numOfEps++;
addEpIntoEpSet(&epset, pDnode->fqdn, pDnode->port);
mndReleaseDnode(pMnode, pDnode);
}

12
source/dnode/mnode/impl/test/db/db.cpp
View File

@ -277,9 +277,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, 0);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1);
EXPECT_EQ(pInfo->inUse, 0);
EXPECT_EQ(pInfo->numOfEps, 1);
SEpAddr* pAddr = &pInfo->epAddr[0];
EXPECT_EQ(pInfo->epset.inUse, 0);
EXPECT_EQ(pInfo->epset.numOfEps, 1);
SEp* pAddr = &pInfo->epset.eps[0];
pAddr->port = htons(pAddr->port);
EXPECT_EQ(pAddr->port, 9030);
EXPECT_STREQ(pAddr->fqdn, "localhost");
@ -293,9 +293,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX);
EXPECT_EQ(pInfo->inUse, 0);
EXPECT_EQ(pInfo->numOfEps, 1);
SEpAddr* pAddr = &pInfo->epAddr[0];
EXPECT_EQ(pInfo->epset.inUse, 0);
EXPECT_EQ(pInfo->epset.numOfEps, 1);
SEp* pAddr = &pInfo->epset.eps[0];
pAddr->port = htons(pAddr->port);
EXPECT_EQ(pAddr->port, 9030);
EXPECT_STREQ(pAddr->fqdn, "localhost");

6
source/dnode/mnode/impl/test/profile/profile.cpp
View File

@ -44,7 +44,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
pRsp->acctId = htonl(pRsp->acctId);
pRsp->clusterId = htobe64(pRsp->clusterId);
pRsp->connId = htonl(pRsp->connId);
pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]);
pRsp->epSet.eps[0].port = htons(pRsp->epSet.eps[0].port);
EXPECT_EQ(pRsp->acctId, 1);
EXPECT_GT(pRsp->clusterId, 0);
@ -53,8 +53,8 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
EXPECT_EQ(pRsp->epSet.inUse, 0);
EXPECT_EQ(pRsp->epSet.numOfEps, 1);
EXPECT_EQ(pRsp->epSet.port[0], 9031);
EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
EXPECT_EQ(pRsp->epSet.eps[0].port, 9031);
EXPECT_STREQ(pRsp->epSet.eps[0].fqdn, "localhost");
connId = pRsp->connId;
}

6
source/dnode/mnode/sdb/src/sdb.c
View File

@ -64,11 +64,7 @@ SSdb *sdbInit(SSdbOpt *pOption) {
void sdbCleanup(SSdb *pSdb) {
mDebug("start to cleanup sdb");
if (pSdb->curVer > pSdb->lastCommitVer) {
mDebug("write sdb file for current ver:%" PRId64 " larger than last commit ver:%" PRId64, pSdb->curVer,
pSdb->lastCommitVer);
sdbWriteFile(pSdb);
}
sdbWriteFile(pSdb);
if (pSdb->currDir != NULL) {
tfree(pSdb->currDir);

15
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -221,7 +221,7 @@ PARSE_SDB_DATA_ERROR:
return code;
}
int32_t sdbWriteFile(SSdb *pSdb) {
static int32_t sdbWriteFileImp(SSdb *pSdb) {
int32_t code = 0;
char tmpfile[PATH_MAX] = {0};
@ -229,7 +229,8 @@ int32_t sdbWriteFile(SSdb *pSdb) {
char curfile[PATH_MAX] = {0};
snprintf(curfile, sizeof(curfile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
mDebug("start to write file:%s", curfile);
mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
pSdb->lastCommitVer);
FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile);
if (fd <= 0) {
@ -323,12 +324,20 @@ int32_t sdbWriteFile(SSdb *pSdb) {
return code;
}
int32_t sdbWriteFile(SSdb *pSdb) {
if (pSdb->curVer == pSdb->lastCommitVer) {
return 0;
}
return sdbWriteFileImp(pSdb);
}
int32_t sdbDeploy(SSdb *pSdb) {
if (sdbRunDeployFp(pSdb) != 0) {
return -1;
}
if (sdbWriteFile(pSdb) != 0) {
if (sdbWriteFileImp(pSdb) != 0) {
return -1;
}

5
source/dnode/vnode/inc/meta.h
View File

@ -37,11 +37,6 @@ typedef struct SMetaCfg {
uint64_t lruSize;
} SMetaCfg;
typedef struct {
uint32_t nCols;
SSchema *pSchema;
} SSchemaWrapper;
typedef struct SMTbCursor SMTbCursor;
typedef struct SMCtbCursor SMCtbCursor;

11
source/dnode/vnode/inc/tq.h
View File

@ -149,11 +149,12 @@ typedef struct STqGroup {
} STqGroup;
typedef struct STqTaskItem {
int8_t status;
int64_t offset;
void* dst;
qTaskInfo_t task;
SSubQueryMsg* pQueryMsg;
int8_t status;
int64_t offset;
void* dst;
qTaskInfo_t task;
STqReadHandle* pReadHandle;
SSubQueryMsg* pQueryMsg;
} STqTaskItem;
// new version

21
source/dnode/vnode/inc/vnode.h
View File

@ -33,6 +33,7 @@ extern "C" {
typedef struct SVnode SVnode;
typedef struct SDnode SDnode;
typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
typedef struct STqCfg {
// TODO
@ -64,17 +65,21 @@ typedef struct {
const char *charset;
uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
PutReqToVQueryQFp putReqToVQueryQFp;
SendReqToDnodeFp sendReqToDnodeFp;
} SVnodeOpt;
typedef struct STqReadHandle {
int64_t ver;
uint64_t tbUid;
SSubmitMsg* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pMeta;
SArray* pColIdList;
int64_t ver;
uint64_t tbUid;
SSubmitMsg* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pVnodeMeta;
SArray* pColIdList; //SArray<int32_t>
int32_t sver;
SSchemaWrapper* pSchemaWrapper;
STSchema* pSchema;
} STqReadHandle;
/* ------------------------ SVnode ------------------------ */

2
source/dnode/vnode/src/inc/vnd.h
View File

@ -55,6 +55,7 @@ typedef struct SVnodeMgr {
// For vnode Mgmt
SDnode* pDnode;
PutReqToVQueryQFp putReqToVQueryQFp;
SendReqToDnodeFp sendReqToDnodeFp;
} SVnodeMgr;
extern SVnodeMgr vnodeMgr;
@ -85,6 +86,7 @@ struct SVnode {
int vnodeScheduleTask(SVnodeTask* task);
int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
// For Log
extern int32_t vDebugFlag;

86
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -20,6 +20,10 @@
#include "tcoding.h"
#include "thash.h"
#define IMPL_WITH_LOCK 1
// #if IMPL_WITH_LOCK
// #endif
typedef struct {
tb_uid_t uid;
int32_t sver;
@ -27,6 +31,9 @@ typedef struct {
} SSchemaKey;
struct SMetaDB {
#if IMPL_WITH_LOCK
pthread_rwlock_t rwlock;
#endif
// DB
DB *pTbDB;
DB *pSchemaDB;
@ -58,6 +65,9 @@ static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
static void metaClearTbCfg(STbCfg *pTbCfg);
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
static void metaDBWLock(SMetaDB *pDB);
static void metaDBRLock(SMetaDB *pDB);
static void metaDBULock(SMetaDB *pDB);
#define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code))
@ -130,8 +140,10 @@ void metaCloseDB(SMeta *pMeta) {
int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
tb_uid_t uid;
char buf[512];
char buf1[512];
void * pBuf;
DBT key, value;
DBT key1, value1;
DBT key2, value2;
SSchema *pSchema = NULL;
if (pTbCfg->type == META_SUPER_TABLE) {
@ -143,19 +155,17 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
{
// save table info
pBuf = buf;
memset(&key, 0, sizeof(key));
memset(&value, 0, sizeof(key));
memset(&key1, 0, sizeof(key1));
memset(&value1, 0, sizeof(key1));
key.data = &uid;
key.size = sizeof(uid);
key1.data = &uid;
key1.size = sizeof(uid);
metaEncodeTbInfo(&pBuf, pTbCfg);
value.data = buf;
value.size = POINTER_DISTANCE(pBuf, buf);
value.app_data = pTbCfg;
pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key, &value, 0);
value1.data = buf;
value1.size = POINTER_DISTANCE(pBuf, buf);
value1.app_data = pTbCfg;
}
// save schema
@ -169,23 +179,28 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
}
if (pSchema) {
pBuf = buf;
memset(&key, 0, sizeof(key));
memset(&value, 0, sizeof(key));
pBuf = buf1;
memset(&key2, 0, sizeof(key2));
memset(&value2, 0, sizeof(key2));
SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0};
key.data = &schemaKey;
key.size = sizeof(schemaKey);
key2.data = &schemaKey;
key2.size = sizeof(schemaKey);
SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema};
metaEncodeSchema(&pBuf, &sw);
value.data = buf;
value.size = POINTER_DISTANCE(pBuf, buf);
pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0);
value2.data = buf1;
value2.size = POINTER_DISTANCE(pBuf, buf1);
}
metaDBWLock(pMeta->pDB);
pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key1, &value1, 0);
if (pSchema) {
pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key2, &value2, 0);
}
metaDBULock(pMeta->pDB);
return 0;
}
@ -234,11 +249,18 @@ static SMetaDB *metaNewDB() {
return NULL;
}
#if IMPL_WITH_LOCK
pthread_rwlock_init(&pDB->rwlock, NULL);
#endif
return pDB;
}
static void metaFreeDB(SMetaDB *pDB) {
if (pDB) {
#if IMPL_WITH_LOCK
pthread_rwlock_destroy(&pDB->rwlock);
#endif
free(pDB);
}
}
@ -467,7 +489,9 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
key.size = sizeof(uid);
// Query
metaDBRLock(pDB);
ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0);
metaDBULock(pDB);
if (ret != 0) {
return NULL;
}
@ -496,7 +520,9 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
key.size = strlen(tbname);
// Query
metaDBRLock(pDB);
ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0);
metaDBULock(pDB);
if (ret != 0) {
return NULL;
}
@ -529,7 +555,9 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
key.size = sizeof(schemaKey);
// Query
metaDBRLock(pDB);
ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0);
metaDBULock(pDB);
if (ret != 0) {
printf("failed to query schema DB since %s================\n", db_strerror(ret));
return NULL;
@ -687,4 +715,22 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
} else {
return 0;
}
}
}
static void metaDBWLock(SMetaDB *pDB) {
#if IMPL_WITH_LOCK
pthread_rwlock_wrlock(&(pDB->rwlock));
#endif
}
static void metaDBRLock(SMetaDB *pDB) {
#if IMPL_WITH_LOCK
pthread_rwlock_rdlock(&(pDB->rwlock));
#endif
}
static void metaDBULock(SMetaDB *pDB) {
#if IMPL_WITH_LOCK
pthread_rwlock_unlock(&(pDB->rwlock));
#endif
}

187
source/dnode/vnode/src/tq/tq.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcompare.h"
#include "tqInt.h"
#include "tqMetaStore.h"
#include "tcompare.h"
// static
// read next version data
@ -484,7 +484,8 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) {
int32_t num = taosArrayGetSize(pConsumer->topics);
int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN);
int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN +
num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN);
if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz);
if (tmpPtr == NULL) {
@ -511,13 +512,13 @@ int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead**
*(int64_t*)ptr = pTopic->committedOffset;
POINTER_SHIFT(ptr, sizeof(int64_t));
}
return 0;
}
const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) {
STqConsumerHandle* pConsumer = *ppConsumer;
const void* ptr = pHead->content;
const void* ptr = pHead->content;
pConsumer->consumerId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
pConsumer->epoch = *(int64_t*)ptr;
@ -668,31 +669,33 @@ int tqItemSSize() {
#endif
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
SRpcMsg rpcMsg;
int64_t reqId = pReq->reqId;
int64_t consumerId = pReq->consumerId;
int64_t reqOffset = pReq->offset;
int64_t fetchOffset = reqOffset;
int64_t blockingTime = pReq->blockingTime;
SMqConsumeReq* pReq = pMsg->pCont;
SRpcMsg rpcMsg;
int64_t reqId = pReq->reqId;
int64_t consumerId = pReq->consumerId;
int64_t reqOffset = pReq->offset;
int64_t fetchOffset = reqOffset;
int64_t blockingTime = pReq->blockingTime;
int rspLen = 0;
int rspLen = 0;
SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 1, .pBlockData = NULL};
STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
int sz = taosArrayGetSize(pConsumer->topics);
ASSERT(pConsumer);
int sz = taosArrayGetSize(pConsumer->topics);
for (int i = 0; i < sz; i++) {
STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i);
//TODO: support multiple topic in one req
// TODO: support multiple topic in one req
if (strcmp(pTopic->topicName, pReq->topic) != 0) {
continue;
}
if (fetchOffset == -1) {
fetchOffset = pTopic->committedOffset + 1;
}
int8_t pos;
int8_t skip = 0;
if (fetchOffset == -1) {
fetchOffset = pTopic->committedOffset + 1;
}
int8_t pos;
int8_t skip = 0;
SWalHead* pHead;
while (1) {
pos = fetchOffset % TQ_BUFFER_SIZE;
@ -726,7 +729,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
qSetStreamInput(task, pCont);
//SArray<SSDataBlock>
// SArray<SSDataBlock>
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
while (1) {
SSDataBlock* pDataBlock;
@ -735,28 +738,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
break;
}
if (pDataBlock != NULL) {
SMqTbData tbData = {
.uid = pDataBlock->info.uid,
.numOfCols = pDataBlock->info.numOfCols,
.numOfRows = pDataBlock->info.rows,
};
for (int i = 0; i < pDataBlock->info.numOfCols; i++) {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t sz = pColData->info.bytes * pDataBlock->info.rows;
SMqColData colData = {
.bytes = pColData->info.bytes,
.colId = pColData->info.colId,
.type = pColData->info.type,
};
memcpy(colData.data, pColData->pData, colData.bytes * pDataBlock->info.rows);
memcpy(&tbData.colData[i], &colData, sz);
}
/*pDataBlock->info.*/
taosArrayPush(pRes, pDataBlock);
} else {
break;
}
}
//TODO copy
rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper;
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
@ -766,6 +754,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
continue;
}
rsp.pBlockData = pRes;
#if 0
pTopic->buffer.output[pos].dst = pRes;
if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) {
pTopic->buffer.firstOffset = pReq->offset;
@ -773,13 +764,20 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) {
pTopic->buffer.lastOffset = pReq->offset;
}
// put output into rsp
SMqConsumeRsp rsp = {
.consumerId = consumerId,
.numOfTopics = 1
};
#endif
}
int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
return -1;
}
void* abuf = buf;
tEncodeSMqConsumeRsp(&abuf, &rsp);
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
rpcSendResponse(pMsg);
return 0;
}
@ -792,6 +790,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
}
strcpy(pConsumer->cgroup, req.cgroup);
pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle));
pConsumer->consumerId = req.newConsumerId;
pConsumer->epoch = 0;
STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1);
if (pTopic == NULL) {
@ -802,6 +802,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
pTopic->sql = strdup(req.sql);
pTopic->logicalPlan = strdup(req.logicalPlan);
pTopic->physicalPlan = strdup(req.physicalPlan);
pTopic->committedOffset = -1;
pTopic->currentOffset = -1;
pTopic->buffer.firstOffset = -1;
pTopic->buffer.lastOffset = -1;
@ -811,9 +813,12 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
pTopic->buffer.output[i].pReadHandle = pReadHandle;
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle);
}
taosArrayPush(pConsumer->topics, pTopic);
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
terrno = TSDB_CODE_SUCCESS;
return 0;
}
@ -823,23 +828,41 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
if (pReadHandle == NULL) {
return NULL;
}
pReadHandle->pMeta = pMeta;
pReadHandle->pVnodeMeta = pMeta;
pReadHandle->pMsg = NULL;
pReadHandle->ver = -1;
pReadHandle->pColIdList = NULL;
pReadHandle->sver = -1;
pReadHandle->pSchema = NULL;
pReadHandle->pSchemaWrapper = NULL;
return pReadHandle;
}
void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) {
pReadHandle->pMsg = pMsg;
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter);
pReadHandle->ver = ver;
memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter));
}
bool tqNextDataBlock(STqReadHandle* pHandle) {
while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) {
if (pHandle->tbUid == pHandle->pBlock->uid) return true;
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
}
if (pHandle->pBlock == NULL) return false;
pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid);
if (pHandle->tbUid == pHandle->pBlock->uid) {
pHandle->pBlock->tid = htonl(pHandle->pBlock->tid);
pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion);
pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen);
pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen);
pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows);
return true;
}
}
return false;
}
@ -854,31 +877,71 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo)
}
SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
int32_t sversion = pHandle->pBlock->sversion;
SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true);
STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion);
SArray* pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData));
/*int32_t sversion = pHandle->pBlock->sversion;*/
// TODO set to real sversion
int32_t sversion = 0;
if (pHandle->sver != sversion) {
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion);
tb_uid_t quid;
STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid);
if (pTbCfg->type == META_CHILD_TABLE) {
quid = pTbCfg->ctbCfg.suid;
} else {
quid = pHandle->pBlock->uid;
}
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true);
pHandle->sver = sversion;
}
STSchema* pTschema = pHandle->pSchema;
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
int32_t numOfRows = pHandle->pBlock->numOfRows;
int32_t numOfCols = pHandle->pSchema->numOfCols;
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
if (pArray == NULL) {
return NULL;
}
SColumnInfoData colInfo;
int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes;
colInfo.pData = malloc(sz);
if (colInfo.pData == NULL) {
return NULL;
int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) {
int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
ASSERT(pColSchema->colId == colId);
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = colId;
colInfo.info.type = pColSchema->type;
colInfo.pData = calloc(1, sz);
if (colInfo.pData == NULL) {
// TODO free
taosArrayDestroy(pArray);
return NULL;
}
taosArrayPush(pArray, &colInfo);
}
SMemRow row;
int32_t kvIdx;
int32_t kvIdx = 0;
tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
for (int i = 0; i < pTschema->numOfCols && kvIdx < pTschema->numOfCols; i++) {
// TODO: filter out unused column
STColumn* pCol = schemaColAt(pTschema, i);
// get all wanted col of that block
for (int32_t i = 0; i < colNumNeed; i++) {
SColumnInfoData* pColData = taosArrayGet(pArray, i);
STColumn* pCol = schemaColAt(pTschema, i);
// TODO
ASSERT(pCol->colId == pColData->info.colId);
void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
// TODO: handle varlen
memcpy(POINTER_SHIFT(colInfo.pData, pCol->offset), val, pCol->bytes);
memcpy(pColData->pData, val, pCol->bytes);
}
}
taosArrayPush(pArray, &colInfo);
return pArray;
}

5
source/dnode/vnode/src/vnd/vnodeMgr.c
View File

@ -26,6 +26,7 @@ int vnodeInit(const SVnodeOpt *pOption) {
vnodeMgr.stop = false;
vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp;
vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp;
// Start commit handers
if (pOption->nthreads > 0) {
@ -96,6 +97,10 @@ int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) {
return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq);
}
void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) {
(*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq);
}
/* ------------------------ STATIC METHODS ------------------------ */
static void* loop(void* arg) {
setThreadName("vnode-commit");

4
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -21,7 +21,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg);
int vnodeQueryOpen(SVnode *pVnode) {
return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode,
(putReqToQueryQFp)vnodePutReqToVQueryQ);
(putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode);
}
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
@ -43,6 +43,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
switch (pMsg->msgType) {
case TDMT_VND_FETCH:
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_FETCH_RSP:
return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_RES_READY:
return qWorkerProcessReadyMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_TASKS_STATUS:

18
source/libs/catalog/src/catalog.c
View File

@ -65,7 +65,6 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S
int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) {
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
ctgDebug("try to get db vgroup from mnode, db:%s", input->db);
@ -216,17 +215,6 @@ int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableN
return TSDB_CODE_SUCCESS;
}
void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
epSet->inUse = 0;
epSet->numOfEps = vgroupInfo->numOfEps;
for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) {
memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i]));
memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i]));
}
}
int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) {
SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName};
char *msg = NULL;
@ -292,7 +280,6 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter,
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen);
@ -308,10 +295,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter,
};
SRpcMsg rpcRsp = {0};
SEpSet epSet;
ctgGenEpSet(&epSet, vgroupInfo);
rpcSendRecv(pTransporter, &epSet, &rpcMsg, &rpcRsp);
rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {

36
source/libs/catalog/test/catalogTests.cpp
View File

@ -195,10 +195,10 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) {
vgInfo.vgId = i + 1;
vgInfo.hashBegin = i * hashUnit;
vgInfo.hashEnd = hashUnit * (i + 1) - 1;
vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1;
vgInfo.inUse = i % vgInfo.numOfEps;
for (int32_t n = 0; n < vgInfo.numOfEps; ++n) {
SEpAddr *addr = &vgInfo.epAddr[n];
vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
vgInfo.epset.inUse = i % vgInfo.epset.numOfEps;
for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) {
SEp *addr = &vgInfo.epset.eps[n];
strcpy(addr->fqdn, "a0");
addr->port = htons(n + 22);
}
@ -229,10 +229,10 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
vg->vgId = htonl(i + 1);
vg->hashBegin = htonl(i * hashUnit);
vg->hashEnd = htonl(hashUnit * (i + 1) - 1);
vg->numOfEps = i % TSDB_MAX_REPLICA + 1;
vg->inUse = i % vg->numOfEps;
for (int32_t n = 0; n < vg->numOfEps; ++n) {
SEpAddr *addr = &vg->epAddr[n];
vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
vg->epset.inUse = i % vg->epset.numOfEps;
for (int32_t n = 0; n < vg->epset.numOfEps; ++n) {
SEp *addr = &vg->epset.eps[n];
strcpy(addr->fqdn, "a0");
addr->port = htons(n + 22);
}
@ -693,7 +693,7 @@ TEST(tableMeta, normalTable) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.numOfEps, 3);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ctgTestSetPrepareTableMeta();
@ -983,7 +983,7 @@ TEST(tableDistVgroup, normalTable) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 8);
ASSERT_EQ(vgInfo->numOfEps, 3);
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
catalogDestroy();
}
@ -1015,7 +1015,7 @@ TEST(tableDistVgroup, childTableCase) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 9);
ASSERT_EQ(vgInfo->numOfEps, 4);
ASSERT_EQ(vgInfo->epset.numOfEps, 4);
catalogDestroy();
}
@ -1046,13 +1046,13 @@ TEST(tableDistVgroup, superTableCase) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 1);
ASSERT_EQ(vgInfo->numOfEps, 1);
ASSERT_EQ(vgInfo->epset.numOfEps, 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1);
ASSERT_EQ(vgInfo->vgId, 2);
ASSERT_EQ(vgInfo->numOfEps, 2);
ASSERT_EQ(vgInfo->epset.numOfEps, 2);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2);
ASSERT_EQ(vgInfo->vgId, 3);
ASSERT_EQ(vgInfo->numOfEps, 3);
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
catalogDestroy();
}
@ -1088,14 +1088,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.numOfEps, 3);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->numOfEps, 3);
ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
taosArrayDestroy(vgList);
ctgTestBuildDBVgroup(&dbVgroup);
@ -1105,14 +1105,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 7);
ASSERT_EQ(vgInfo.numOfEps, 2);
ASSERT_EQ(vgInfo.epset.numOfEps, 2);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->numOfEps, 3);
ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
taosArrayDestroy(vgList);
catalogDestroy();

28
source/libs/executor/src/executorimpl.c
View File

@ -5070,6 +5070,7 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) {
SStreamBlockScanInfo* pInfo = pOperator->info;
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
pBlockInfo->rows = 0;
while (tqNextDataBlock(pInfo->readerHandle)) {
pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
@ -5115,7 +5116,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
tfree(pMsgBody);
}
void processRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle;
assert(pMsg->ahandle != NULL);
@ -5163,14 +5164,9 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
SEpSet epSet = {0};
epSet.numOfEps = pSource->addr.numOfEps;
epSet.port[0] = pSource->addr.epAddr[0].port;
tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0]));
int64_t startTs = taosGetTimestampUs();
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources);
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, pExchangeInfo->current, totalSources);
pMsg->header.vgId = htonl(pSource->addr.nodeId);
pMsg->sId = htobe64(pSource->schedId);
@ -5192,7 +5188,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
pMsgSendInfo->fp = loadRemoteDataCallback;
int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo);
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo);
tsem_wait(&pExchangeInfo->ready);
SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp;
@ -5296,13 +5292,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
pOperator->exec = doLoadRemoteData;
pOperator->pTaskInfo = pTaskInfo;
#if 1
{ // todo refactor
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "EX";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRspMsg;
rpcInit.cfp = qProcessFetchRsp;
rpcInit.sessions = tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)"root";
@ -5316,7 +5313,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
return NULL; // todo
}
}
#endif
return pOperator;
}
@ -5467,6 +5464,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp
return NULL;
}
// todo dynamic set the value of 4096
pInfo->pRes = createOutputBuf_rv(pExprInfo, 4096);
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo);
SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t));
for(int32_t i = 0; i < numOfOutput; ++i) {
@ -5897,7 +5897,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) {
finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset);
pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
return pInfo->pRes;
return (pInfo->pRes->info.rows != 0)? pInfo->pRes:NULL;
}
static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) {
@ -7431,7 +7431,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "SeqTableTagScan";
// pOperator->operatorType = OP_TagScan;
pOperator->operatorType = OP_TagScan;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
@ -8826,14 +8826,14 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) {
}
void doDestroyTask(SExecTaskInfo *pTaskInfo) {
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo);
// taosArrayDestroy(pTaskInfo->summary.queryProfEvents);
// taosHashCleanup(pTaskInfo->summary.operatorProfResults);
tfree(pTaskInfo->sql);
tfree(pTaskInfo->id.str);
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
tfree(pTaskInfo);
}

2
source/libs/parser/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
parser
PRIVATE os util catalog function transport qcom
PRIVATE os util nodes catalog function transport qcom
)
if(${BUILD_TEST})

43
source/libs/parser/inc/astCreateContext.h Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_AST_CREATER_H_
#define _TD_AST_CREATER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "nodes.h"
#include "parser.h"
typedef struct SAstCreateContext {
SParseContext* pQueryCxt;
bool notSupport;
bool valid;
SNode* pRootNode;
} SAstCreateContext;
int32_t createAstCreateContext(const SParseContext* pQueryCxt, SAstCreateContext* pCxt);
int32_t destroyAstCreateContext(SAstCreateContext* pCxt);
void* acquireRaii(SAstCreateContext* pCxt, void* p);
void* releaseRaii(SAstCreateContext* pCxt, void* p);
#ifdef __cplusplus
}
#endif
#endif /*_TD_AST_CREATER_H_*/

47
source/libs/parser/inc/astCreateFuncs.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "nodes.h"
#include "nodesShowStmts.h"
#include "astCreateContext.h"
#include "ttoken.h"
#ifndef _TD_AST_CREATE_FUNCS_H_
#define _TD_AST_CREATE_FUNCS_H_
#ifdef __cplusplus
extern "C" {
#endif
bool checkTableName(const SToken* pTableName);
SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode);
SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList);
SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit);
SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias);
#ifdef __cplusplus
}
#endif
#endif /*_TD_AST_CREATE_FUNCS_H_*/

199
source/libs/parser/inc/new_sql.y Normal file
View File

@ -0,0 +1,199 @@
//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql
//usage: lemon sql.y
%name NewParse
%token_prefix NEW_TK_
%token_type { SToken }
%default_type { SNode* }
%default_destructor { nodesDestroyNode($$); }
%extra_argument { SAstCreateContext* pCxt }
%include {
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdbool.h>
#include "nodes.h"
#include "ttoken.h"
#include "ttokendef.h"
#include "astCreateFuncs.h"
#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno])
}
%syntax_error {
if(TOKEN.z) {
char msg[] = "syntax error near \"%s\"";
int32_t sqlLen = strlen(&TOKEN.z[0]);
if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > pCxt->pQueryCxt->msgLen) {
char tmpstr[128] = {0};
memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1);
sprintf(pCxt->pQueryCxt->pMsg, msg, tmpstr);
} else {
sprintf(pCxt->pQueryCxt->pMsg, msg, &TOKEN.z[0]);
}
} else {
sprintf(pCxt->pQueryCxt->pMsg, "Incomplete SQL statement");
}
pCxt->valid = false;
}
%parse_accept { printf("parsing complete!\n" );}
//%left OR.
//%left AND.
//%right NOT.
%left UNION ALL MINUS EXCEPT INTERSECT.
//%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN.
//%left GT GE LT LE.
//%left BITAND BITOR LSHIFT RSHIFT.
%left NK_PLUS NK_MINUS.
//%left DIVIDE TIMES.
%left NK_STAR NK_SLASH. //REM.
//%left CONCAT.
//%right UMINUS UPLUS BITNOT.
cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); }
cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; }
//////////////////////// value_function /////////////////////////////////
value_function ::= NK_ID NK_LP value_expression NK_RP.
value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP.
//////////////////////// value_expression_primary /////////////////////////////////
value_expression_primary ::= NK_LP value_expression NK_RP.
value_expression_primary ::= nonparenthesized_value_expression_primary.
nonparenthesized_value_expression_primary ::= literal.
// ?
nonparenthesized_value_expression_primary ::= column_reference.
literal ::= NK_LITERAL.
column_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); }
column_reference(A) ::= table_name(B) NK_DOT NK_ID(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); }
//////////////////////// value_expression /////////////////////////////////
value_expression ::= common_value_expression.
common_value_expression ::= numeric_value_expression.
numeric_value_expression ::= numeric_primary.
numeric_value_expression ::= NK_PLUS numeric_primary.
numeric_value_expression ::= NK_MINUS numeric_primary.
numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression.
numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression.
numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression.
numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression.
numeric_primary ::= value_expression_primary.
numeric_primary ::= value_function.
//////////////////////// query_specification /////////////////////////////////
query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D). { PARSER_TRACE; A = createSelectStmt(pCxt, B, C, D); }
%type set_quantifier_opt { bool }
%destructor set_quantifier_opt {}
set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; }
set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; }
set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; }
%type select_list { SNodeList* }
%destructor select_list { nodesDestroyNodeList($$); }
select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; }
select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; }
%type select_sublist { SNodeList* }
%destructor select_sublist { nodesDestroyNodeList($$); }
select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); }
select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; }
select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); }
select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); }
from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; }
//%type table_reference_list { SNodeList* }
//%destructor table_reference_list { nodesDestroyNodeList($$); }
table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; }
//table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, B, C); }
//table_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createRealTableNode(pCxt, ); }
table_reference(A) ::= table_factor(B). { PARSER_TRACE; A = B; }
//table_reference ::= joined_table.
table_factor(A) ::= table_primary(B). { PARSER_TRACE; A = B; }
table_primary(A) ::= table_name(B). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B); }
table_primary(A) ::= db_name(B) NK_DOT table_name(C). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C); }
table_primary ::= derived_table.
derived_table ::= table_subquery.
%type db_name { SToken }
db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; }
%type table_name { SToken }
table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; }
//////////////////////// subquery /////////////////////////////////
subquery ::= NK_LR query_expression NK_RP.
table_subquery ::= subquery.
// query_expression
query_expression(A) ::= with_clause_opt query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). {
PARSER_TRACE;
addOrderByList(pCxt, B, C);
addSlimit(pCxt, B, D);
addLimit(pCxt, B, E);
A = B;
}
// WITH AS
with_clause_opt ::= . {}
query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; }
query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); }
query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; }
query_primary(A) ::= NK_LP query_expression_body(B) order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;}
%type order_by_clause_opt { SNodeList* }
%destructor order_by_clause_opt { nodesDestroyNodeList($$); }
order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; }
order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; }
slimit_clause_opt(A) ::= . { A = NULL; }
slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); }
slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); }
limit_clause_opt(A) ::= . { A = NULL; }
limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); }
limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); }
//////////////////////// sort_specification_list /////////////////////////////////
%type sort_specification_list { SNodeList* }
%destructor sort_specification_list { nodesDestroyNodeList($$); }
sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); }
sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
sort_specification(A) ::= value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); }
%type ordering_specification_opt EOrder
%destructor ordering_specification_opt {}
ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; }
ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; }
ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; }
%type null_ordering_opt ENullOrder
%destructor null_ordering_opt {}
null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; }
null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; }
null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; }

20
source/nodes/src/nodesCode.c → source/libs/parser/inc/parserImpl.h
View File

@ -14,11 +14,23 @@
*/
#include "nodes.h"
#include "parser.h"
int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
#ifndef _TD_AST_CREATE_FUNCS_H_
#define _TD_AST_CREATE_FUNCS_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SQuery {
SNode* pRoot;
} SQuery;
int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery);
#ifdef __cplusplus
}
#endif
int32_t stringToNode(const char* pStr, SNode** pNode) {
}
#endif /*_TD_AST_CREATE_FUNCS_H_*/

40
source/libs/parser/src/astCreateContext.c Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ttoken.h"
#include "astCreateContext.h"
void* acquireRaii(SAstCreateContext* pCxt, void* p) {
if (NULL == p) {
return NULL;
}
return p;
}
void* releaseRaii(SAstCreateContext* pCxt, void* p) {
if (NULL == p) {
return NULL;
}
return p;
}
int32_t createAstCreater(const SParseContext* pQueryCxt, SAstCreateContext* pCxt) {
}
int32_t destroyAstCreater(SAstCreateContext* pCxt) {
}

91
source/libs/parser/src/astCreateFuncs.c Normal file
View File

@ -0,0 +1,91 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "astCreateFuncs.h"
#include "astCreateContext.h"
bool checkTableName(const SToken* pTableName) {
printf("%p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z);
return pTableName->n < TSDB_TABLE_NAME_LEN ? true : false;
}
SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) {
}
SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) {
}
SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) {
}
SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) {
}
SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) {
}
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
}
SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) {
}
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) {
}
SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName) {
SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE);
if (NULL != pDbName) {
printf("DbName %p : %d, %d, %s\n", pDbName, pDbName->type, pDbName->n, pDbName->z);
strncpy(realTable->dbName, pDbName->z, pDbName->n);
}
printf("TableName %p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z);
strncpy(realTable->table.tableName, pTableName->z, pTableName->n);
return acquireRaii(pCxt, realTable);
}
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) {
SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
select->isDistinct = isDistinct;
if (NULL == pProjectionList) {
select->isStar = true;
}
select->pProjectionList = releaseRaii(pCxt, pProjectionList);
printf("pTable = %p, name = %s\n", pTable, ((SRealTableNode*)pTable)->table.tableName);
select->pFromTable = releaseRaii(pCxt, pTable);
return acquireRaii(pCxt, select);
}
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) {
}
SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) {
}
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) {
}

10
source/nodes/src/nodesUtil.c → source/libs/parser/src/astTranslate.c
View File

@ -13,12 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "nodes.h"
// int32_t doTranslate() {
bool isTimeorderQuery(const SNode* pQuery) {
}
bool isTimelineQuery(const SNode* pQuery) {
}
// }

11
source/libs/parser/src/astValidate.c
View File

@ -3644,6 +3644,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
return TSDB_CODE_SUCCESS;
}
//TODO remove it
int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) {
SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList);
@ -3651,21 +3652,17 @@ int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgL
return code;
}
int32_t vgroupNum = taosArrayGetSize(vgroupList);
size_t vgroupNum = taosArrayGetSize(vgroupList);
SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupNum);
SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum);
vgList->numOfVgroups = vgroupNum;
for (int32_t i = 0; i < vgroupNum; ++i) {
SVgroupInfo *vg = taosArrayGet(vgroupList, i);
vgList->vgroups[i].vgId = vg->vgId;
vgList->vgroups[i].numOfEps = vg->numOfEps;
memcpy(vgList->vgroups[i].epAddr, vg->epAddr, sizeof(vgList->vgroups[i].epAddr));
vgList->vgroups[i] = *vg;
}
*pVgList = vgList;
taosArrayDestroy(vgroupList);
return TSDB_CODE_SUCCESS;

8
source/libs/parser/src/dCDAstProcess.c
View File

@ -58,13 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
SVgroupInfo* info = taosArrayGet(array, 0);
pShowReq->head.vgId = htonl(info->vgId);
pEpSet->numOfEps = info->numOfEps;
pEpSet->inUse = info->inUse;
for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
pEpSet->port[i] = info->epAddr[i].port;
}
*pEpSet = info->epset;
*outputLen = sizeof(SVShowTablesReq);
*output = pShowReq;

1718
source/libs/parser/src/new_sql.c Normal file
View File

File diff suppressed because it is too large Load Diff

2
source/libs/parser/src/parser.c
View File

@ -249,7 +249,7 @@ void qDestroyQuery(SQueryNode* pQueryNode) {
return;
}
int32_t type = nodeType(pQueryNode);
int32_t type = queryNodeType(pQueryNode);
if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) {
SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode;
taosArrayDestroy(pModifInfo->pDataBlocks);

136
source/libs/parser/src/parserImpl.c Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "parserImpl.h"
#include "ttoken.h"
#include "astCreateContext.h"
typedef void* (*FMalloc)(size_t);
typedef void (*FFree)(void*);
extern void* NewParseAlloc(FMalloc);
extern void NewParse(void*, int, SToken, void*);
extern void NewParseFree(void*, FFree);
uint32_t toNewTokenId(uint32_t tokenId) {
switch (tokenId) {
case TK_UNION:
return NEW_TK_UNION;
case TK_ALL:
return NEW_TK_ALL;
case TK_MINUS:
return NEW_TK_NK_MINUS;
case TK_PLUS:
return NEW_TK_NK_PLUS;
case TK_STAR:
return NEW_TK_NK_STAR;
case TK_SLASH:
return NEW_TK_NK_SLASH;
case TK_SHOW:
return NEW_TK_SHOW;
case TK_DATABASES:
return NEW_TK_DATABASES;
case TK_ID:
return NEW_TK_NK_ID;
case TK_LP:
return NEW_TK_NK_LP;
case TK_RP:
return NEW_TK_NK_RP;
case TK_COMMA:
return NEW_TK_NK_COMMA;
case TK_DOT:
return NEW_TK_NK_DOT;
case TK_SELECT:
return NEW_TK_SELECT;
case TK_DISTINCT:
return NEW_TK_DISTINCT;
case TK_AS:
return NEW_TK_AS;
case TK_FROM:
return NEW_TK_FROM;
case TK_ORDER:
return NEW_TK_ORDER;
case TK_BY:
return NEW_TK_BY;
case TK_ASC:
return NEW_TK_ASC;
case TK_DESC:
return NEW_TK_DESC;
}
return tokenId;
}
uint32_t getToken(const char* z, uint32_t* tokenId) {
uint32_t n = tGetToken(z, tokenId);
*tokenId = toNewTokenId(*tokenId);
return n;
}
int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) {
SAstCreateContext cxt = { .pQueryCxt = pParseCxt, .valid = true, .pRootNode = NULL };
void *pParser = NewParseAlloc(malloc);
int32_t i = 0;
while (1) {
SToken t0 = {0};
printf("===========================\n");
if (cxt.pQueryCxt->pSql[i] == 0) {
NewParse(pParser, 0, t0, &cxt);
goto abort_parse;
}
printf("input: [%s]\n", cxt.pQueryCxt->pSql + i);
t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type);
t0.z = (char *)(cxt.pQueryCxt->pSql + i);
printf("token %p : %d %d [%s]\n", &t0, t0.type, t0.n, t0.z);
i += t0.n;
switch (t0.type) {
case TK_SPACE:
case TK_COMMENT: {
break;
}
case TK_SEMI: {
NewParse(pParser, 0, t0, &cxt);
goto abort_parse;
}
case TK_QUESTION:
case TK_ILLEGAL: {
snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z);
cxt.valid = false;
goto abort_parse;
}
case TK_HEX:
case TK_OCT:
case TK_BIN: {
snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unsupported token: \"%s\"", t0.z);
cxt.valid = false;
goto abort_parse;
}
default:
NewParse(pParser, t0.type, t0, &cxt);
if (!cxt.valid) {
goto abort_parse;
}
}
}
abort_parse:
NewParseFree(pParser, free);
pQuery->pRoot = cxt.pRootNode;
return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED;
}

42
source/libs/parser/src/parserUtil.c
View File

@ -1426,35 +1426,6 @@ bool isQueryWithLimit(SQueryStmtInfo* pQueryInfo) {
return false;
}
SVgroupsInfo* vgroupInfoClone(SVgroupsInfo *vgroupList) {
if (vgroupList == NULL) {
return NULL;
}
size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups;
SVgroupsInfo* pNew = malloc(size);
if (pNew == NULL) {
return NULL;
}
pNew->numOfVgroups = vgroupList->numOfVgroups;
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
SVgroupMsg* pNewVInfo = &pNew->vgroups[i];
SVgroupMsg* pvInfo = &vgroupList->vgroups[i];
pNewVInfo->vgId = pvInfo->vgId;
pNewVInfo->numOfEps = pvInfo->numOfEps;
for(int32_t j = 0; j < pvInfo->numOfEps; ++j) {
pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port;
tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN);
}
}
return pNew;
}
void* vgroupInfoClear(SVgroupsInfo *vgroupList) {
if (vgroupList == NULL) {
return NULL;
@ -1505,19 +1476,6 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
return p;
}
SVgroupsInfo* vgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) {
assert(pVgroupsInfo != NULL);
size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo);
SVgroupsInfo* pInfo = calloc(1, size);
pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups;
for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) {
memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg));
}
return pInfo;
}
int32_t getNumOfOutput(SFieldInfo* pFieldInfo) {
return pFieldInfo->numOfOutput;
}

2
source/libs/parser/test/CMakeLists.txt
View File

@ -15,5 +15,5 @@ TARGET_INCLUDE_DIRECTORIES(
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner qcom
PUBLIC os util common nodes parser catalog transport gtest function planner qcom
)

27
source/libs/parser/test/mockCatalogService.cpp
View File

@ -15,6 +15,7 @@
#include "mockCatalogService.h"
#include "tep.h"
#include <iomanip>
#include <iostream>
#include <map>
@ -39,7 +40,16 @@ public:
virtual TableBuilder& setVgid(int16_t vgid) {
schema()->vgId = vgid;
meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, };
vgroup.epset.eps[0] = (SEp){"dnode_1", 6030};
vgroup.epset.eps[1] = (SEp){"dnode_2", 6030};
vgroup.epset.eps[2] = (SEp){"dnode_3", 6030};
vgroup.epset.inUse = 0;
vgroup.epset.numOfEps = 3;
meta_->vgs.emplace_back(vgroup);
return *this;
}
@ -112,9 +122,7 @@ public:
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
// todo
vgInfo->vgId = 1;
vgInfo->numOfEps = 1;
vgInfo->epAddr[0].port = 6030;
strcpy(vgInfo->epAddr[0].fqdn, "node1");
addEpIntoEpSet(&vgInfo->epset, "node1", 6030);
return 0;
}
@ -133,9 +141,16 @@ public:
meta_[db][tbname].reset(new MockTableMeta());
meta_[db][tbname]->schema.reset(table.release());
meta_[db][tbname]->schema->uid = id_++;
meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,};
addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030);
addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030);
addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030);
vgroup.epset.inUse = 0;
meta_[db][tbname]->vgs.emplace_back(vgroup);
// super table
meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
meta_[db][stbname]->vgs.emplace_back(vgroup);
}
void showTables() const {

143
source/libs/parser/test/newParserTest.cpp Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include "parserImpl.h"
using namespace std;
using namespace testing;
class NewParserTest : public Test {
protected:
void setDatabase(const string& acctId, const string& db) {
acctId_ = acctId;
db_ = db;
}
void bind(const char* sql) {
reset();
cxt_.acctId = atoi(acctId_.c_str());
cxt_.db = (char*) db_.c_str();
strcpy(sqlBuf_, sql);
cxt_.sqlLen = strlen(sql);
sqlBuf_[cxt_.sqlLen] = '\0';
cxt_.pSql = sqlBuf_;
}
int32_t run() {
int32_t code = doParse(&cxt_, &query_);
if (code != TSDB_CODE_SUCCESS) {
cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl;
return code;
}
cout << nodeType(query_.pRoot) << endl;
if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) {
// SNode* pWhereCond;
// SNodeList* pPartitionByList; // SNode
// SNode* pWindowClause;
// SNodeList* pGroupByList; // SGroupingSetNode
// SNodeList* pOrderByList; // SOrderByExprNode
// SLimitNode limit;
// SLimitNode slimit;
SSelectStmt* select = (SSelectStmt*)query_.pRoot;
string sql("SELECT ");
if (select->isDistinct) {
sql.append("DISTINCT ");
}
if (nullptr == select->pProjectionList) {
sql.append("* ");
} else {
nodeListToSql(select->pProjectionList, sql);
}
sql.append("FROM ");
tableToSql(select->pFromTable, sql);
cout << sql << endl;
}
// char* pStr = NULL;
// int32_t len = 0;
// code = nodesNodeToString(query_.pRoot, &pStr, &len);
// if (code != TSDB_CODE_SUCCESS) {
// cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl;
// return code;
// }
// cout << "node tree:\n" << pStr << endl;
return TSDB_CODE_SUCCESS;
}
private:
static const int max_err_len = 1024;
static const int max_sql_len = 1024 * 1024;
void tableToSql(const SNode* node, string& sql) {
const STableNode* table = (const STableNode*)node;
cout << "node : " << nodeType(node) << endl;
switch (nodeType(node)) {
case QUERY_NODE_REAL_TABLE: {
SRealTableNode* realTable = (SRealTableNode*)table;
if ('\0' != realTable->dbName[0]) {
sql.append(realTable->dbName);
sql.append(".");
}
sql.append(realTable->table.tableName);
break;
}
default:
break;
}
}
void nodeListToSql(const SNodeList* nodelist, string& sql, const string& seq = ",") {
SNode* node = nullptr;
bool firstNode = true;
FOREACH(node, nodelist) {
if (!firstNode) {
sql.append(", ");
}
switch (nodeType(node)) {
case QUERY_NODE_COLUMN:
sql.append(((SColumnNode*)node)->colName);
break;
}
}
}
void reset() {
memset(&cxt_, 0, sizeof(cxt_));
memset(errMagBuf_, 0, max_err_len);
cxt_.pMsg = errMagBuf_;
cxt_.msgLen = max_err_len;
}
string acctId_;
string db_;
char errMagBuf_[max_err_len];
char sqlBuf_[max_sql_len];
SParseContext cxt_;
SQuery query_;
};
// SELECT * FROM t1
TEST_F(NewParserTest, selectStar) {
setDatabase("root", "test");
bind("SELECT * FROM t1");
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
bind("SELECT * FROM test.t1");
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
}

4
source/libs/planner/src/logicPlan.c
View File

@ -74,7 +74,7 @@ int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryP
}
int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) {
switch (nodeType(pNode)) {
switch (queryNodeType(pNode)) {
case TSDB_SQL_SELECT: {
return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan);
}
@ -395,7 +395,7 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) {
}
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
int32_t type = nodeType(pQueryNode);
int32_t type = queryNodeType(pQueryNode);
if (type == QNODE_MODIFY) {
SDataPayloadInfo* pInfo = pQueryNode->pExtInfo;

37
source/libs/planner/src/physicalPlan.c
View File

@ -251,24 +251,9 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
return subplan;
}
static void vgroupInfoToEpSet(const SVgroupInfo* vg, SQueryNodeAddr* execNode) {
execNode->nodeId = vg->vgId;
execNode->inUse = vg->inUse;
execNode->numOfEps = vg->numOfEps;
for (int8_t i = 0; i < vg->numOfEps; ++i) {
execNode->epAddr[i] = vg->epAddr[i];
}
return;
}
static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) {
execNode->nodeId = vg->vgId;
execNode->inUse = 0; // todo
execNode->numOfEps = vg->numOfEps;
for (int8_t i = 0; i < vg->numOfEps; ++i) {
execNode->epAddr[i] = vg->epAddr[i];
}
return;
static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) {
pNodeAddr->nodeId = vg->vgId;
pNodeAddr->epset = vg->epset;
}
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) {
@ -277,7 +262,8 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod
STORE_CURRENT_SUBPLAN(pCxt);
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
subplan->msgType = TDMT_VND_QUERY;
vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode);
vgroupInfoToNodeAddr(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode);
subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo);
subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode);
RECOVERY_CURRENT_SUBPLAN(pCxt);
@ -297,11 +283,12 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) {
return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType);
}
static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, SSubplan* subplan) {
vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[0]), &subplan->execNode);
// TODO: the SVgroupInfo index
static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) {
SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList;
vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode);
int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan;
return createUserTableScanNode(pPlanNode, pTable, type);
return createUserTableScanNode(pPlanNode, pTableInfo, type);
}
static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
@ -374,7 +361,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY);
SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i);
vgroupInfoToEpSet(&blocks->vg, &subplan->execNode);
subplan->execNode.epset = blocks->vg.epset;
subplan->pDataSink = createDataInserter(pCxt, blocks, NULL);
subplan->pNode = NULL;
subplan->type = QUERY_TYPE_MODIFY;
@ -461,7 +448,7 @@ static void destroyDataSinkNode(SDataSink* pSinkNode) {
return;
}
if (nodeType(pSinkNode) == DSINK_Dispatch) {
if (queryNodeType(pSinkNode) == DSINK_Dispatch) {
SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode;
tfree(pDdSink->sink.schema.pSchema);
}

14
source/libs/planner/src/physicalPlanJson.c
View File

@ -736,7 +736,7 @@ static const char* jkEpAddrFqdn = "Fqdn";
static const char* jkEpAddrPort = "Port";
static bool epAddrToJson(const void* obj, cJSON* json) {
const SEpAddr* ep = (const SEpAddr*)obj;
const SEp* ep = (const SEp*)obj;
bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
if (res) {
res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port);
@ -745,7 +745,7 @@ static bool epAddrToJson(const void* obj, cJSON* json) {
}
static bool epAddrFromJson(const cJSON* json, void* obj) {
SEpAddr* ep = (SEpAddr*)obj;
SEp* ep = (SEp*)obj;
copyString(json, jkEpAddrFqdn, ep->fqdn);
ep->port = getNumber(json, jkEpAddrPort);
return true;
@ -763,11 +763,11 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) {
bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
if (res) {
res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->inUse);
res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse);
}
if (res) {
res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epAddr, sizeof(SEpAddr), pAddr->numOfEps);
res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps);
}
return res;
}
@ -776,11 +776,11 @@ static bool queryNodeAddrFromJson(const cJSON* json, void* obj) {
SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj;
pAddr->nodeId = getNumber(json, jkNodeAddrId);
pAddr->inUse = getNumber(json, jkNodeAddrInUse);
pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse);
int32_t numOfEps = 0;
bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epAddr, sizeof(SEpAddr), &numOfEps);
pAddr->numOfEps = numOfEps;
bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps);
pAddr->epset.numOfEps = numOfEps;
return res;
}

6
source/libs/planner/test/phyPlanTests.cpp
View File

@ -124,12 +124,10 @@ private:
}
void copyStorageMeta(SVgroupsInfo** dst, const std::vector<SVgroupInfo>& src) {
*dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size());
*dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * src.size());
(*dst)->numOfVgroups = src.size();
for (int32_t i = 0; i < src.size(); ++i) {
(*dst)->vgroups[i].vgId = src[i].vgId;
(*dst)->vgroups[i].numOfEps = src[i].numOfEps;
memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps);
(*dst)->vgroups[i] = src[i];
}
}

4
source/libs/qcom/src/querymsg.c
View File

@ -127,8 +127,8 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin);
pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd);
for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
pRsp->vgroupInfo[i].epAddr[n].port = ntohs(pRsp->vgroupInfo[i].epAddr[n].port);
for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) {
pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port);
}
if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) {

1
source/libs/qworker/inc/qworkerInt.h
View File

@ -137,6 +137,7 @@ typedef struct SQWorkerMgmt {
SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx
void *nodeObj;
putReqToQueryQFp putToQueueFp;
sendReqToDnodeFp sendReqFp;
} SQWorkerMgmt;
#define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId

12
source/libs/qworker/src/qworker.c
View File

@ -1100,8 +1100,6 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
atomic_store_8(&ctx->queryInQueue, 0);
atomic_store_8(&ctx->queryContinue, 0);
DataSinkHandle sinkHandle = ctx->sinkHandle;
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd));
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
@ -1313,12 +1311,13 @@ _return:
QW_RET(code);
}
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) {
if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) {
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
putReqToQueryQFp fp1, sendReqToDnodeFp fp2) {
if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) {
qError("invalid param to init qworker");
QW_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt));
if (NULL == mgmt) {
qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt));
@ -1361,7 +1360,8 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW
mgmt->nodeType = nodeType;
mgmt->nodeId = nodeId;
mgmt->nodeObj = nodeObj;
mgmt->putToQueueFp = fp;
mgmt->putToQueueFp = fp1;
mgmt->sendReqFp = fp2;
*qWorkerMgmt = mgmt;

5
source/libs/qworker/src/qworkerMsg.c
View File

@ -421,6 +421,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
qProcessFetchRsp(NULL, pMsg, NULL);
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;

6
source/libs/qworker/test/qworkerTests.cpp
View File

@ -1081,7 +1081,7 @@ TEST(rcTest, shortExecshortDelay) {
qwtTestStop = false;
qwtTestQuitThreadNum = 0;
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL);
ASSERT_EQ(code, 0);
qwtTestMaxExecTaskUsec = 0;
@ -1162,7 +1162,7 @@ TEST(rcTest, longExecshortDelay) {
qwtTestStop = false;
qwtTestQuitThreadNum = 0;
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL);
ASSERT_EQ(code, 0);
qwtTestMaxExecTaskUsec = 1000000;
@ -1245,7 +1245,7 @@ TEST(rcTest, shortExeclongDelay) {
qwtTestStop = false;
qwtTestQuitThreadNum = 0;
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL);
ASSERT_EQ(code, 0);
qwtTestMaxExecTaskUsec = 0;

26
source/libs/scheduler/src/scheduler.c
View File

@ -417,13 +417,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
if (pTask->plan->execNode.numOfEps > 0) {
if (pTask->plan->execNode.epset.numOfEps > 0) {
if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) {
SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.numOfEps);
SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps);
return TSDB_CODE_SUCCESS;
}
@ -446,7 +446,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
}
if (addNum <= 0) {
SCH_TASK_ELOG("no available execNode as candidate addr, nodeNum:%d", nodeNum);
SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum);
return TSDB_CODE_QRY_INVALID_INPUT;
}
@ -1050,31 +1050,19 @@ _return:
SCH_RET(code);
}
void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) {
epSet->inUse = addr->inUse;
epSet->numOfEps = addr->numOfEps;
for (int8_t i = 0; i < epSet->numOfEps; ++i) {
strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn));
epSet->port[i] = addr->epAddr[i].port;
}
}
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) {
uint32_t msgSize = 0;
void *msg = NULL;
int32_t code = 0;
bool isCandidateAddr = false;
SEpSet epSet;
if (NULL == addr) {
addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx));
isCandidateAddr = true;
}
schConvertAddrToEpSet(addr, &epSet);
SEpSet epSet = addr->epset;
switch (msgType) {
case TDMT_VND_CREATE_TABLE:
case TDMT_VND_SUBMIT: {
@ -1218,8 +1206,6 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) {
SCH_TASK_ELOG("subplanToString error, code:%x, msg:%p, len:%d", code, pTask->msg, pTask->msgLen);
SCH_ERR_JRET(code);
}
// printf("physical plan:%s\n", pTask->msg);
}
SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask));
@ -1300,7 +1286,7 @@ void schDropJobAllTasks(SSchJob *pJob) {
int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) {
qDebug("QID:0x%"PRIx64" job started", pDag->queryId);
if (pNodeList && taosArrayGetSize(pNodeList) <= 0) {
if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) {
qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId);
}

40
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -29,7 +29,6 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wliteral-suffix"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
@ -92,11 +91,11 @@ void schtBuildQueryDag(SQueryDag *dag) {
scanPlan->id.templateId = 0x0000000000000002;
scanPlan->id.subplanId = 0x0000000000000003;
scanPlan->type = QUERY_TYPE_SCAN;
scanPlan->execNode.numOfEps = 1;
scanPlan->execNode.nodeId = 1;
scanPlan->execNode.inUse = 0;
scanPlan->execNode.epAddr[0].port = 6030;
strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0");
scanPlan->execNode.epset.inUse = 0;
addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030);
scanPlan->pChildren = NULL;
scanPlan->level = 1;
scanPlan->pParents = taosArrayInit(1, POINTER_BYTES);
@ -108,7 +107,8 @@ void schtBuildQueryDag(SQueryDag *dag) {
mergePlan->id.subplanId = 0x5555555555;
mergePlan->type = QUERY_TYPE_MERGE;
mergePlan->level = 0;
mergePlan->execNode.numOfEps = 0;
mergePlan->execNode.epset.numOfEps = 0;
mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES);
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
@ -144,11 +144,11 @@ void schtBuildInsertDag(SQueryDag *dag) {
insertPlan[0].id.subplanId = 0x0000000000000004;
insertPlan[0].type = QUERY_TYPE_MODIFY;
insertPlan[0].level = 0;
insertPlan[0].execNode.numOfEps = 1;
insertPlan[0].execNode.nodeId = 1;
insertPlan[0].execNode.inUse = 0;
insertPlan[0].execNode.epAddr[0].port = 6030;
strcpy(insertPlan[0].execNode.epAddr[0].fqdn, "ep0");
insertPlan[0].execNode.epset.inUse = 0;
addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030);
insertPlan[0].pChildren = NULL;
insertPlan[0].pParents = NULL;
insertPlan[0].pNode = NULL;
@ -160,11 +160,11 @@ void schtBuildInsertDag(SQueryDag *dag) {
insertPlan[1].id.subplanId = 0x0000000000000005;
insertPlan[1].type = QUERY_TYPE_MODIFY;
insertPlan[1].level = 0;
insertPlan[1].execNode.numOfEps = 1;
insertPlan[1].execNode.nodeId = 1;
insertPlan[1].execNode.inUse = 1;
insertPlan[1].execNode.epAddr[0].port = 6030;
strcpy(insertPlan[1].execNode.epAddr[0].fqdn, "ep1");
insertPlan[1].execNode.epset.inUse = 0;
addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030);
insertPlan[1].pChildren = NULL;
insertPlan[1].pParents = NULL;
insertPlan[1].pNode = NULL;
@ -371,9 +371,9 @@ void* schtRunJobThread(void *aa) {
while (!schtTestStop) {
schtBuildQueryDag(&dag);
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEpAddr qnodeAddr = {0};
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
@ -523,9 +523,9 @@ TEST(queryTest, normalCase) {
schtInitLogFile();
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEpAddr qnodeAddr = {0};
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
@ -627,9 +627,9 @@ TEST(insertTest, normalCase) {
schtInitLogFile();
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEpAddr qnodeAddr = {0};
SEp qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);

4
source/libs/transport/CMakeLists.txt
View File

@ -13,7 +13,7 @@ target_link_libraries(
PUBLIC util
PUBLIC common
)
if (${BUILD_WITH_UV})
if (${BUILD_WITH_UV_TRANS})
target_include_directories(
transport
PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include"
@ -25,7 +25,7 @@ if (${BUILD_WITH_UV})
PUBLIC uv_a
)
add_definitions(-DUSE_UV)
endif(${BUILD_WITH_UV})
endif(${BUILD_WITH_UV_TRANS})
if (${BUILD_TEST})
add_subdirectory(test)

7
source/libs/transport/inc/transComm.h
View File

@ -202,6 +202,8 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen);
void transConnCtxDestroy(STransConnCtx* ctx);
void transFreeMsg(void* msg);
//
typedef struct SConnBuffer {
char* buf;
int len;
@ -209,4 +211,9 @@ typedef struct SConnBuffer {
int left;
} SConnBuffer;
int transInitBuffer(SConnBuffer* buf);
int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf);
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
#endif

12
source/libs/transport/src/rpcMain.c
View File

@ -814,9 +814,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
SEpSet * pEpSet = &pContext->epSet;
pConn =
rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType);
if (pConn == NULL || pConn->user[0] == 0) {
pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
pConn = rpcOpenConn(pRpc, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType);
}
if (pConn) {
@ -1188,7 +1188,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
// for UDP, port may be changed by server, the port in epSet shall be used for cache
if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse],
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.eps[pContext->epSet.inUse].port,
pConn->connType);
} else {
rpcCloseConn(pConn);
@ -1202,9 +1202,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps,
pContext->epSet.inUse);
for (int i = 0; i < pContext->epSet.numOfEps; ++i) {
pContext->epSet.port[i] = htons(pContext->epSet.port[i]);
tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.fqdn[i],
pContext->epSet.port[i]);
pContext->epSet.eps[i].port = htons(pContext->epSet.eps[i].port);
tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.eps[i].fqdn,
pContext->epSet.eps[i].port);
}
}
rpcSendReqToServer(pRpc, pContext);

11
source/libs/transport/src/trans.c
View File

@ -30,7 +30,8 @@ void* rpcOpen(const SRpcInit* pInit) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
}
pRpc->cfp = pInit->cfp;
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
// pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->numOfThreads = pInit->numOfThreads;
pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
@ -55,7 +56,13 @@ void* rpcMallocCont(int contLen) {
}
return start + sizeof(STransMsgHead);
}
void rpcFreeCont(void* cont) { return; }
void rpcFreeCont(void* cont) {
// impl
if (cont == NULL) {
return;
}
free((char*)cont - TRANS_MSG_OVERHEAD);
}
void* rpcReallocCont(void* ptr, int contLen) { return NULL; }
void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {}

159
source/libs/transport/src/transCli.c
View File

@ -31,6 +31,7 @@ typedef struct SCliConn {
char secured;
uint64_t expireTime;
int8_t notifyCount; // timers already notify to client
int32_t ref;
} SCliConn;
typedef struct SCliMsg {
@ -94,8 +95,10 @@ static void clientHandleExcept(SCliConn* conn);
// handle req from app
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void clientMsgDestroy(SCliMsg* pMsg);
static void destroyTransConnCtx(STransConnCtx* ctx);
static void destroyUserdata(SRpcMsg* userdata);
static void destroyCmsg(SCliMsg* cmsg);
static void transDestroyConnCtx(STransConnCtx* ctx);
// thread obj
static SCliThrdObj* createThrdObj();
static void destroyThrdObj(SCliThrdObj* pThrd);
@ -103,7 +106,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd);
static void* clientThread(void* arg);
static void clientHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx;
SCliMsg* pMsg = conn->data;
STransConnCtx* pCtx = pMsg->ctx;
SRpcInfo* pRpc = pCtx->pTransInst;
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
@ -112,41 +116,53 @@ static void clientHandleResp(SCliConn* conn) {
SRpcMsg rpcMsg;
rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = transContFromHead(pHead);
rpcMsg.pCont = transContFromHead((char*)pHead);
rpcMsg.code = pHead->code;
rpcMsg.msgType = pHead->msgType;
rpcMsg.ahandle = pCtx->ahandle;
tDebug("conn %p handle resp", conn);
(pRpc->cfp)(NULL, &rpcMsg, NULL);
conn->notifyCount += 1;
// buf's mem alread translated to rpcMsg.pCont
transClearBuffer(&conn->readBuf);
uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb);
SCliThrdObj* pThrd = conn->hostThrd;
tfree(conn->data);
addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
destroyCmsg(pMsg);
conn->data = NULL;
// start thread's timer of conn pool if not active
if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) {
uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
destroyTransConnCtx(pCtx);
}
static void clientHandleExcept(SCliConn* pConn) {
if (pConn->data == NULL) {
// handle conn except in conn pool
clientConnDestroy(pConn, true);
return;
}
tDebug("conn %p start to destroy", pConn);
SCliMsg* pMsg = pConn->data;
STransConnCtx* pCtx = pMsg->ctx;
SRpcInfo* pRpc = pCtx->pTransInst;
destroyUserdata(&pMsg->msg);
transFreeMsg((pMsg->msg.pCont));
pMsg->msg.pCont = NULL;
STransConnCtx* pCtx = pMsg->ctx;
SRpcMsg rpcMsg = {0};
rpcMsg.ahandle = pCtx->ahandle;
rpcMsg.code = -1;
// SRpcInfo* pRpc = pMsg->ctx->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
tfree(pConn->data);
(pCtx->pTransInst->cfp)(NULL, &rpcMsg, NULL);
pConn->notifyCount += 1;
destroyTransConnCtx(pCtx);
destroyCmsg(pMsg);
pConn->data = NULL;
// transDestroyConnCtx(pCtx);
clientConnDestroy(pConn, true);
}
@ -213,14 +229,20 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
}
queue* h = QUEUE_HEAD(&plist->conn);
QUEUE_REMOVE(h);
return QUEUE_DATA(h, SCliConn, conn);
SCliConn* conn = QUEUE_DATA(h, SCliConn, conn);
QUEUE_INIT(&conn->conn);
return conn;
}
static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
char key[128] = {0};
tstrncpy(key, ip, strlen(ip));
tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
tDebug("conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap);
SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
conn->notifyCount = 0;
@ -237,40 +259,18 @@ static bool clientReadComplete(SConnBuffer* data) {
if (msgLen > data->len) {
data->left = msgLen - data->len;
return false;
} else {
} else if (msgLen == data->len) {
data->left = 0;
return true;
}
} else {
return false;
}
}
static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
// impl later
static const int CAPACITY = 512;
static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
if (pBuf->cap == 0) {
pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char));
pBuf->len = 0;
pBuf->cap = CAPACITY;
pBuf->left = -1;
buf->base = pBuf->buf;
buf->len = CAPACITY;
} else {
if (pBuf->len >= pBuf->cap) {
if (pBuf->left == -1) {
pBuf->cap *= 2;
pBuf->buf = realloc(pBuf->buf, pBuf->cap);
} else if (pBuf->len + pBuf->left > pBuf->cap) {
pBuf->cap = pBuf->len + pBuf->left;
pBuf->buf = realloc(pBuf->buf, pBuf->cap);
}
}
buf->base = pBuf->buf + pBuf->len;
buf->len = pBuf->cap - pBuf->len;
}
transAllocBuffer(pBuf, buf);
}
static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
// impl later
@ -279,6 +279,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
if (nread > 0) {
pBuf->len += nread;
if (clientReadComplete(pBuf)) {
uv_read_stop((uv_stream_t*)conn->stream);
tDebug("conn %p read complete", conn);
clientHandleResp(conn);
} else {
@ -288,10 +289,12 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
}
assert(nread <= 0);
if (nread == 0) {
tError("conn %p closed", conn);
// ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb
// nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under
// read(2).
return;
}
if (nread < 0) {
if (nread < 0 || nread == UV_EOF) {
tError("conn %p read error: %s", conn, uv_err_name(nread));
clientHandleExcept(conn);
}
@ -300,43 +303,47 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
}
static void clientConnDestroy(SCliConn* conn, bool clear) {
tDebug("conn %p destroy", conn);
if (clear) {
uv_close((uv_handle_t*)conn->stream, NULL);
//
conn->ref--;
if (conn->ref == 0) {
tDebug("conn %p remove from conn pool", conn);
QUEUE_REMOVE(&conn->conn);
tDebug("conn %p remove from conn pool successfully", conn);
if (clear) {
uv_close((uv_handle_t*)conn->stream, clientDestroy);
}
}
free(conn->stream);
free(conn->readBuf.buf);
free(conn->writeReq);
free(conn);
}
static void clientDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data;
// QUEUE_REMOVE(&conn->conn);
clientConnDestroy(conn, false);
// transDestroyBuffer(&conn->readBuf);
free(conn->stream);
free(conn->writeReq);
tDebug("conn %p destroy successfully", conn);
free(conn);
// clientConnDestroy(conn, false);
}
static void clientWriteCb(uv_write_t* req, int status) {
SCliConn* pConn = req->data;
SCliMsg* pMsg = pConn->data;
transFreeMsg((pMsg->msg.pCont));
pMsg->msg.pCont = NULL;
if (status == 0) {
tDebug("conn %p data already was written out", pConn);
SCliMsg* pMsg = pConn->data;
if (pMsg == NULL) {
destroy
// handle
return;
}
destroyUserdata(&pMsg->msg);
} else {
tError("conn %p failed to write: %s", pConn, uv_err_name(status));
clientHandleExcept(pConn);
return;
}
SCliThrdObj* pThrd = pConn->hostThrd;
// if (pConn->stream == NULL) {
// pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
// uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream);
// pConn->stream->data = pConn;
//}
uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb);
// impl later
uv_read_start((uv_stream_t*)pConn->stream, clientAllocBufferCb, clientReadCb);
}
static void clientWrite(SCliConn* pConn) {
@ -381,14 +388,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
tDebug("conn %p get from conn pool", conn);
conn->data = pMsg;
conn->writeReq->data = conn;
conn->readBuf.len = 0;
memset(conn->readBuf.buf, 0, conn->readBuf.cap);
conn->readBuf.left = -1;
transDestroyBuffer(&conn->readBuf);
clientWrite(conn);
} else {
SCliConn* conn = calloc(1, sizeof(SCliConn));
conn->ref++;
// read/write stream handle
conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream));
@ -397,6 +401,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
// write req handle
conn->writeReq = malloc(sizeof(uv_write_t));
conn->writeReq->data = conn;
QUEUE_INIT(&conn->conn);
conn->connReq.data = conn;
@ -459,8 +464,20 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
}
return cli;
}
static void clientMsgDestroy(SCliMsg* pMsg) {
// impl later
static void destroyUserdata(SRpcMsg* userdata) {
if (userdata->pCont == NULL) {
return;
}
transFreeMsg(userdata->pCont);
userdata->pCont = NULL;
}
static void destroyCmsg(SCliMsg* pMsg) {
if (pMsg == NULL) {
return;
}
transDestroyConnCtx(pMsg->ctx);
destroyUserdata(&pMsg->msg);
free(pMsg);
}
static SCliThrdObj* createThrdObj() {
@ -493,7 +510,7 @@ static void destroyThrdObj(SCliThrdObj* pThrd) {
free(pThrd);
}
static void destroyTransConnCtx(STransConnCtx* ctx) {
static void transDestroyConnCtx(STransConnCtx* ctx) {
if (ctx != NULL) {
free(ctx->ip);
}

47
source/libs/transport/src/transComm.c
View File

@ -198,4 +198,51 @@ void transFreeMsg(void* msg) {
}
free((char*)msg - sizeof(STransMsgHead));
}
int transInitBuffer(SConnBuffer* buf) {
transClearBuffer(buf);
return 0;
}
int transClearBuffer(SConnBuffer* buf) {
memset(buf, 0, sizeof(*buf));
return 0;
}
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
/*
* formate of data buffer:
* |<--------------------------data from socket------------------------------->|
* |<------STransMsgHead------->|<-------------------other data--------------->|
*/
static const int CAPACITY = 1024;
SConnBuffer* p = connBuf;
if (p->cap == 0) {
p->buf = (char*)calloc(CAPACITY, sizeof(char));
p->len = 0;
p->cap = CAPACITY;
p->left = -1;
uvBuf->base = p->buf;
uvBuf->len = CAPACITY;
} else {
if (p->len >= p->cap) {
if (p->left == -1) {
p->cap *= 2;
p->buf = realloc(p->buf, p->cap);
} else if (p->len + p->left > p->cap) {
p->cap = p->len + p->left;
p->buf = realloc(p->buf, p->len + p->left);
}
}
uvBuf->base = p->buf + p->len;
uvBuf->len = p->cap - p->len;
}
return 0;
}
int transDestroyBuffer(SConnBuffer* buf) {
if (buf->cap > 0) {
tfree(buf->buf);
}
transClearBuffer(buf);
}
#endif

215
source/libs/transport/src/transSrv.c
View File

@ -17,7 +17,7 @@
#include "transComm.h"
typedef struct SConn {
typedef struct SSrvConn {
uv_tcp_t* pTcp;
uv_write_t* pWriter;
uv_timer_t* pTimer;
@ -26,13 +26,14 @@ typedef struct SConn {
queue queue;
int ref;
int persist; // persist connection or not
SConnBuffer connBuf; // read buf,
SConnBuffer readBuf; // read buf,
int inType;
void* pTransInst; // rpc init
void* ahandle; //
void* hostThrd;
void* pSrvMsg;
SRpcMsg sendMsg;
// SRpcMsg sendMsg;
// del later
char secured;
int spi;
@ -40,7 +41,13 @@ typedef struct SConn {
char user[TSDB_UNI_LEN]; // user ID for the link
char secret[TSDB_PASSWORD_LEN];
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
} SConn;
} SSrvConn;
typedef struct SSrvMsg {
SSrvConn* pConn;
SRpcMsg msg;
queue q;
} SSrvMsg;
typedef struct SWorkThrdObj {
pthread_t thread;
@ -48,8 +55,8 @@ typedef struct SWorkThrdObj {
int fd;
uv_loop_t* loop;
uv_async_t* workerAsync; //
queue conn;
pthread_mutex_t connMtx;
queue msg;
pthread_mutex_t msgMtx;
void* pTransInst;
} SWorkThrdObj;
@ -68,9 +75,9 @@ typedef struct SServerObj {
static const char* notify = "a";
// refactor later
static int transAddAuthPart(SConn* pConn, char* msg, int msgLen);
static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen);
static int uvAuthMsg(SConn* pConn, char* msg, int msgLen);
static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen);
static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
@ -82,12 +89,13 @@ static void uvOnAcceptCb(uv_stream_t* stream, int status);
static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf);
static void uvWorkerAsyncCb(uv_async_t* handle);
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb);
static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb);
static void uvStartSendResp(SSrvMsg* msg);
static void destroySmsg(SSrvMsg* smsg);
// check whether already read complete packet
static bool readComplete(SConnBuffer* buf);
static SConn* createConn();
static void destroyConn(SConn* conn, bool clear /*clear handle or not*/);
static bool readComplete(SConnBuffer* buf);
static SSrvConn* createConn();
static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
static void uvDestroyConn(uv_handle_t* handle);
@ -105,31 +113,9 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b
* |<--------------------------data from socket------------------------------->|
* |<------STransMsgHead------->|<-------------------other data--------------->|
*/
static const int CAPACITY = 1024;
SConn* conn = handle->data;
SConnBuffer* pBuf = &conn->connBuf;
if (pBuf->cap == 0) {
pBuf->buf = (char*)calloc(CAPACITY, sizeof(char));
pBuf->len = 0;
pBuf->cap = CAPACITY;
pBuf->left = -1;
buf->base = pBuf->buf;
buf->len = CAPACITY;
} else {
if (pBuf->len >= pBuf->cap) {
if (pBuf->left == -1) {
pBuf->cap *= 2;
pBuf->buf = realloc(pBuf->buf, pBuf->cap);
} else if (pBuf->len + pBuf->left > pBuf->cap) {
pBuf->cap = pBuf->len + pBuf->left;
pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left);
}
}
buf->base = pBuf->buf + pBuf->len;
buf->len = pBuf->cap - pBuf->len;
}
SSrvConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
transAllocBuffer(pBuf, buf);
}
// check data read from socket completely or not
@ -159,7 +145,7 @@ static bool readComplete(SConnBuffer* data) {
// // impl later
// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg;
// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle;
// SConn* pConn = pRecv->thandle;
// SSrvConn* pConn = pRecv->thandle;
// tDump(pRecv->msg, pRecv->msgLen);
// terrno = 0;
// // SRpcReqContext* pContest;
@ -167,7 +153,7 @@ static bool readComplete(SConnBuffer* data) {
// // do auth and check
//}
static int uvAuthMsg(SConn* pConn, char* msg, int len) {
static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) {
STransMsgHead* pHead = (STransMsgHead*)msg;
int code = 0;
@ -222,14 +208,14 @@ static int uvAuthMsg(SConn* pConn, char* msg, int len) {
// refers specifically to query or insert timeout
static void uvHandleActivityTimeout(uv_timer_t* handle) {
SConn* conn = handle->data;
SSrvConn* conn = handle->data;
tDebug("%p timeout since no activity", conn);
}
static void uvHandleReq(SConn* pConn) {
static void uvHandleReq(SSrvConn* pConn) {
SRecvInfo info;
SRecvInfo* p = &info;
SConnBuffer* pBuf = &pConn->connBuf;
SConnBuffer* pBuf = &pConn->readBuf;
p->msg = pBuf->buf;
p->msgLen = pBuf->len;
p->ip = 0;
@ -255,7 +241,6 @@ static void uvHandleReq(SConn* pConn) {
pHead->code = htonl(pHead->code);
int32_t dlen = 0;
SRpcMsg rpcMsg;
if (transDecompressMsg(NULL, 0, NULL)) {
// add compress later
// pHead = rpcDecompressRpcMsg(pHead);
@ -264,6 +249,8 @@ static void uvHandleReq(SConn* pConn) {
// impl later
//
}
SRpcMsg rpcMsg;
rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = pHead->content;
rpcMsg.msgType = pHead->msgType;
@ -271,6 +258,7 @@ static void uvHandleReq(SConn* pConn) {
rpcMsg.ahandle = NULL;
rpcMsg.handle = pConn;
transClearBuffer(&pConn->readBuf);
pConn->ref++;
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
// uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
@ -280,8 +268,8 @@ static void uvHandleReq(SConn* pConn) {
void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
// opt
SConn* conn = cli->data;
SConnBuffer* pBuf = &conn->connBuf;
SSrvConn* conn = cli->data;
SConnBuffer* pBuf = &conn->readBuf;
if (nread > 0) {
pBuf->len += nread;
tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread);
@ -294,11 +282,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
return;
}
if (nread == 0) {
tDebug("conn %p except read", conn);
// destroyConn(conn, true);
return;
}
if (nread != UV_EOF) {
if (nread < 0 || nread != UV_EOF) {
if (conn->ref > 1) {
conn->ref++; // ref > 1 signed that write is in progress
}
tDebug("conn %p read error: %s", conn, uv_err_name(nread));
destroyConn(conn, true);
}
@ -310,25 +299,23 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b
void uvOnTimeoutCb(uv_timer_t* handle) {
// opt
SConn* pConn = handle->data;
SSrvConn* pConn = handle->data;
tDebug("conn %p time out", pConn);
}
void uvOnWriteCb(uv_write_t* req, int status) {
SConn* conn = req->data;
SSrvConn* conn = req->data;
SConnBuffer* buf = &conn->connBuf;
buf->len = 0;
memset(buf->buf, 0, buf->cap);
buf->left = -1;
SRpcMsg* pMsg = &conn->sendMsg;
transFreeMsg(pMsg->pCont);
SSrvMsg* smsg = conn->pSrvMsg;
destroySmsg(smsg);
conn->pSrvMsg = NULL;
transClearBuffer(&conn->readBuf);
if (status == 0) {
tDebug("conn %p data already was written on stream", conn);
} else {
tDebug("conn %p failed to write data, %s", conn, uv_err_name(status));
//
destroyConn(conn, true);
}
// opt
@ -341,16 +328,16 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
}
}
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) {
static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
// impl later;
tDebug("conn %p prepare to send resp", conn);
SRpcMsg* pMsg = &conn->sendMsg;
tDebug("conn %p prepare to send resp", smsg->pConn);
SRpcMsg* pMsg = &smsg->msg;
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
pMsg->contLen = 0;
}
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
pHead->msgType = conn->inType + 1;
pHead->msgType = smsg->pConn->inType + 1;
// add more info
char* msg = (char*)pHead;
int32_t len = transMsgLenFromCont(pMsg->contLen);
@ -361,28 +348,53 @@ static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) {
wb->base = msg;
wb->len = len;
}
static void uvStartSendResp(SSrvMsg* smsg) {
// impl
uv_buf_t wb;
uvPrepareSendData(smsg, &wb);
SSrvConn* pConn = smsg->pConn;
uv_timer_stop(pConn->pTimer);
pConn->pSrvMsg = smsg;
// conn->pWriter->data = smsg;
uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb);
// SRpcMsg* rpcMsg = smsg->msg;
return;
}
static void destroySmsg(SSrvMsg* smsg) {
if (smsg == NULL) {
return;
}
transFreeMsg(smsg->msg.pCont);
free(smsg);
}
void uvWorkerAsyncCb(uv_async_t* handle) {
SWorkThrdObj* pThrd = handle->data;
SConn* conn = NULL;
SSrvConn* conn = NULL;
queue wq;
// batch process to avoid to lock/unlock frequently
pthread_mutex_lock(&pThrd->connMtx);
QUEUE_MOVE(&pThrd->conn, &wq);
pthread_mutex_unlock(&pThrd->connMtx);
pthread_mutex_lock(&pThrd->msgMtx);
QUEUE_MOVE(&pThrd->msg, &wq);
pthread_mutex_unlock(&pThrd->msgMtx);
while (!QUEUE_IS_EMPTY(&wq)) {
queue* head = QUEUE_HEAD(&wq);
QUEUE_REMOVE(head);
SConn* conn = QUEUE_DATA(head, SConn, queue);
if (conn == NULL) {
tError("except occurred, do nothing");
return;
}
uv_buf_t wb;
uvPrepareSendData(conn, &wb);
uv_timer_stop(conn->pTimer);
uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
SSrvMsg* msg = QUEUE_DATA(head, SSrvMsg, q);
if (msg == NULL) {
tError("except occurred, continue");
continue;
}
uvStartSendResp(msg);
// uv_buf_t wb;
// uvPrepareSendData(msg, &wb);
// uv_timer_stop(conn->pTimer);
// uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
}
}
@ -435,7 +447,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP);
SConn* pConn = createConn();
SSrvConn* pConn = createConn();
pConn->pTransInst = pThrd->pTransInst;
/* init conn timer*/
@ -484,8 +496,8 @@ static bool addHandleToWorkloop(void* arg) {
pThrd->pipe->data = pThrd;
QUEUE_INIT(&pThrd->conn);
pthread_mutex_init(&pThrd->connMtx, NULL);
QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL);
pThrd->workerAsync = malloc(sizeof(uv_async_t));
uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb);
@ -523,34 +535,39 @@ void* workerThread(void* arg) {
uv_run(pThrd->loop, UV_RUN_DEFAULT);
}
static SConn* createConn() {
SConn* pConn = (SConn*)calloc(1, sizeof(SConn));
static SSrvConn* createConn() {
SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn));
tDebug("conn %p created", pConn);
++pConn->ref;
return pConn;
}
static void destroyConn(SConn* conn, bool clear) {
static void destroyConn(SSrvConn* conn, bool clear) {
if (conn == NULL) {
return;
}
if (--conn->ref == 0) {
tDebug("conn %p try to destroy", conn);
if (--conn->ref > 0) {
return;
}
transDestroyBuffer(&conn->readBuf);
destroySmsg(conn->pSrvMsg);
conn->pSrvMsg = NULL;
if (clear) {
uv_close((uv_handle_t*)conn->pTcp, NULL);
uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);
}
}
static void uvDestroyConn(uv_handle_t* handle) {
SSrvConn* conn = handle->data;
tDebug("conn %p destroy", conn);
uv_timer_stop(conn->pTimer);
free(conn->pTimer);
free(conn->pTcp);
free(conn->connBuf.buf);
// free(conn->pTcp);
free(conn->pWriter);
free(conn);
}
static void uvDestroyConn(uv_handle_t* handle) {
SConn* conn = handle->data;
destroyConn(conn, false);
}
static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) {
static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) {
STransMsgHead* pHead = (STransMsgHead*)msg;
if (pConn->spi && pConn->secured == 0) {
@ -632,6 +649,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
pthread_join(pThrd->thread, NULL);
// free(srv->pipe[i]);
free(pThrd->loop);
pthread_mutex_destroy(&pThrd->msgMtx);
free(pThrd);
}
void taosCloseServer(void* arg) {
@ -648,17 +666,20 @@ void taosCloseServer(void* arg) {
}
void rpcSendResponse(const SRpcMsg* pMsg) {
SConn* pConn = pMsg->handle;
SSrvConn* pConn = pMsg->handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
// opt later
pConn->sendMsg = *pMsg;
pthread_mutex_lock(&pThrd->connMtx);
QUEUE_PUSH(&pThrd->conn, &pConn->queue);
pthread_mutex_unlock(&pThrd->connMtx);
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *pMsg;
pthread_mutex_lock(&pThrd->msgMtx);
QUEUE_PUSH(&pThrd->msg, &srvMsg->q);
pthread_mutex_unlock(&pThrd->msgMtx);
tDebug("conn %p start to send resp", pConn);
uv_async_send(pConn->pWorkerAsync);
uv_async_send(pThrd->workerAsync);
}
#endif

15
source/libs/transport/test/rclient.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tep.h>
#include "os.h"
#include "rpcLog.h"
#include "taoserror.h"
@ -64,6 +65,7 @@ static void *sendRequest(void *param) {
// tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem);
tDebug("recv response succefully");
// usleep(100000000);
}
@ -86,12 +88,9 @@ int main(int argc, char *argv[]) {
pthread_attr_t thattr;
// server info
epSet.numOfEps = 1;
epSet.inUse = 0;
epSet.port[0] = 7000;
epSet.port[1] = 7000;
strcpy(epSet.fqdn[0], serverIp);
strcpy(epSet.fqdn[1], "192.168.0.1");
addEpIntoEpSet(&epSet, serverIp, 7000);
addEpIntoEpSet(&epSet, "192.168.0.1", 7000);
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
@ -109,9 +108,9 @@ int main(int argc, char *argv[]) {
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
epSet.port[0] = atoi(argv[++i]);
epSet.eps[0].port = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) {
tstrncpy(epSet.fqdn[0], argv[++i], sizeof(epSet.fqdn[0]));
tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn));
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
@ -135,7 +134,7 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]);
printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);

2
source/libs/transport/test/rserver.c
View File

@ -165,6 +165,7 @@ int main(int argc, char *argv[]) {
tError("failed to start RPC server");
return -1;
}
// sleep(5);
tInfo("RPC server is running, ctrl-c to exit");
@ -172,7 +173,6 @@ int main(int argc, char *argv[]) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
taosAddIntoQset(qset, qhandle, NULL);

1
source/libs/wal/CMakeLists.txt
View File

@ -11,6 +11,7 @@ target_link_libraries(
PUBLIC cjson
PUBLIC os
PUBLIC util
PUBLIC common
)
if(${BUILD_TEST})

2
source/libs/wal/src/walWrite.c
View File

@ -257,7 +257,7 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
return 0;
}
int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen) {
int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) {
if (pWal == NULL) return -1;
int code = 0;

2
source/nodes/src/nodesClone.c → source/nodes/src/nodesCloneFuncs.c
View File

@ -15,6 +15,6 @@
#include "nodes.h"
void cloneNode(const SNode* pNode) {
void nodesCloneNode(const SNode* pNode) {
}

45
source/nodes/src/nodesCodeFuncs.c Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "nodes.h"
int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN:
case QUERY_NODE_VALUE:
case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_IS_NULL_CONDITION:
case QUERY_NODE_FUNCTION:
case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE:
case QUERY_NODE_GROUPING_SET:
case QUERY_NODE_ORDER_BY_EXPR:
case QUERY_NODE_LIMIT:
case QUERY_NODE_STATE_WINDOW:
case QUERY_NODE_SESSION_WINDOW:
case QUERY_NODE_INTERVAL_WINDOW:
case QUERY_NODE_SET_OPERATOR:
case QUERY_NODE_SELECT_STMT:
case QUERY_NODE_SHOW_STMT:
default:
break;
}
}
int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
}

23
source/nodes/src/nodesEqual.c → source/nodes/src/nodesEqualFuncs.c
View File

@ -32,17 +32,17 @@
#define COMPARE_NODE_FIELD(fldname) \
do { \
if (!nodeEqual(a->fldname, b->fldname)) \
if (!nodesEqualNode(a->fldname, b->fldname)) \
return false; \
} while (0)
#define COMPARE_ARRAY_FIELD(fldname) \
#define COMPARE_NODE_LIST_FIELD(fldname) \
do { \
if (!nodeArrayEqual(a->fldname, b->fldname)) \
if (!nodeNodeListEqual(a->fldname, b->fldname)) \
return false; \
} while (0)
static bool nodeArrayEqual(const SArray* a, const SArray* b) {
static bool nodeNodeListEqual(const SNodeList* a, const SNodeList* b) {
if (a == b) {
return true;
}
@ -51,13 +51,13 @@ static bool nodeArrayEqual(const SArray* a, const SArray* b) {
return false;
}
if (taosArrayGetSize(a) != taosArrayGetSize(b)) {
if (LIST_LENGTH(a) != LIST_LENGTH(b)) {
return false;
}
size_t size = taosArrayGetSize(a);
for (size_t i = 0; i < size; ++i) {
if (!nodeEqual((SNode*)taosArrayGetP(a, i), (SNode*)taosArrayGetP(b, i))) {
SNode* na, *nb;
FORBOTH(na, a, nb, b) {
if (!nodesEqualNode(na, nb)) {
return false;
}
}
@ -85,7 +85,7 @@ static bool operatorNodeEqual(const SOperatorNode* a, const SOperatorNode* b) {
static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicConditionNode* b) {
COMPARE_SCALAR_FIELD(condType);
COMPARE_ARRAY_FIELD(pParameterList);
COMPARE_NODE_LIST_FIELD(pParameterList);
return true;
}
@ -97,11 +97,11 @@ static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCond
static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) {
COMPARE_SCALAR_FIELD(funcId);
COMPARE_ARRAY_FIELD(pParameterList);
COMPARE_NODE_LIST_FIELD(pParameterList);
return true;
}
bool nodeEqual(const SNode* a, const SNode* b) {
bool nodesEqualNode(const SNode* a, const SNode* b) {
if (a == b) {
return true;
}
@ -132,6 +132,7 @@ bool nodeEqual(const SNode* a, const SNode* b) {
case QUERY_NODE_JOIN_TABLE:
case QUERY_NODE_GROUPING_SET:
case QUERY_NODE_ORDER_BY_EXPR:
case QUERY_NODE_LIMIT:
return false; // todo
default:
break;

39
source/nodes/src/nodesTraverse.c → source/nodes/src/nodesTraverseFuncs.c
View File

@ -17,17 +17,17 @@
typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext);
bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext) {
size_t size = taosArrayGetSize(pArray);
for (size_t i = 0; i < size; ++i) {
if (!nodeTreeWalker((SNode*)taosArrayGetP(pArray, i), walker, pContext)) {
return false;
}
bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) {
SNode* node;
FOREACH(node, pNodeList) {
if (!nodesWalkNode(node, walker, pContext)) {
return false;
}
return true;
}
return true;
}
bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
if (NULL == pNode) {
return true;
}
@ -39,38 +39,39 @@ bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN:
case QUERY_NODE_VALUE:
case QUERY_NODE_LIMIT:
// these node types with no subnodes
return true;
case QUERY_NODE_OPERATOR: {
SOperatorNode* pOpNode = (SOperatorNode*)pNode;
if (!nodeTreeWalker(pOpNode->pLeft, walker, pContext)) {
if (!nodesWalkNode(pOpNode->pLeft, walker, pContext)) {
return false;
}
return nodeTreeWalker(pOpNode->pRight, walker, pContext);
return nodesWalkNode(pOpNode->pRight, walker, pContext);
}
case QUERY_NODE_LOGIC_CONDITION:
return nodeArrayWalker(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext);
return nodesWalkNodeList(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext);
case QUERY_NODE_IS_NULL_CONDITION:
return nodeTreeWalker(((SIsNullCondNode*)pNode)->pExpr, walker, pContext);
return nodesWalkNode(((SIsNullCondNode*)pNode)->pExpr, walker, pContext);
case QUERY_NODE_FUNCTION:
return nodeArrayWalker(((SFunctionNode*)pNode)->pParameterList, walker, pContext);
return nodesWalkNodeList(((SFunctionNode*)pNode)->pParameterList, walker, pContext);
case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_TEMP_TABLE:
return true; // todo
case QUERY_NODE_JOIN_TABLE: {
SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode;
if (!nodeTreeWalker(pJoinTableNode->pLeft, walker, pContext)) {
if (!nodesWalkNode(pJoinTableNode->pLeft, walker, pContext)) {
return false;
}
if (!nodeTreeWalker(pJoinTableNode->pRight, walker, pContext)) {
if (!nodesWalkNode(pJoinTableNode->pRight, walker, pContext)) {
return false;
}
return nodeTreeWalker(pJoinTableNode->pOnCond, walker, pContext);
return nodesWalkNode(pJoinTableNode->pOnCond, walker, pContext);
}
case QUERY_NODE_GROUPING_SET:
return nodeArrayWalker(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext);
return nodesWalkNodeList(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext);
case QUERY_NODE_ORDER_BY_EXPR:
return nodeTreeWalker(((SOrderByExprNode*)pNode)->pExpr, walker, pContext);
return nodesWalkNode(((SOrderByExprNode*)pNode)->pExpr, walker, pContext);
default:
break;
}
@ -78,6 +79,6 @@ bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
return false;
}
bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
}

83
source/nodes/src/nodesUtilFuncs.c Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "nodes.h"
#include "nodesShowStmts.h"
bool nodesIsTimeorderQuery(const SNode* pQuery) {
}
bool nodesIsTimelineQuery(const SNode* pQuery) {
}
static SNode* makeNode(ENodeType type, size_t size) {
SNode* p = calloc(1, size);
setNodeType(p, type);
return p;
}
SNode* nodesMakeNode(ENodeType type) {
switch (type) {
case QUERY_NODE_COLUMN:
return makeNode(type, sizeof(SColumnNode));
case QUERY_NODE_VALUE:
return makeNode(type, sizeof(SValueNode));
case QUERY_NODE_OPERATOR:
return makeNode(type, sizeof(SOperatorNode));
case QUERY_NODE_LOGIC_CONDITION:
return makeNode(type, sizeof(SLogicConditionNode));
case QUERY_NODE_IS_NULL_CONDITION:
return makeNode(type, sizeof(SIsNullCondNode));
case QUERY_NODE_FUNCTION:
return makeNode(type, sizeof(SFunctionNode));
case QUERY_NODE_REAL_TABLE:
return makeNode(type, sizeof(SRealTableNode));
case QUERY_NODE_TEMP_TABLE:
return makeNode(type, sizeof(STempTableNode));
case QUERY_NODE_JOIN_TABLE:
return makeNode(type, sizeof(SJoinTableNode));
case QUERY_NODE_GROUPING_SET:
return makeNode(type, sizeof(SGroupingSetNode));
case QUERY_NODE_ORDER_BY_EXPR:
return makeNode(type, sizeof(SOrderByExprNode));
case QUERY_NODE_LIMIT:
return makeNode(type, sizeof(SLimitNode));
case QUERY_NODE_STATE_WINDOW:
return makeNode(type, sizeof(SStateWindowNode));
case QUERY_NODE_SESSION_WINDOW:
return makeNode(type, sizeof(SSessionWindowNode));
case QUERY_NODE_INTERVAL_WINDOW:
return makeNode(type, sizeof(SIntervalWindowNode));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
return makeNode(type, sizeof(SSelectStmt));
case QUERY_NODE_SHOW_STMT:
return makeNode(type, sizeof(SShowStmt));
default:
break;
}
return NULL;
}
void nodesDestroyNode(SNode* pNode) {
}
void nodesDestroyNodeList(SNodeList* pList) {
}

2
source/util/src/tworker.c
View File

@ -181,7 +181,7 @@ static void *tFWorkerThreadFp(SQWorker *worker) {
}
STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tFWorkerThreadFp);
return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp);
}
void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); }

6
tests/script/tsim/dnode/basic1.sim
View File

@ -178,9 +178,9 @@ if $rows != 3 then
endi
sql select * from st
#if $rows != 15 then
# return -1
#endi
if $rows != 15 then
return -1
endi
print =============== drop dnode
sql drop dnode 2;

12
tests/script/tsim/table/basic1.sim
View File

@ -139,9 +139,9 @@ endi
print =============== query data frpm st
sql select * from st
#if $rows != 21 then
# return -1
#endi
if $rows != 21 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
@ -200,8 +200,8 @@ endi
print =============== query data frpm st
sql select * from st
#if $rows != 21 then
# return -1
#endi
if $rows != 21 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT