diff --git a/.gitignore b/.gitignore index 2308ea7896..f0067d26ba 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ +source/libs/parser/inc/new_sql.* # Emacs # -*- mode: gitignore; -*- @@ -101,4 +102,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test \ No newline at end of file +!contrib/test diff --git a/cmake/cmake.options b/cmake/cmake.options index faa45256fb..e84d02800c 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -50,6 +50,12 @@ option( OFF ) +option( + BUILD_WITH_UV_TRANS + "If build with libuv_trans " + OFF +) + option( BUILD_WITH_CRAFT "If build with canonical-raft" diff --git a/include/common/common.h b/include/common/common.h index 9b8a465442..0299a29eb4 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -80,6 +80,80 @@ typedef struct SColumnInfoData { char *pData; // the corresponding block data in memory } SColumnInfoData; +static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { + int64_t tbUid = pBlock->info.uid; + int32_t numOfCols = pBlock->info.numOfCols; + int32_t rows = pBlock->info.rows; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, tbUid); + tlen += taosEncodeFixedI32(buf, numOfCols); + tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tlen += taosEncodeFixedI16(buf, pColData->info.colId); + tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI16(buf, pColData->info.bytes); + int32_t colSz = rows * pColData->info.bytes; + tlen += taosEncodeBinary(buf, pColData->pData, colSz); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI32(buf, &pBlock->info.numOfCols); + buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI32(buf, &sz); + pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData data; + buf = taosDecodeFixedI16(buf, &data.info.colId); + buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI16(buf, &data.info.bytes); + int32_t colSz = pBlock->info.rows * data.info.bytes; + buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); + taosArrayPush(pBlock->pDataBlock, &data); + } + return buf; +} + +static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { + int32_t tlen = 0; + int32_t sz = 0; + tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); + if (pRsp->pBlockData) { + sz = taosArrayGetSize(pRsp->pBlockData); + } + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = (SSDataBlock*) taosArrayGet(pRsp->pBlockData, i); + tlen += tEncodeDataBlock(buf, pBlock); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { + int32_t sz; + buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); + if (pRsp->schemas == NULL) return NULL; + buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); + buf = taosDecodeFixedI32(buf, &sz); + pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock block; + tDecodeDataBlock(buf, &block); + taosArrayPush(pRsp->pBlockData, &block); + } + return buf; +} + //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { diff --git a/include/common/tep.h b/include/common/tep.h index c8f45e4c82..69dd385a37 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -1,6 +1,10 @@ #ifndef TDENGINE_TEP_H #define TDENGINE_TEP_H +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "tmsg.h" @@ -9,10 +13,16 @@ typedef struct SCorEpSet { SEpSet epSet; } SCorEpSet; -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); +void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); + bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); SEpSet getEpSet_s(SCorEpSet *pEpSet); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TEP_H diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8e62b4d0db..5de006307d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -161,10 +161,10 @@ typedef struct { #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta -typedef struct { +typedef struct SEp { char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SEpAddr; +} SEp; typedef struct { int32_t contLen; @@ -273,8 +273,7 @@ typedef struct { typedef struct SEpSet { int8_t inUse; int8_t numOfEps; - uint16_t port[TSDB_MAX_REPLICA]; - char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; + SEp eps[TSDB_MAX_REPLICA]; } SEpSet; static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { @@ -282,8 +281,8 @@ static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { tlen += taosEncodeFixedI8(buf, pEp->inUse); tlen += taosEncodeFixedI8(buf, pEp->numOfEps); for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - tlen += taosEncodeFixedU16(buf, pEp->port[i]); - tlen += taosEncodeString(buf, pEp->fqdn[i]); + tlen += taosEncodeFixedU16(buf, pEp->eps[i].port); + tlen += taosEncodeString(buf, pEp->eps[i].fqdn); } return tlen; } @@ -292,8 +291,8 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { buf = taosDecodeFixedI8(buf, &pEp->inUse); buf = taosDecodeFixedI8(buf, &pEp->numOfEps); for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - buf = taosDecodeFixedU16(buf, &pEp->port[i]); - buf = taosDecodeStringTo(buf, pEp->fqdn[i]); + buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); + buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); } return buf; } @@ -624,8 +623,7 @@ typedef struct { int32_t id; int8_t isMnode; int8_t align; - uint16_t port; - char fqdn[TSDB_FQDN_LEN]; + SEp ep; } SDnodeEp; typedef struct { @@ -698,24 +696,17 @@ typedef struct { char tableNames[]; } SMultiTableInfoReq; +// todo refactor typedef struct SVgroupInfo { int32_t vgId; uint32_t hashBegin; uint32_t hashEnd; - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpSet epset; } SVgroupInfo; typedef struct { - int32_t vgId; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; -} SVgroupMsg; - -typedef struct { - int32_t numOfVgroups; - SVgroupMsg vgroups[]; + int32_t numOfVgroups; + SVgroupInfo vgroups[]; } SVgroupsInfo; typedef struct { @@ -1572,9 +1563,7 @@ typedef struct SMqSetCVgReq { char* sql; char* logicalPlan; char* physicalPlan; - uint32_t qmsgLen; - void* qmsg; - //SSubQueryMsg msg; + char* qmsg; } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { @@ -1606,7 +1595,6 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeFixedU32(buf, pReq->qmsgLen); tlen += taosEncodeString(buf, (char*)pReq->qmsg); //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; @@ -1621,7 +1609,6 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeFixedU32(buf, &pReq->qmsgLen); buf = taosDecodeString(buf, (char**)&pReq->qmsg); //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; @@ -1635,41 +1622,76 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; -typedef struct SMqColData { - int16_t colId; - int16_t type; - int16_t bytes; - char data[]; -} SMqColData; +typedef struct { + uint32_t nCols; + SSchema *pSchema; +} SSchemaWrapper; + +static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeString(buf, pSchema->name); + return tlen; +} + +static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) { + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeStringTo(buf, pSchema->name); + return buf; +} + +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { + int32_t tlen = 0; + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int32_t i = 0; i < pSW->nCols; i ++) { + tlen += tEncodeSSchema(buf, &pSW->pSchema[i]); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { + buf = taosDecodeFixedU32(buf, &pSW->nCols); + pSW->pSchema = (SSchema*) calloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + return NULL; + } + for (int32_t i = 0; i < pSW->nCols; i ++) { + buf = tDecodeSSchema(buf, &pSW->pSchema[i]); + } + return buf; +} typedef struct SMqTbData { int64_t uid; - int32_t numOfCols; int32_t numOfRows; - SMqColData colData[]; + char* colData; } SMqTbData; typedef struct SMqTopicBlk { - char topicName[TSDB_TOPIC_FNAME_LEN]; - int64_t committedOffset; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t bodyLen; - int32_t numOfTb; - SMqTbData tbData[]; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t bodyLen; + int32_t numOfTb; + SMqTbData* tbData; } SMqTopicData; typedef struct SMqConsumeRsp { - int64_t reqId; - int64_t consumerId; - int32_t bodyLen; - int32_t numOfTopics; - SMqTopicData data[]; + int64_t consumerId; + SSchemaWrapper* schemas; + int32_t numOfTopics; + SArray* pBlockData; //SArray } SMqConsumeRsp; // one req for one vg+topic typedef struct SMqConsumeReq { + SMsgHead head; //0: commit only, current offset //1: consume only, poll next offset //2: commit current and consume next offset @@ -1702,7 +1724,7 @@ typedef struct SMqCMGetSubEpRsp { static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; - tlen += taosEncodeFixedI16(buf, pVgEp->vgId); + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 90926da120..cbc8c0e87c 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -207,6 +207,36 @@ #define TK_INTO 189 #define TK_VALUES 190 +#define NEW_TK_UNION 1 +#define NEW_TK_ALL 2 +#define NEW_TK_MINUS 3 +#define NEW_TK_EXCEPT 4 +#define NEW_TK_INTERSECT 5 +#define NEW_TK_NK_PLUS 6 +#define NEW_TK_NK_MINUS 7 +#define NEW_TK_NK_STAR 8 +#define NEW_TK_NK_SLASH 9 +#define NEW_TK_SHOW 10 +#define NEW_TK_DATABASES 11 +#define NEW_TK_NK_ID 12 +#define NEW_TK_NK_LP 13 +#define NEW_TK_NK_RP 14 +#define NEW_TK_NK_COMMA 15 +#define NEW_TK_NK_LITERAL 16 +#define NEW_TK_NK_DOT 17 +#define NEW_TK_SELECT 18 +#define NEW_TK_DISTINCT 19 +#define NEW_TK_AS 20 +#define NEW_TK_FROM 21 +#define NEW_TK_WITH 22 +#define NEW_TK_RECURSIVE 23 +#define NEW_TK_ORDER 24 +#define NEW_TK_BY 25 +#define NEW_TK_ASC 26 +#define NEW_TK_DESC 27 +#define NEW_TK_NULLS 28 +#define NEW_TK_FIRST 29 +#define NEW_TK_LAST 30 #define TK_SPACE 300 #define TK_COMMENT 301 diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d2e602a5d6..26733696d2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -24,6 +24,7 @@ extern "C" { typedef void* qTaskInfo_t; typedef void* DataSinkHandle; +struct SRpcMsg; struct SSubplan; /** @@ -208,6 +209,8 @@ void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle); */ void** qDeregisterQInfo(void* pMgmt, void* pQInfo); +void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); + #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 374ae02e4a..d245d04166 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -35,7 +35,7 @@ typedef struct SQueryNode { int16_t type; } SQueryNode; -#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) +#define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) typedef struct SField { char name[TSDB_COL_NAME_LEN]; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6498abb7fc..c7ca2d9e1d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -128,20 +128,9 @@ typedef struct SMsgSendInfo { typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpSet epset; } SQueryNodeAddr; -static FORCE_INLINE void tConvertQueryAddrToEpSet(SEpSet* pEpSet, const SQueryNodeAddr* pAddr) { - pEpSet->inUse = pAddr->inUse; - pEpSet->numOfEps = pAddr->numOfEps; - for (int j = 0; j < TSDB_MAX_REPLICA; j++) { - pEpSet->port[j] = pAddr->epAddr[j].port; - memcpy(pEpSet->fqdn[j], pAddr->epAddr[j].fqdn, TSDB_FQDN_LEN); - } -} - int32_t initTaskQueue(); int32_t cleanupTaskQueue(); diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 5d815d15e0..5e3320ffdb 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -49,9 +49,10 @@ typedef struct { } SQWorkerStat; typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); +typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *); - -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp); +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, + putReqToQueryQFp fp1, sendReqToDnodeFp fp2); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); @@ -65,6 +66,8 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/include/libs/transport/transport.h b/include/libs/transport/transport.h index a05a76931a..f5ffc125ea 100644 --- a/include/libs/transport/transport.h +++ b/include/libs/transport/transport.h @@ -20,21 +20,6 @@ extern "C" { #endif -//typedef struct SEpAddr { -// char fqdn[TSDB_FQDN_LEN]; -// uint16_t port; -//} SEpAddr; -// -//typedef struct SVgroup { -// int32_t vgId; -// int8_t numOfEps; -// SEpAddr epAddr[TSDB_MAX_REPLICA]; -//} SVgroup; -// -//typedef struct SVgroupsInfo { -// int32_t numOfVgroups; -// SVgroup vgroups[]; -//} SVgroupsInfo; #ifdef __cplusplus } diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 641b485f4c..45f1d88c30 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -19,6 +19,7 @@ #include "tarray.h" #include "tdef.h" #include "tlog.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif @@ -159,7 +160,7 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write -int64_t walWrite(SWal *, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen); +int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); void walFsync(SWal *, bool force); // apis for lifecycle management diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index e6b41adf14..e297059bc8 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include "tarray.h" #include "tdef.h" typedef enum ENodeType { @@ -35,12 +34,14 @@ typedef enum ENodeType { QUERY_NODE_JOIN_TABLE, QUERY_NODE_GROUPING_SET, QUERY_NODE_ORDER_BY_EXPR, + QUERY_NODE_LIMIT, QUERY_NODE_STATE_WINDOW, QUERY_NODE_SESSION_WINDOW, QUERY_NODE_INTERVAL_WINDOW, QUERY_NODE_SET_OPERATOR, - QUERY_NODE_SELECT_STMT + QUERY_NODE_SELECT_STMT, + QUERY_NODE_SHOW_STMT } ENodeType; /** @@ -52,6 +53,27 @@ typedef struct SNode { } SNode; #define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +typedef struct SListCell { + SNode* pNode; + struct SListCell* pNext; +} SListCell; + +typedef struct SNodeList { + int16_t length; + SListCell* pHeader; +} SNodeList; + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHeader : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHeader : NULL), *cell2 = (NULL != (list2) ? (list2)->pHeader : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) typedef struct SDataType { uint8_t type; @@ -128,7 +150,7 @@ typedef enum ELogicConditionType { typedef struct SLogicConditionNode { ENodeType type; // QUERY_NODE_LOGIC_CONDITION ELogicConditionType condType; - SArray* pParameterList; + SNodeList* pParameterList; } SLogicConditionNode; typedef struct SIsNullCondNode { @@ -141,7 +163,7 @@ typedef struct SFunctionNode { SExprNode type; // QUERY_NODE_FUNCTION char functionName[TSDB_FUNC_NAME_LEN]; int32_t funcId; - SArray* pParameterList; // SNode + SNodeList* pParameterList; // SNode } SFunctionNode; typedef struct STableNode { @@ -151,12 +173,12 @@ typedef struct STableNode { } STableNode; typedef struct SRealTableNode { - STableNode type; // QUERY_NODE_REAL_TABLE + STableNode table; // QUERY_NODE_REAL_TABLE char dbName[TSDB_DB_NAME_LEN]; } SRealTableNode; typedef struct STempTableNode { - STableNode type; // QUERY_NODE_TEMP_TABLE + STableNode table; // QUERY_NODE_TEMP_TABLE SNode* pSubquery; } STempTableNode; @@ -165,7 +187,7 @@ typedef enum EJoinType { } EJoinType; typedef struct SJoinTableNode { - STableNode type; // QUERY_NODE_JOIN_TABLE + STableNode table; // QUERY_NODE_JOIN_TABLE EJoinType joinType; SNode* pLeft; SNode* pRight; @@ -179,7 +201,7 @@ typedef enum EGroupingSetType { typedef struct SGroupingSetNode { ENodeType type; // QUERY_NODE_GROUPING_SET EGroupingSetType groupingSetType; - SArray* pParameterList; + SNodeList* pParameterList; } SGroupingSetNode; typedef enum EOrder { @@ -188,7 +210,8 @@ typedef enum EOrder { } EOrder; typedef enum ENullOrder { - NULL_ORDER_FIRST = 1, + NULL_ORDER_DEFAULT = 1, + NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder; @@ -199,10 +222,11 @@ typedef struct SOrderByExprNode { ENullOrder nullOrder; } SOrderByExprNode; -typedef struct SLimitInfo { +typedef struct SLimitNode { + ENodeType type; // QUERY_NODE_LIMIT uint64_t limit; uint64_t offset; -} SLimitInfo; +} SLimitNode; typedef struct SStateWindowNode { ENodeType type; // QUERY_NODE_STATE_WINDOW @@ -225,15 +249,16 @@ typedef struct SIntervalWindowNode { typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - SArray* pProjectionList; // SNode + bool isStar; + SNodeList* pProjectionList; // SNode SNode* pFromTable; SNode* pWhereCond; - SArray* pPartitionByList; // SNode + SNodeList* pPartitionByList; // SNode SNode* pWindowClause; - SArray* pGroupByList; // SGroupingSetNode - SArray* pOrderByList; // SOrderByExprNode - SLimitInfo limit; - SLimitInfo slimit; + SNodeList* pGroupByList; // SGroupingSetNode + SNodeList* pOrderByList; // SOrderByExprNode + SLimitNode limit; + SLimitNode slimit; } SSelectStmt; typedef enum ESetOperatorType { @@ -249,20 +274,24 @@ typedef struct SSetOperator { typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); -bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext); -bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); +bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); +bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext); -bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); +bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodeEqual(const SNode* a, const SNode* b); +bool nodesEqualNode(const SNode* a, const SNode* b); -void cloneNode(const SNode* pNode); +void nodesCloneNode(const SNode* pNode); -int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen); -int32_t stringToNode(const char* pStr, SNode** pNode); +int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t nodesStringToNode(const char* pStr, SNode** pNode); -bool isTimeorderQuery(const SNode* pQuery); -bool isTimelineQuery(const SNode* pQuery); +bool nodesIsTimeorderQuery(const SNode* pQuery); +bool nodesIsTimelineQuery(const SNode* pQuery); + +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); +void nodesDestroyNodeList(SNodeList* pList); #ifdef __cplusplus } diff --git a/include/nodes/nodesShowStmts.h b/include/nodes/nodesShowStmts.h new file mode 100644 index 0000000000..312fbbc4f4 --- /dev/null +++ b/include/nodes/nodesShowStmts.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_NODES_SHOW_STMTS_H_ +#define _TD_NODES_SHOW_STMTS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nodes.h" + +typedef enum EShowStmtType { + SHOW_TYPE_DATABASE = 1 +} EShowStmtType; + +typedef struct SShowStmt { + ENodeType type; // QUERY_NODE_SHOW_STMT + EShowStmtType showType; +} SShowStmt; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_NODES_SHOW_STMTS_H_*/ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 69bf085491..8f49fce558 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -253,6 +253,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4) #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) +#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 28faa76785..cb73701bfa 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -127,6 +127,8 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; + SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; + hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); @@ -517,4 +519,4 @@ setConfRet taos_set_config(const char *config){ pthread_mutex_unlock(&setConfMutex); return ret; } -#endif \ No newline at end of file +#endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 4cb2cd2aae..0379a76832 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -300,11 +300,13 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { static void* hbThreadFunc(void* param) { setThreadName("hb"); while (1) { - int8_t threadStop = atomic_load_8(&clientHbMgr.threadStop); - if(threadStop) { + int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); + if(1 == threadStop) { break; } + pthread_mutex_lock(&clientHbMgr.lock); + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for(int i = 0; i < sz; i++) { SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -352,6 +354,9 @@ static void* hbThreadFunc(void* param) { atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } + + pthread_mutex_unlock(&clientHbMgr.lock); + taosMsleep(HEARTBEAT_INTERVAL); } return NULL; @@ -371,7 +376,13 @@ static int32_t hbCreateThread() { } static void hbStopThread() { - atomic_store_8(&clientHbMgr.threadStop, 1); + if (atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 0, 1)) { + return; + } + + while (2 != atomic_load_8(&clientHbMgr.threadStop)) { + usleep(10); + } } SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { @@ -409,11 +420,18 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { return NULL; } + pthread_mutex_lock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); + pthread_mutex_unlock(&clientHbMgr.lock); + return pAppHbMgr; } void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { + if (NULL == pAppHbMgr) { + return; + } + pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); @@ -421,7 +439,9 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); if (pAppHbMgr == pTarget) { taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; taosHashCleanup(pTarget->connInfo); + pTarget->connInfo = NULL; } } @@ -446,11 +466,17 @@ int hbMgrInit() { } void hbMgrCleanUp() { + hbStopThread(); + // destroy all appHbMgr int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - taosArrayDestroy(clientHbMgr.appHbMgrs); + pthread_mutex_lock(&clientHbMgr.lock); + taosArrayDestroy(clientHbMgr.appHbMgrs); + pthread_mutex_unlock(&clientHbMgr.lock); + + clientHbMgr.appHbMgrs = NULL; } int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { @@ -502,6 +528,9 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); + if (atomic_load_32(&pAppHbMgr->connKeyCnt) <= 0) { + appHbMgrCleanup(pAppHbMgr); + } } int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 1132cf63e4..4100781a0b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -101,7 +101,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } if (port) { - epSet.epSet.port[0] = port; + epSet.epSet.eps[0].port = port; } } else { if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { @@ -328,6 +328,7 @@ struct tmq_t { char clientId[256]; int64_t consumerId; int64_t status; + tsem_t rspSem; STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; @@ -344,6 +345,7 @@ tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t err strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->commit_cb = conf->commit_cb; + tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); return pTmq; @@ -372,11 +374,27 @@ int32_t tmq_list_append(tmq_list_t* ptr, char* src) { } +int32_t tmq_null_cb(void* param, const SDataBuf* pMsg, int32_t code) { + if (code == 0) { + // + } + // + return 0; +} + TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SRequestObj *pRequest = NULL; - tmq->status = 1; int32_t sz = topic_list->cnt; - tmq->clientTopics = taosArrayInit(sz, sizeof(void*)); + //destroy ex + taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + + SCMSubscribeReq req; + req.topicNum = sz; + req.consumerId = tmq->consumerId; + req.consumerGroup = strdup(tmq->groupId); + req.topicNames = taosArrayInit(sz, sizeof(void*)); + for (int i = 0; i < sz; i++) { char* topicName = topic_list->elems[i]; @@ -391,16 +409,21 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { } tNameExtractFullName(&name, topicFname); tscDebug("subscribe topic: %s", topicFname); - taosArrayPush(tmq->clientTopics, &topicFname); + SMqClientTopic topic = { + .nextVgIdx = 0, + .sql = NULL, + .sqlLen = 0, + .topicId = 0, + .topicName = topicFname, + .vgs = NULL + }; + topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); + taosArrayPush(tmq->clientTopics, &topic); /*SMqClientTopic topic = {*/ /*.*/ /*};*/ + taosArrayPush(req.topicNames, &topicFname); } - SCMSubscribeReq req; - req.topicNum = taosArrayGetSize(tmq->clientTopics); - req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); - req.topicNames = tmq->clientTopics; int tlen = tSerializeSCMSubscribeReq(NULL, &req); void* buf = malloc(tlen); @@ -419,18 +442,19 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + /*sendInfo->fp*/ SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pRequest->body.rspSem); _return: - if (body != NULL) { - destroySendMsgInfo(body); - } + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { pRequest->code = terrno; @@ -564,19 +588,19 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; pRequest->type = TDMT_MND_CREATE_TOPIC; - SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pRequest->body.rspSem); _return: qDestroyQuery(pQueryNode); - if (body != NULL) { - destroySendMsgInfo(body); - } + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { pRequest->code = terrno; @@ -592,11 +616,38 @@ struct tmq_message_t { }; int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { + SMqConsumeRsp rsp; + tDecodeSMqConsumeRsp(pMsg->pData, &rsp); + int32_t colNum = rsp.schemas->nCols; + for (int32_t i = 0; i < colNum; i++) { + printf("| %s |", rsp.schemas->pSchema[i].name); + } + printf("\n"); + int32_t sz = taosArrayGetSize(rsp.pBlockData); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < colNum; j++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, j); + for (int32_t k = 0; k < rows; k++) { + void* var = POINTER_SHIFT(pColInfoData->pData, k * pColInfoData->info.bytes); + if (j == 0) printf(" %ld ", *(int64_t*)var); + if (j == 1) printf(" %d ", *(int32_t*)var); + } + } + /*pDataBlock->*/ + } return 0; } int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { tmq_t* tmq = (tmq_t*)param; + if (code != 0) { + tsem_post(&tmq->rspSem); + return 0; + } + tscDebug("tmq ask ep cb called"); + bool set = false; SMqCMGetSubEpRsp rsp; tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); int32_t sz = taosArrayGetSize(rsp.topics); @@ -611,38 +662,35 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { for (int32_t j = 0; j < vgSz; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); SMqClientVg clientVg = { + .pollCnt = 0, + .committedOffset = -1, + .currentOffset = -1, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; taosArrayPush(topic.vgs, &clientVg); + set = true; } taosArrayPush(tmq->clientTopics, &topic); } + if(set) tmq->status = 1; // unlock + tsem_post(&tmq->rspSem); return 0; } tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { - if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) { - return NULL; - } - SRequestObj *pRequest = NULL; - SMqConsumeReq req = {0}; - req.reqType = 1; - req.blockingTime = blocking_time; - req.consumerId = tmq->consumerId; - tmq_message_t* tmq_message = NULL; - strcpy(req.cgroup, tmq->groupId); - if (taosArrayGetSize(tmq->clientTopics) == 0) { + if (taosArrayGetSize(tmq->clientTopics) == 0 || tmq->status == 0) { int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { tscError("failed to malloc get subscribe ep buf"); } - buf->consumerId = htobe64(buf->consumerId); + buf->consumerId = htobe64(tmq->consumerId); + strcpy(buf->cgroup, tmq->groupId); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); if (pRequest == NULL) { tscError("failed to malloc subscribe ep request"); } @@ -659,23 +707,38 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(&pRequest->body.rspSem); + tsem_wait(&tmq->rspSem); } - SMqClientTopic* pTopic = taosArrayGetP(tmq->clientTopics, tmq->nextTopicIdx); + if (taosArrayGetSize(tmq->clientTopics) == 0) { + tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + return NULL; + } + + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + pReq->reqType = 1; + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + tmq_message_t* tmq_message = NULL; + strcpy(pReq->cgroup, tmq->groupId); + + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(req.topic, pTopic->topicName); + strcpy(pReq->topic, pTopic->topicName); int32_t nextVgIdx = pTopic->nextVgIdx; pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx); - req.offset = pVg->currentOffset; + pReq->offset = pVg->currentOffset; - pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) }; - pRequest->type = TDMT_VND_CONSUME; + pReq->head.vgId = htonl(pVg->vgId); + pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; - sendInfo->param = &tmq_message; + /*sendInfo->param = &tmq_message;*/ sendInfo->fp = tmq_poll_cb_inner; int64_t transporterId = 0; @@ -729,7 +792,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; @@ -759,7 +821,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return -1; } - taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; } @@ -769,7 +831,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return -1; } - taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); mgmtEpSet->numOfEps++; } @@ -974,14 +1036,7 @@ void* doFetchRow(SRequestObj* pRequest) { SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - epSet.numOfEps = pVgroupInfo->numOfEps; - epSet.inUse = pVgroupInfo->inUse; - - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); - epSet.port[i] = pVgroupInfo->epAddr[i].port; - } - + epSet = pVgroupInfo->epset; } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -998,14 +1053,7 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->body.requestMsg.pData = pShowReq; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - - epSet.numOfEps = pVgroupInfo->numOfEps; - epSet.inUse = pVgroupInfo->inUse; - - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); - epSet.port[i] = pVgroupInfo->epAddr[i].port; - } + epSet = pVgroupInfo->epset; int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ba2a21d7ea..fda7287a4b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -46,6 +46,8 @@ void taos_cleanup(void) { clientConnRefPool = -1; taosCloseRef(id); + hbMgrCleanUp(); + rpcCleanup(); catalogDestroy(); taosCloseLog(); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 7b4b3353fb..028f44da14 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -54,7 +54,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pConnect->epSet.numOfEps > 0); for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { - pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]); + pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port); } if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { @@ -62,7 +62,8 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { } for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id); + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn, + pConnect->epSet.eps[i].port, pTscObj->id); } pTscObj->connId = pConnect->connId; @@ -72,6 +73,8 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + pTscObj->connType = HEARTBEAT_TYPE_QUERY; + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); // pRequest->body.resInfo.pRspMsg = pMsg->pData; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a1adf58f6a..4dda0b3757 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -62,545 +62,488 @@ TEST(testCase, connect_Test) { } //TEST(testCase, create_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, create_account_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, drop_account_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, show_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show users"); - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, drop_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, show_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show databases"); - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); //} - +// //TEST(testCase, create_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create database abc1 vgroups 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); //} - +// //TEST(testCase, create_dnode_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - //if (taos_errno(pRes) != 0) { - //printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); //} - +// //TEST(testCase, drop_dnode_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); - //if (taos_errno(pRes) != 0) { - //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //pRes = taos_query(pConn, "drop dnode 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// pRes = taos_query(pConn, "drop dnode 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, use_db_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in use db, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); //} - - //TEST(testCase, drop_db_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //showDB(pConn); - - //TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //showDB(pConn); - - //pRes = taos_query(pConn, "create database abc1"); - //if (taos_errno(pRes) != 0) { - //printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - //taos_close(pConn); +// +// TEST(testCase, drop_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// showDB(pConn); +// +// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// showDB(pConn); +// +// pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, create_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); - //} - - //pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - //pRes = taos_query(pConn, "drop stable `123_$^)`"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); - //} - - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } +// +// pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// pRes = taos_query(pConn, "drop stable `123_$^)`"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_close(pConn); //} - +// //TEST(testCase, create_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - //ASSERT_EQ(taos_errno(pRes), 0); - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - //ASSERT_NE(taos_errno(pRes), 0); - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); +// ASSERT_EQ(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); +// ASSERT_NE(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, create_ctable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, show_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, show_vgroup_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "show vgroups"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, create_multiple_tables) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //taos_close(pConn); - //return; - //} - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //taos_free_result(pRes); - //pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - - //for (int32_t i = 0; i < 20; ++i) { - //char sql[512] = {0}; - //snprintf(sql, tListLen(sql), - //"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - //(i + 1) * 30, (i + 2) * 40); - //TAOS_RES* pres = taos_query(pConn, sql); - //if (taos_errno(pres) != 0) { - //printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - //} - //taos_free_result(pres); - //} - - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create stable tables, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); //} - +// //TEST(testCase, show_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "show abc1.tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //int32_t count = 0; - //char str[512] = {0}; - - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%d: %s\n", ++count, str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show abc1.tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// int32_t count = 0; +// char str[512] = {0}; +// +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%d: %s\n", ++count, str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, drop_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in using db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "drop stable st1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); //} - +// //TEST(testCase, generated_request_id_test) { - //SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - //for (int32_t i = 0; i < 50000; ++i) { - //uint64_t v = generateRequestId(); - //void* result = taosHashGet(phash, &v, sizeof(v)); - //if (result != nullptr) { - //printf("0x%lx, index:%d\n", v, i); - //} - //assert(result == nullptr); - //taosHashPut(phash, &v, sizeof(v), NULL, 0); - //} - - //taosHashCleanup(phash); +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); //} - -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} - -//TEST(testCase, insert_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, tmq_subscribe_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //tmq_conf_t* conf = tmq_conf_new(); - //tmq_conf_set(conf, "group.id", "tg1"); - //tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); - - //tmq_list_t* topic_list = tmq_list_new(); - //tmq_list_append(topic_list, "test_topic_1"); - //tmq_subscribe(tmq, topic_list); - - //while (1) { - //tmq_message_t* msg = tmq_consume_poll(tmq, 0); - //printf("get msg\n"); - //if (msg == NULL) break; - //} -//} - -TEST(testCase, tmq_consume_Test) { -} - -TEST(testCase, tmq_commit_TEST) { -} - +// //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -610,7 +553,7 @@ TEST(testCase, tmq_commit_TEST) { // // pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); // if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// printf("failed to create into table t_2, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); // ASSERT_TRUE(false); // } @@ -618,58 +561,116 @@ TEST(testCase, tmq_commit_TEST) { // taos_free_result(pRes); // taos_close(pConn); //} - -//TEST(testCase, projection_query_tables) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //for(int32_t i = 0; i < 100000; ++i) { - //char sql[512] = {0}; - //sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - //TAOS_RES* p = taos_query(pConn, sql); - //if (taos_errno(p) != 0) { - //printf("failed to insert data, reason:%s\n", taos_errstr(p)); - //} - - //taos_free_result(p); - //} - - //pRes = taos_query(pConn, "select * from tu"); - //if (taos_errno(pRes) != 0) { - //printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); +// +//#if 0 +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// //taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); //} - +// +// +//TEST(testCase, tmq_subscribe_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// tmq_conf_t* conf = tmq_conf_new(); +// tmq_conf_set(conf, "group.id", "tg1"); +// tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); +// +// tmq_list_t* topic_list = tmq_list_new(); +// tmq_list_append(topic_list, "test_topic_1"); +// tmq_subscribe(tmq, topic_list); +// +// while (1) { +// tmq_message_t* msg = tmq_consume_poll(tmq, 0); +// printf("get msg\n"); +// //if (msg == NULL) break; +// } +//} +//#endif +// +//TEST(testCase, tmq_consume_Test) { +//} +// +//TEST(testCase, tmq_commit_TEST) { +//} +// +//TEST(testCase, projection_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// for(int32_t i = 0; i < 100000; ++i) { +// char sql[512] = {0}; +// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); +// TAOS_RES* p = taos_query(pConn, sql); +// if (taos_errno(p) != 0) { +// printf("failed to insert data, reason:%s\n", taos_errstr(p)); +// } +// +// taos_free_result(p); +// } +// +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -677,7 +678,7 @@ TEST(testCase, tmq_commit_TEST) { // TAOS_RES* pRes = taos_query(pConn, "use abc1"); // taos_free_result(pRes); // -// pRes = taos_query(pConn, "select ts from m1"); +// pRes = taos_query(pConn, "select ts from st1"); // if (taos_errno(pRes) != 0) { // printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); @@ -698,38 +699,38 @@ TEST(testCase, tmq_commit_TEST) { // taos_close(pConn); //} -//TEST(testCase, agg_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use dbv"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select count(*) from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, agg_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tx using st1 tags(111111111111111)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "select count(*) from t_x_19"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} #pragma GCC diagnostic pop diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 9cc99e7f51..cf38ab8dd9 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -2,32 +2,43 @@ #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); +int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { + pEp->port = 0; + strcpy(pEp->fqdn, ep); - char *temp = strchr(fqdn, ':'); + char *temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - *port = atoi(temp+1); + pEp->port = atoi(temp+1); } - if (*port == 0) { - *port = tsServerPort; + if (pEp->port == 0) { + pEp->port = tsServerPort; return -1; } return 0; } +void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { + if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { + return; + } + + int32_t index = pEpSet->numOfEps; + tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); + pEpSet->eps[index].port = port; + pEpSet->numOfEps += 1; +} + bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + if (s1->eps[i].port != s2->eps[i].port + || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) return false; } return true; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7a0966df57..a353c018b7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1080,9 +1080,7 @@ static void doInitGlobalConfig(void) { void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckAndPrintCfg() { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; - + SEp ep = {0}; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); } @@ -1097,15 +1095,15 @@ int32_t taosCheckAndPrintCfg() { if (tsFirst[0] == 0) { strcpy(tsFirst, tsLocalEp); } else { - taosGetFqdnPortFromEp(tsFirst, fqdn, &port); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", fqdn, port); + taosGetFqdnPortFromEp(tsFirst, &ep); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); } if (tsSecond[0] == 0) { strcpy(tsSecond, tsLocalEp); } else { - taosGetFqdnPortFromEp(tsSecond, fqdn, &port); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port); + taosGetFqdnPortFromEp(tsSecond, &ep); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); } taosCheckDataDirCfg(); diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 74fb5f1437..02dced53c2 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -289,6 +289,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { .charset = pCfg->charset, .nthreads = pCfg->numOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, + .sendReqToDnodeFp = dndSendReqToDnode }; if (vnodeInit(&vnodeOpt) != 0) { diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index b127fb1d64..0674d719b9 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -57,13 +57,13 @@ void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { if (pPort != NULL) { - *pPort = pDnodeEp->port; + *pPort = pDnodeEp->ep.port; } if (pFqdn != NULL) { - tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN); + tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); } if (pEp != NULL) { - snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); } } @@ -85,12 +85,12 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], pDnode->cfg.localFqdn) == 0 && epSet.port[i] == pDnode->cfg.serverPort) { + dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) { epSet.inUse = (i + 1) % epSet.numOfEps; } - epSet.port[i] = htons(epSet.port[i]); + epSet.eps[i].port = htons(epSet.eps[i].port); } rpcSendRedirectRsp(pReq->handle, &epSet); @@ -104,7 +104,7 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { pMgmt->mnodeEpSet = *pEpSet; for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]); + dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } taosWUnLockLatch(&pMgmt->latch); @@ -116,7 +116,7 @@ static void dndPrintDnodes(SDnode *pDnode) { dDebug("print dnode ep list, num:%d", pMgmt->dnodeEps->num); for (int32_t i = 0; i < pMgmt->dnodeEps->num; i++) { SDnodeEp *pEp = &pMgmt->dnodeEps->eps[i]; - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode); + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); } } @@ -145,8 +145,8 @@ static void dndResetDnodes(SDnode *pDnode, SDnodeEps *pDnodeEps) { if (!pDnodeEp->isMnode) continue; if (mIndex >= TSDB_MAX_REPLICA) continue; pMgmt->mnodeEpSet.numOfEps++; - strcpy(pMgmt->mnodeEpSet.fqdn[mIndex], pDnodeEp->fqdn); - pMgmt->mnodeEpSet.port[mIndex] = pDnodeEp->port; + + pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; mIndex++; } @@ -167,7 +167,7 @@ static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) { SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { char epstr[TSDB_EP_LEN + 1]; - snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); changed = strcmp(pEp, epstr) != 0; } @@ -251,11 +251,12 @@ static int32_t dndReadDnodes(SDnode *pDnode) { SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i]; - cJSON *dnodeId = cJSON_GetObjectItem(node, "id"); - if (!dnodeId || dnodeId->type != cJSON_Number) { + cJSON *did = cJSON_GetObjectItem(node, "id"); + if (!did || did->type != cJSON_Number) { dError("failed to read %s since dnodeId not found", pMgmt->file); goto PRASE_DNODE_OVER; } + pDnodeEp->id = dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); @@ -263,14 +264,15 @@ static int32_t dndReadDnodes(SDnode *pDnode) { dError("failed to read %s since dnodeFqdn not found", pMgmt->file); goto PRASE_DNODE_OVER; } - tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + tstrncpy(pDnodeEp->ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); if (!dnodePort || dnodePort->type != cJSON_Number) { dError("failed to read %s since dnodePort not found", pMgmt->file); goto PRASE_DNODE_OVER; } - pDnodeEp->port = dnodePort->valueint; + + pDnodeEp->ep.port = dnodePort->valueint; cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); if (!isMnode || isMnode->type != cJSON_Number) { @@ -298,7 +300,8 @@ PRASE_DNODE_OVER: pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; pMgmt->dnodeEps->eps[0].isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); + + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &(pMgmt->dnodeEps->eps[0].ep)); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -329,8 +332,8 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { for (int32_t i = 0; i < pMgmt->dnodeEps->num; ++i) { SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i]; len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->port); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); if (i < pMgmt->dnodeEps->num - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); @@ -395,7 +398,7 @@ void dndSendStatusReq(SDnode *pDnode) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:0x%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pMgmt->latch); pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; @@ -450,7 +453,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { pDnodeEps->num = htonl(pDnodeEps->num); for (int32_t i = 0; i < pDnodeEps->num; ++i) { pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); - pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + pDnodeEps->eps[i].ep.port = htons(pDnodeEps->eps[i].ep.port); } dndUpdateDnodeEps(pDnode, pDnodeEps); @@ -529,7 +532,7 @@ int32_t dndInitMgmt(SDnode *pDnode) { } if (pMgmt->dropped) { - dError("dnode will not start for its already dropped"); + dError("dnode not start since its already dropped"); return -1; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 275b09fc0f..257c72bff5 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -122,6 +122,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; @@ -148,6 +149,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index dcb73d13c7..c4d14ef697 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dndVnodes.h" #include "dndTransport.h" +#include "dndMgmt.h" typedef struct { int32_t vgId; @@ -527,7 +528,6 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; - pCfg->wsize = pCreate->cacheBlockSize; pCfg->lsize = pCreate->cacheBlockSize; pCfg->isHeapAllocator = true; pCfg->ttl = 4; @@ -578,6 +578,12 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { SWrapperCfg wrapperCfg = {0}; dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); + if (pCreate->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); + return -1; + } + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { dDebug("vgId:%d, already exist", pCreate->vgId); diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 3d61db8268..8403dbf034 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tep.h" #include "sut.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { @@ -61,11 +62,7 @@ void TestClient::Cleanup() { SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SEpSet epSet = {0}; - epSet.inUse = 0; - epSet.numOfEps = 1; - epSet.port[0] = port; - memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN); - + addEpIntoEpSet(&epSet, fqdn, port); rpcSendRequest(clientRpc, &epSet, pReq, NULL); tsem_wait(&sem); diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 1799bf66b0..17092ed737 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -68,6 +68,44 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); } } + + { + int32_t contLen = sizeof(SCreateVnodeReq); + + SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(3); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9527); + } + + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION); + } } TEST_F(DndTestVnode, 02_ALTER_Vnode) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 02ce3a1591..5ec9173fc8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -363,9 +363,7 @@ typedef struct SMqConsumerEp { int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; - uint32_t qmsgLen; char* qmsg; - //SSubQueryMsg qExec; } SMqConsumerEp; static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { @@ -374,9 +372,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumerEp->lastConsumerHbTs); + tlen += taosEncodeFixedI64(buf, pConsumerEp->lastVgHbTs); //tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); - tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen); - tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen); + tlen += taosEncodeString(buf, pConsumerEp->qmsg); return tlen; } @@ -385,9 +384,10 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumerEp->lastConsumerHbTs); + buf = taosDecodeFixedI64(buf, &pConsumerEp->lastVgHbTs); //buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); - buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen); - buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen); + buf = taosDecodeString(buf, &pConsumerEp->qmsg); return buf; } @@ -423,18 +423,27 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { free(pSub); return NULL; } - pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->assigned == NULL) { + pSub->lostConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->lostConsumer == NULL) { taosArrayDestroy(pSub->availConsumer); - taosArrayDestroy(pSub->idleConsumer); + taosArrayDestroy(pSub->assigned); + free(pSub); + return NULL; + } + pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->idleConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); free(pSub); return NULL; } pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->assigned == NULL) { + if (pSub->unassignedVg == NULL) { taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); taosArrayDestroy(pSub->idleConsumer); - taosArrayDestroy(pSub->unassignedVg); free(pSub); return NULL; } @@ -461,6 +470,13 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb tlen += tEncodeSMqConsumerEp(buf, pCEp); } + sz = taosArrayGetSize(pSub->lostConsumer); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->lostConsumer, i); + tlen += tEncodeSMqConsumerEp(buf, pCEp); + } + sz = taosArrayGetSize(pSub->idleConsumer); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -485,20 +501,47 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pSub->assigned = taosArrayInit(sz, sizeof(int64_t)); - if (pSub->assigned == NULL) { + pSub->availConsumer = taosArrayInit(sz, sizeof(int64_t)); + if (pSub->availConsumer == NULL) { return NULL; } for (int32_t i = 0; i < sz; i++) { int64_t consumerId; buf = taosDecodeFixedI64(buf, &consumerId); - taosArrayPush(pSub->assigned, &consumerId); + taosArrayPush(pSub->availConsumer, &consumerId); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->assigned = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->assigned == NULL) { + taosArrayDestroy(pSub->availConsumer); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->assigned, &cEp); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->lostConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->lostConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->lostConsumer, &cEp); } buf = taosDecodeFixedI32(buf, &sz); pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->idleConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); return NULL; } for (int32_t i = 0; i < sz; i++) { @@ -507,10 +550,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) taosArrayPush(pSub->idleConsumer, &cEp); } + buf = taosDecodeFixedI32(buf, &sz); pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->unassignedVg == NULL) { + taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); taosArrayDestroy(pSub->idleConsumer); return NULL; } @@ -580,7 +626,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic int32_t tlen = 0; tlen += taosEncodeString(buf, pConsumerTopic->name); tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch); - int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + int32_t sz = 0; + if (pConsumerTopic->pVgInfo != NULL) { + sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + } tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 94dff2e559..c1e070b16e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -226,10 +226,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; - if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1; @@ -498,7 +498,7 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbReq *pAlter) { +static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { @@ -649,7 +649,7 @@ static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter); if (code != 0) { mndReleaseDb(pMnode, pDb); mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); @@ -816,22 +816,22 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - SVgroupInfo *pInfo = &vgList[vindex]; + SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex]; pInfo->vgId = htonl(pVgroup->vgId); pInfo->hashBegin = htonl(pVgroup->hashBegin); pInfo->hashEnd = htonl(pVgroup->hashEnd); - pInfo->numOfEps = pVgroup->replica; + pInfo->epset.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEpAddr *pEpArrr = &pInfo->epAddr[gid]; + SEp * pEp = &pInfo->epset.eps[gid]; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { - memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - pEpArrr->port = htons(pDnode->port); + memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEp->port = htons(pDnode->port); } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - pInfo->inUse = gid; + pInfo->epset.inUse = gid; } } vindex++; @@ -1223,7 +1223,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 prec = TSDB_TIME_PRECISION_NANO_STR; break; default: - assert(false); + prec = "none"; break; } STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index efdacf957b..9a96bdb0df 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -203,8 +203,8 @@ void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { } SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) { - SEpSet epSet = {.inUse = 0, .numOfEps = 1, .port[0] = pDnode->port}; - memcpy(epSet.fqdn[0], pDnode->fqdn, TSDB_FQDN_LEN); + SEpSet epSet = {0}; + addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port); return epSet; } @@ -261,8 +261,8 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) { SDnodeEp *pEp = &pEps->eps[numOfEps]; pEp->id = htonl(pDnode->id); - pEp->port = htons(pDnode->port); - memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEp->ep.port = htons(pDnode->port); + memcpy(pEp->ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); pEp->isMnode = 0; if (mndIsMnode(pMnode, pDnode->id)) { pEp->isMnode = 1; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index f9dde7bc75..0e086fdb22 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -20,6 +20,7 @@ #include "mndTrans.h" #define SDB_FUNC_VER 1 +#define SDB_FUNC_RESERVE_SIZE 64 static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); @@ -60,7 +61,7 @@ void mndCleanupFunc(SMnode *pMnode) {} static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj); + int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj) + SDB_FUNC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size); if (pRaw == NULL) goto FUNC_ENCODE_OVER; @@ -78,6 +79,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, FUNC_ENCODE_OVER); terrno = 0; @@ -131,6 +133,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_DECODE_OVER) terrno = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 75ed5b0a1e..fbb4e5cef7 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -237,8 +237,8 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->pDnode == NULL) break; - pEpSet->port[pEpSet->numOfEps] = htons(pObj->pDnode->port); - memcpy(pEpSet->fqdn[pEpSet->numOfEps], pObj->pDnode->fqdn, TSDB_FQDN_LEN); + pEpSet->eps[pEpSet->numOfEps].port = htons(pObj->pDnode->port); + memcpy(pEpSet->eps[pEpSet->numOfEps].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); if (pObj->role == TAOS_SYNC_STATE_LEADER) { pEpSet->inUse = pEpSet->numOfEps; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index df6a4a82f3..4269196c2e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -66,13 +66,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->pCont; + SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp; int64_t consumerId = be64toh(pReq->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); if (pConsumer == NULL) { - /*terrno = */ + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); @@ -91,9 +91,13 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); for (int32_t j = 0; j < assignedSz; j++) { - SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j); if (pCEp->consumerId == consumerId) { - taosArrayPush(pSub->assigned, pCEp); + SMqSubVgEp vgEp = { + .epSet = pCEp->epSet, + .vgId = pCEp->vgId + }; + taosArrayPush(topicEp.vgs, &vgEp); } } if (taosArrayGetSize(topicEp.vgs) != 0) { @@ -101,7 +105,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } } int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); - void *buf = malloc(tlen); + void *buf = rpcMallocCont(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -161,8 +165,6 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { pReq->sql = strdup(pTopic->sql); pReq->logicalPlan = strdup(pTopic->logicalPlan); pReq->physicalPlan = strdup(pTopic->physicalPlan); - pReq->qmsgLen = pCEp->qmsgLen; - /*memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen);*/ pReq->qmsg = strdup(pCEp->qmsg); int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); void *reqStr = malloc(tlen); @@ -192,7 +194,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); } - mndReleaseTopic(pMnode, pTopic); + /*mndReleaseTopic(pMnode, pTopic);*/ mndTransDrop(pTrans); } pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); @@ -206,12 +208,12 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas SArray *pArray; SArray *inner = taosArrayGet(pDag->pSubplans, 0); SSubplan *plan = taosArrayGetP(inner, 0); - plan->execNode.inUse = 0; - strcpy(plan->execNode.epAddr[0].fqdn, "localhost"); - plan->execNode.epAddr[0].port = 6030; - plan->execNode.nodeId = 2; - plan->execNode.numOfEps = 1; + plan->execNode.nodeId = 2; + SEpSet* pEpSet = &plan->execNode.epset; + + pEpSet->inUse = 0; + addEpIntoEpSet(pEpSet, "localhost", 6030); if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { return -1; } @@ -220,20 +222,15 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas for (int32_t i = 0; i < sz; i++) { SMqConsumerEp CEp; CEp.status = 0; + CEp.consumerId = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo *pTaskInfo = taosArrayGet(pArray, i); - tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); + CEp.epSet = pTaskInfo->addr.epset; + /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], * CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ CEp.vgId = pTaskInfo->addr.nodeId; CEp.qmsg = strdup(pTaskInfo->msg->msg); - CEp.qmsgLen = strlen(CEp.qmsg) + 1; - printf("abc:\n%s\n", CEp.qmsg); - /*CEp.qmsg = malloc(CEp.qmsgLen);*/ - /*if (CEp.qmsg == NULL) {*/ - /*return -1;*/ - /*}*/ - /*memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen);*/ taosArrayPush(unassignedVg, &CEp); } @@ -257,8 +254,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume req.sql = pTopic->sql; req.logicalPlan = pTopic->logicalPlan; req.physicalPlan = pTopic->physicalPlan; - req.qmsg = strdup(pCEp->qmsg); - req.qmsgLen = strlen(req.qmsg); + req.qmsg = pCEp->qmsg; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { @@ -631,14 +627,14 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); + /*mndReleaseConsumer(pMnode, pConsumer);*/ return -1; } if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); - return 0; + /*mndReleaseConsumer(pMnode, pConsumer);*/ + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index fa043cf7a0..6b4cb4ba59 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -237,7 +237,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); topicObj.createTime = taosGetTimestampMs(); topicObj.updateTime = topicObj.createTime; - topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); topicObj.dbUid = pDb->uid; topicObj.version = 1; topicObj.sql = strdup(pCreate->sql); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 03226c7400..808e6dcbe5 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -921,7 +921,7 @@ static int32_t mndProcessTransMsg(SMnodeMsg *pMsg) { void mndTransPullup(SMnode *pMnode) { STrans *pTrans = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -930,4 +930,6 @@ void mndTransPullup(SMnode *pMnode) { mndTransExecute(pMnode, pTrans); sdbRelease(pMnode->pSdb, pTrans); } + + sdbWriteFile(pMnode->pSdb); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 04fedbb3ce..01d39ebdc2 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -178,7 +178,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId); - if (pVgroup == NULL) { + if (pVgroup == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; } return pVgroup; @@ -424,9 +424,7 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { epset.inUse = epset.numOfEps; } - epset.port[epset.numOfEps] = pDnode->port; - memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN); - epset.numOfEps++; + addEpIntoEpSet(&epset, pDnode->fqdn, pDnode->port); mndReleaseDnode(pMnode, pDnode); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 5d5947b644..964a483aac 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -277,9 +277,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddr* pAddr = &pInfo->epAddr[0]; + EXPECT_EQ(pInfo->epset.inUse, 0); + EXPECT_EQ(pInfo->epset.numOfEps, 1); + SEp* pAddr = &pInfo->epset.eps[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); @@ -293,9 +293,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddr* pAddr = &pInfo->epAddr[0]; + EXPECT_EQ(pInfo->epset.inUse, 0); + EXPECT_EQ(pInfo->epset.numOfEps, 1); + SEp* pAddr = &pInfo->epset.eps[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 991b4c1249..36550f875e 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -44,7 +44,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { pRsp->acctId = htonl(pRsp->acctId); pRsp->clusterId = htobe64(pRsp->clusterId); pRsp->connId = htonl(pRsp->connId); - pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + pRsp->epSet.eps[0].port = htons(pRsp->epSet.eps[0].port); EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); @@ -53,8 +53,8 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9031); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + EXPECT_EQ(pRsp->epSet.eps[0].port, 9031); + EXPECT_STREQ(pRsp->epSet.eps[0].fqdn, "localhost"); connId = pRsp->connId; } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 39b5bb4d5b..26809ea059 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -64,11 +64,7 @@ SSdb *sdbInit(SSdbOpt *pOption) { void sdbCleanup(SSdb *pSdb) { mDebug("start to cleanup sdb"); - if (pSdb->curVer > pSdb->lastCommitVer) { - mDebug("write sdb file for current ver:%" PRId64 " larger than last commit ver:%" PRId64, pSdb->curVer, - pSdb->lastCommitVer); - sdbWriteFile(pSdb); - } + sdbWriteFile(pSdb); if (pSdb->currDir != NULL) { tfree(pSdb->currDir); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 1f6d6cbda8..6d17423324 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -221,7 +221,7 @@ PARSE_SDB_DATA_ERROR: return code; } -int32_t sdbWriteFile(SSdb *pSdb) { +static int32_t sdbWriteFileImp(SSdb *pSdb) { int32_t code = 0; char tmpfile[PATH_MAX] = {0}; @@ -229,7 +229,8 @@ int32_t sdbWriteFile(SSdb *pSdb) { char curfile[PATH_MAX] = {0}; snprintf(curfile, sizeof(curfile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); - mDebug("start to write file:%s", curfile); + mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, + pSdb->lastCommitVer); FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); if (fd <= 0) { @@ -323,12 +324,20 @@ int32_t sdbWriteFile(SSdb *pSdb) { return code; } +int32_t sdbWriteFile(SSdb *pSdb) { + if (pSdb->curVer == pSdb->lastCommitVer) { + return 0; + } + + return sdbWriteFileImp(pSdb); +} + int32_t sdbDeploy(SSdb *pSdb) { if (sdbRunDeployFp(pSdb) != 0) { return -1; } - if (sdbWriteFile(pSdb) != 0) { + if (sdbWriteFileImp(pSdb) != 0) { return -1; } diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 383073871e..44a352ec54 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -37,11 +37,6 @@ typedef struct SMetaCfg { uint64_t lruSize; } SMetaCfg; -typedef struct { - uint32_t nCols; - SSchema *pSchema; -} SSchemaWrapper; - typedef struct SMTbCursor SMTbCursor; typedef struct SMCtbCursor SMCtbCursor; diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 3a1e5b9c95..faaf769e1a 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -149,11 +149,12 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; - SSubQueryMsg* pQueryMsg; + int8_t status; + int64_t offset; + void* dst; + qTaskInfo_t task; + STqReadHandle* pReadHandle; + SSubQueryMsg* pQueryMsg; } STqTaskItem; // new version diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2accfd6279..c8b47bf4a6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -33,6 +33,7 @@ extern "C" { typedef struct SVnode SVnode; typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); typedef struct STqCfg { // TODO @@ -64,17 +65,21 @@ typedef struct { const char *charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; + SendReqToDnodeFp sendReqToDnodeFp; } SVnodeOpt; typedef struct STqReadHandle { - int64_t ver; - uint64_t tbUid; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pMeta; - SArray* pColIdList; + int64_t ver; + uint64_t tbUid; + SSubmitMsg* pMsg; + SSubmitBlk* pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta* pVnodeMeta; + SArray* pColIdList; //SArray + int32_t sver; + SSchemaWrapper* pSchemaWrapper; + STSchema* pSchema; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 1fa65b2a73..f442697fb0 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -55,6 +55,7 @@ typedef struct SVnodeMgr { // For vnode Mgmt SDnode* pDnode; PutReqToVQueryQFp putReqToVQueryQFp; + SendReqToDnodeFp sendReqToDnodeFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -85,6 +86,7 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); +void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); // For Log extern int32_t vDebugFlag; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index dfdc144750..e5ccd02e48 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -20,6 +20,10 @@ #include "tcoding.h" #include "thash.h" +#define IMPL_WITH_LOCK 1 +// #if IMPL_WITH_LOCK +// #endif + typedef struct { tb_uid_t uid; int32_t sver; @@ -27,6 +31,9 @@ typedef struct { } SSchemaKey; struct SMetaDB { +#if IMPL_WITH_LOCK + pthread_rwlock_t rwlock; +#endif // DB DB *pTbDB; DB *pSchemaDB; @@ -58,6 +65,9 @@ static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static void metaDBWLock(SMetaDB *pDB); +static void metaDBRLock(SMetaDB *pDB); +static void metaDBULock(SMetaDB *pDB); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -130,8 +140,10 @@ void metaCloseDB(SMeta *pMeta) { int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { tb_uid_t uid; char buf[512]; + char buf1[512]; void * pBuf; - DBT key, value; + DBT key1, value1; + DBT key2, value2; SSchema *pSchema = NULL; if (pTbCfg->type == META_SUPER_TABLE) { @@ -143,19 +155,17 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { { // save table info pBuf = buf; - memset(&key, 0, sizeof(key)); - memset(&value, 0, sizeof(key)); + memset(&key1, 0, sizeof(key1)); + memset(&value1, 0, sizeof(key1)); - key.data = &uid; - key.size = sizeof(uid); + key1.data = &uid; + key1.size = sizeof(uid); metaEncodeTbInfo(&pBuf, pTbCfg); - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); - value.app_data = pTbCfg; - - pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key, &value, 0); + value1.data = buf; + value1.size = POINTER_DISTANCE(pBuf, buf); + value1.app_data = pTbCfg; } // save schema @@ -169,23 +179,28 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } if (pSchema) { - pBuf = buf; - memset(&key, 0, sizeof(key)); - memset(&value, 0, sizeof(key)); + pBuf = buf1; + memset(&key2, 0, sizeof(key2)); + memset(&value2, 0, sizeof(key2)); SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0}; - key.data = &schemaKey; - key.size = sizeof(schemaKey); + key2.data = &schemaKey; + key2.size = sizeof(schemaKey); SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema}; metaEncodeSchema(&pBuf, &sw); - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); - - pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0); + value2.data = buf1; + value2.size = POINTER_DISTANCE(pBuf, buf1); } + metaDBWLock(pMeta->pDB); + pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key1, &value1, 0); + if (pSchema) { + pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key2, &value2, 0); + } + metaDBULock(pMeta->pDB); + return 0; } @@ -234,11 +249,18 @@ static SMetaDB *metaNewDB() { return NULL; } +#if IMPL_WITH_LOCK + pthread_rwlock_init(&pDB->rwlock, NULL); +#endif + return pDB; } static void metaFreeDB(SMetaDB *pDB) { if (pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_destroy(&pDB->rwlock); +#endif free(pDB); } } @@ -467,7 +489,9 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { key.size = sizeof(uid); // Query + metaDBRLock(pDB); ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0); + metaDBULock(pDB); if (ret != 0) { return NULL; } @@ -496,7 +520,9 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { key.size = strlen(tbname); // Query + metaDBRLock(pDB); ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0); + metaDBULock(pDB); if (ret != 0) { return NULL; } @@ -529,7 +555,9 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo key.size = sizeof(schemaKey); // Query + metaDBRLock(pDB); ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); + metaDBULock(pDB); if (ret != 0) { printf("failed to query schema DB since %s================\n", db_strerror(ret)); return NULL; @@ -687,4 +715,22 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { } else { return 0; } -} \ No newline at end of file +} + +static void metaDBWLock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_wrlock(&(pDB->rwlock)); +#endif +} + +static void metaDBRLock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_rdlock(&(pDB->rwlock)); +#endif +} + +static void metaDBULock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_unlock(&(pDB->rwlock)); +#endif +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c164487aa2..3195691a13 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ +#include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -#include "tcompare.h" // static // read next version data @@ -484,7 +484,8 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { int32_t num = taosArrayGetSize(pConsumer->topics); - int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); + int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); if (tmpPtr == NULL) { @@ -511,13 +512,13 @@ int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** *(int64_t*)ptr = pTopic->committedOffset; POINTER_SHIFT(ptr, sizeof(int64_t)); } - + return 0; } const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) { STqConsumerHandle* pConsumer = *ppConsumer; - const void* ptr = pHead->content; + const void* ptr = pHead->content; pConsumer->consumerId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); pConsumer->epoch = *(int64_t*)ptr; @@ -668,31 +669,33 @@ int tqItemSSize() { #endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { - SMqConsumeReq* pReq = pMsg->pCont; - SRpcMsg rpcMsg; - int64_t reqId = pReq->reqId; - int64_t consumerId = pReq->consumerId; - int64_t reqOffset = pReq->offset; - int64_t fetchOffset = reqOffset; - int64_t blockingTime = pReq->blockingTime; + SMqConsumeReq* pReq = pMsg->pCont; + SRpcMsg rpcMsg; + int64_t reqId = pReq->reqId; + int64_t consumerId = pReq->consumerId; + int64_t reqOffset = pReq->offset; + int64_t fetchOffset = reqOffset; + int64_t blockingTime = pReq->blockingTime; - int rspLen = 0; + int rspLen = 0; + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 1, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - int sz = taosArrayGetSize(pConsumer->topics); + ASSERT(pConsumer); + int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - //TODO: support multiple topic in one req + // TODO: support multiple topic in one req if (strcmp(pTopic->topicName, pReq->topic) != 0) { continue; } - if (fetchOffset == -1) { - fetchOffset = pTopic->committedOffset + 1; - } - int8_t pos; - int8_t skip = 0; + if (fetchOffset == -1) { + fetchOffset = pTopic->committedOffset + 1; + } + int8_t pos; + int8_t skip = 0; SWalHead* pHead; while (1) { pos = fetchOffset % TQ_BUFFER_SIZE; @@ -726,7 +729,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { qSetStreamInput(task, pCont); - //SArray + // SArray SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { SSDataBlock* pDataBlock; @@ -735,28 +738,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { break; } if (pDataBlock != NULL) { - SMqTbData tbData = { - .uid = pDataBlock->info.uid, - .numOfCols = pDataBlock->info.numOfCols, - .numOfRows = pDataBlock->info.rows, - }; - for (int i = 0; i < pDataBlock->info.numOfCols; i++) { - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, i); - int32_t sz = pColData->info.bytes * pDataBlock->info.rows; - SMqColData colData = { - .bytes = pColData->info.bytes, - .colId = pColData->info.colId, - .type = pColData->info.type, - }; - memcpy(colData.data, pColData->pData, colData.bytes * pDataBlock->info.rows); - memcpy(&tbData.colData[i], &colData, sz); - } - /*pDataBlock->info.*/ taosArrayPush(pRes, pDataBlock); } else { break; } } + //TODO copy + rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; atomic_store_8(&pTopic->buffer.output[pos].status, 0); @@ -766,6 +754,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { continue; } + rsp.pBlockData = pRes; + +#if 0 pTopic->buffer.output[pos].dst = pRes; if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) { pTopic->buffer.firstOffset = pReq->offset; @@ -773,13 +764,20 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) { pTopic->buffer.lastOffset = pReq->offset; } - // put output into rsp - SMqConsumeRsp rsp = { - .consumerId = consumerId, - .numOfTopics = 1 - }; +#endif } - + int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + void* abuf = buf; + tEncodeSMqConsumeRsp(&abuf, &rsp); + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + rpcSendResponse(pMsg); return 0; } @@ -792,6 +790,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { } strcpy(pConsumer->cgroup, req.cgroup); pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); + pConsumer->consumerId = req.newConsumerId; + pConsumer->epoch = 0; STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); if (pTopic == NULL) { @@ -802,6 +802,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->sql = strdup(req.sql); pTopic->logicalPlan = strdup(req.logicalPlan); pTopic->physicalPlan = strdup(req.physicalPlan); + pTopic->committedOffset = -1; + pTopic->currentOffset = -1; pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -811,9 +813,12 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle); } taosArrayPush(pConsumer->topics, pTopic); + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); terrno = TSDB_CODE_SUCCESS; return 0; } @@ -823,23 +828,41 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { if (pReadHandle == NULL) { return NULL; } - pReadHandle->pMeta = pMeta; + pReadHandle->pVnodeMeta = pMeta; pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; + pReadHandle->sver = -1; + pReadHandle->pSchema = NULL; + pReadHandle->pSchemaWrapper = NULL; return pReadHandle; } void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { pReadHandle->pMsg = pMsg; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); pReadHandle->ver = ver; memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); } bool tqNextDataBlock(STqReadHandle* pHandle) { - while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) { - if (pHandle->tbUid == pHandle->pBlock->uid) return true; + while (1) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + return false; + } + if (pHandle->pBlock == NULL) return false; + + pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); + if (pHandle->tbUid == pHandle->pBlock->uid) { + pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); + pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); + pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); + pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); + pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); + return true; + } } return false; } @@ -854,31 +877,71 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) } SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { - int32_t sversion = pHandle->pBlock->sversion; - SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); - STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); - SArray* pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); + /*int32_t sversion = pHandle->pBlock->sversion;*/ + // TODO set to real sversion + int32_t sversion = 0; + if (pHandle->sver != sversion) { + pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); + + tb_uid_t quid; + STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = pHandle->pBlock->uid; + } + pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); + pHandle->sver = sversion; + } + + STSchema* pTschema = pHandle->pSchema; + SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + + int32_t numOfRows = pHandle->pBlock->numOfRows; + int32_t numOfCols = pHandle->pSchema->numOfCols; + int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; } - SColumnInfoData colInfo; - int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; - colInfo.pData = malloc(sz); - if (colInfo.pData == NULL) { - return NULL; + + int j = 0; + for (int32_t i = 0; i < colNumNeed; i++) { + int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { + j++; + } + SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; + ASSERT(pColSchema->colId == colId); + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = calloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + taosArrayPush(pArray, &colInfo); } SMemRow row; - int32_t kvIdx; + int32_t kvIdx = 0; + tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { - for (int i = 0; i < pTschema->numOfCols && kvIdx < pTschema->numOfCols; i++) { - // TODO: filter out unused column - STColumn* pCol = schemaColAt(pTschema, i); + // get all wanted col of that block + for (int32_t i = 0; i < colNumNeed; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + STColumn* pCol = schemaColAt(pTschema, i); + // TODO + ASSERT(pCol->colId == pColData->info.colId); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - // TODO: handle varlen - memcpy(POINTER_SHIFT(colInfo.pData, pCol->offset), val, pCol->bytes); + memcpy(pColData->pData, val, pCol->bytes); } } - taosArrayPush(pArray, &colInfo); return pArray; } diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index d762844120..477deed8c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -26,6 +26,7 @@ int vnodeInit(const SVnodeOpt *pOption) { vnodeMgr.stop = false; vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; + vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp; // Start commit handers if (pOption->nthreads > 0) { @@ -96,6 +97,10 @@ int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } +void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { + (*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq); +} + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { setThreadName("vnode-commit"); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index b211894281..6692264c54 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -21,7 +21,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutReqToVQueryQ); + (putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode); } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -43,6 +43,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_FETCH_RSP: + return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_RES_READY: return qWorkerProcessReadyMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_TASKS_STATUS: diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6b68ee87e5..f5487f7da8 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -65,7 +65,6 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; ctgDebug("try to get db vgroup from mnode, db:%s", input->db); @@ -216,17 +215,6 @@ int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableN return TSDB_CODE_SUCCESS; } - -void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { - epSet->inUse = 0; - epSet->numOfEps = vgroupInfo->numOfEps; - - for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) { - memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i])); - memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i])); - } -} - int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; char *msg = NULL; @@ -292,7 +280,6 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); @@ -308,10 +295,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, }; SRpcMsg rpcRsp = {0}; - SEpSet epSet; - - ctgGenEpSet(&epSet, vgroupInfo); - rpcSendRecv(pTransporter, &epSet, &rpcMsg, &rpcRsp); + rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 35272c0dce..c6e5e359bf 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -195,10 +195,10 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { vgInfo.vgId = i + 1; vgInfo.hashBegin = i * hashUnit; vgInfo.hashEnd = hashUnit * (i + 1) - 1; - vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1; - vgInfo.inUse = i % vgInfo.numOfEps; - for (int32_t n = 0; n < vgInfo.numOfEps; ++n) { - SEpAddr *addr = &vgInfo.epAddr[n]; + vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vgInfo.epset.inUse = i % vgInfo.epset.numOfEps; + for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) { + SEp *addr = &vgInfo.epset.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } @@ -229,10 +229,10 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM vg->vgId = htonl(i + 1); vg->hashBegin = htonl(i * hashUnit); vg->hashEnd = htonl(hashUnit * (i + 1) - 1); - vg->numOfEps = i % TSDB_MAX_REPLICA + 1; - vg->inUse = i % vg->numOfEps; - for (int32_t n = 0; n < vg->numOfEps; ++n) { - SEpAddr *addr = &vg->epAddr[n]; + vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg->epset.inUse = i % vg->epset.numOfEps; + for (int32_t n = 0; n < vg->epset.numOfEps; ++n) { + SEp *addr = &vg->epset.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } @@ -693,7 +693,7 @@ TEST(tableMeta, normalTable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.numOfEps, 3); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); ctgTestSetPrepareTableMeta(); @@ -983,7 +983,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 8); - ASSERT_EQ(vgInfo->numOfEps, 3); + ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); } @@ -1015,7 +1015,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 9); - ASSERT_EQ(vgInfo->numOfEps, 4); + ASSERT_EQ(vgInfo->epset.numOfEps, 4); catalogDestroy(); } @@ -1046,13 +1046,13 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 1); - ASSERT_EQ(vgInfo->numOfEps, 1); + ASSERT_EQ(vgInfo->epset.numOfEps, 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1); ASSERT_EQ(vgInfo->vgId, 2); - ASSERT_EQ(vgInfo->numOfEps, 2); + ASSERT_EQ(vgInfo->epset.numOfEps, 2); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2); ASSERT_EQ(vgInfo->vgId, 3); - ASSERT_EQ(vgInfo->numOfEps, 3); + ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); } @@ -1088,14 +1088,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.numOfEps, 3); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->numOfEps, 3); + ASSERT_EQ(pvgInfo->epset.numOfEps, 3); taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); @@ -1105,14 +1105,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); - ASSERT_EQ(vgInfo.numOfEps, 2); + ASSERT_EQ(vgInfo.epset.numOfEps, 2); code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->numOfEps, 3); + ASSERT_EQ(pvgInfo->epset.numOfEps, 3); taosArrayDestroy(vgList); catalogDestroy(); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 45d1f57e68..6efd7ef87e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5070,6 +5070,7 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { SStreamBlockScanInfo* pInfo = pOperator->info; SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + pBlockInfo->rows = 0; while (tqNextDataBlock(pInfo->readerHandle)) { pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { @@ -5115,7 +5116,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody); } -void processRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { +void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle; assert(pMsg->ahandle != NULL); @@ -5163,14 +5164,9 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - SEpSet epSet = {0}; - epSet.numOfEps = pSource->addr.numOfEps; - epSet.port[0] = pSource->addr.epAddr[0].port; - tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - int64_t startTs = taosGetTimestampUs(); qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, pExchangeInfo->current, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); @@ -5192,7 +5188,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); tsem_wait(&pExchangeInfo->ready); SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; @@ -5296,13 +5292,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->exec = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; +#if 1 { // todo refactor SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "EX"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processRspMsg; + rpcInit.cfp = qProcessFetchRsp; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; @@ -5316,7 +5313,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return NULL; // todo } } - +#endif return pOperator; } @@ -5467,6 +5464,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp return NULL; } + // todo dynamic set the value of 4096 + pInfo->pRes = createOutputBuf_rv(pExprInfo, 4096); + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo); SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); for(int32_t i = 0; i < numOfOutput; ++i) { @@ -5897,7 +5897,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); - return pInfo->pRes; + return (pInfo->pRes->info.rows != 0)? pInfo->pRes:NULL; } static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { @@ -7431,7 +7431,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; -// pOperator->operatorType = OP_TagScan; + pOperator->operatorType = OP_TagScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -8826,14 +8826,14 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { } void doDestroyTask(SExecTaskInfo *pTaskInfo) { + qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); + doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo); // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); tfree(pTaskInfo->sql); tfree(pTaskInfo->id.str); - qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); - tfree(pTaskInfo); } diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 7f77876c4c..417c56aba1 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util catalog function transport qcom + PRIVATE os util nodes catalog function transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h new file mode 100644 index 0000000000..dcdc6a045b --- /dev/null +++ b/source/libs/parser/inc/astCreateContext.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AST_CREATER_H_ +#define _TD_AST_CREATER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nodes.h" +#include "parser.h" + +typedef struct SAstCreateContext { + SParseContext* pQueryCxt; + bool notSupport; + bool valid; + SNode* pRootNode; +} SAstCreateContext; + +int32_t createAstCreateContext(const SParseContext* pQueryCxt, SAstCreateContext* pCxt); +int32_t destroyAstCreateContext(SAstCreateContext* pCxt); + +void* acquireRaii(SAstCreateContext* pCxt, void* p); +void* releaseRaii(SAstCreateContext* pCxt, void* p); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AST_CREATER_H_*/ diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h new file mode 100644 index 0000000000..c4c5dcba6c --- /dev/null +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" +#include "nodesShowStmts.h" +#include "astCreateContext.h" +#include "ttoken.h" + +#ifndef _TD_AST_CREATE_FUNCS_H_ +#define _TD_AST_CREATE_FUNCS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +bool checkTableName(const SToken* pTableName); +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); +SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); +SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); +SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName); +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); +SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AST_CREATE_FUNCS_H_*/ diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y new file mode 100644 index 0000000000..9237f63f35 --- /dev/null +++ b/source/libs/parser/inc/new_sql.y @@ -0,0 +1,199 @@ +//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql +//usage: lemon sql.y + +%name NewParse + +%token_prefix NEW_TK_ +%token_type { SToken } +%default_type { SNode* } +%default_destructor { nodesDestroyNode($$); } + +%extra_argument { SAstCreateContext* pCxt } + +%include { +#include +#include +#include +#include +#include + +#include "nodes.h" +#include "ttoken.h" +#include "ttokendef.h" +#include "astCreateFuncs.h" + +#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +} + +%syntax_error { + if(TOKEN.z) { + char msg[] = "syntax error near \"%s\""; + int32_t sqlLen = strlen(&TOKEN.z[0]); + + if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > pCxt->pQueryCxt->msgLen) { + char tmpstr[128] = {0}; + memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); + sprintf(pCxt->pQueryCxt->pMsg, msg, tmpstr); + } else { + sprintf(pCxt->pQueryCxt->pMsg, msg, &TOKEN.z[0]); + } + } else { + sprintf(pCxt->pQueryCxt->pMsg, "Incomplete SQL statement"); + } + pCxt->valid = false; +} + +%parse_accept { printf("parsing complete!\n" );} + +//%left OR. +//%left AND. +//%right NOT. +%left UNION ALL MINUS EXCEPT INTERSECT. +//%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. +//%left GT GE LT LE. +//%left BITAND BITOR LSHIFT RSHIFT. +%left NK_PLUS NK_MINUS. +//%left DIVIDE TIMES. +%left NK_STAR NK_SLASH. //REM. +//%left CONCAT. +//%right UMINUS UPLUS BITNOT. + +cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } + +cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } + +//////////////////////// value_function ///////////////////////////////// +value_function ::= NK_ID NK_LP value_expression NK_RP. +value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP. + +//////////////////////// value_expression_primary ///////////////////////////////// +value_expression_primary ::= NK_LP value_expression NK_RP. +value_expression_primary ::= nonparenthesized_value_expression_primary. + +nonparenthesized_value_expression_primary ::= literal. +// ? +nonparenthesized_value_expression_primary ::= column_reference. + +literal ::= NK_LITERAL. + +column_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } +column_reference(A) ::= table_name(B) NK_DOT NK_ID(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +//////////////////////// value_expression ///////////////////////////////// +value_expression ::= common_value_expression. + +common_value_expression ::= numeric_value_expression. + +numeric_value_expression ::= numeric_primary. +numeric_value_expression ::= NK_PLUS numeric_primary. +numeric_value_expression ::= NK_MINUS numeric_primary. +numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression. +numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression. +numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression. +numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression. + +numeric_primary ::= value_expression_primary. +numeric_primary ::= value_function. + +//////////////////////// query_specification ///////////////////////////////// +query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D). { PARSER_TRACE; A = createSelectStmt(pCxt, B, C, D); } + +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt {} +set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } +set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } +set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } + +%type select_list { SNodeList* } +%destructor select_list { nodesDestroyNodeList($$); } +select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } +select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } + +%type select_sublist { SNodeList* } +%destructor select_sublist { nodesDestroyNodeList($$); } +select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } +select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } + +//%type table_reference_list { SNodeList* } +//%destructor table_reference_list { nodesDestroyNodeList($$); } +table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } +//table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, B, C); } + +//table_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createRealTableNode(pCxt, ); } +table_reference(A) ::= table_factor(B). { PARSER_TRACE; A = B; } +//table_reference ::= joined_table. + +table_factor(A) ::= table_primary(B). { PARSER_TRACE; A = B; } + +table_primary(A) ::= table_name(B). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C); } +table_primary ::= derived_table. + +derived_table ::= table_subquery. + +%type db_name { SToken } +db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } +%type table_name { SToken } +table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +//////////////////////// subquery ///////////////////////////////// +subquery ::= NK_LR query_expression NK_RP. + +table_subquery ::= subquery. + +// query_expression +query_expression(A) ::= with_clause_opt query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + PARSER_TRACE; + addOrderByList(pCxt, B, C); + addSlimit(pCxt, B, D); + addLimit(pCxt, B, E); + A = B; + } + +// WITH AS +with_clause_opt ::= . {} + +query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } +query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } +query_primary(A) ::= NK_LP query_expression_body(B) order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { nodesDestroyNodeList($$); } +order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } + +slimit_clause_opt(A) ::= . { A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } + +//////////////////////// sort_specification_list ///////////////////////////////// +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { nodesDestroyNodeList($$); } +sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } + +%type ordering_specification_opt EOrder +%destructor ordering_specification_opt {} +ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } + +%type null_ordering_opt ENullOrder +%destructor null_ordering_opt {} +null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } diff --git a/source/nodes/src/nodesCode.c b/source/libs/parser/inc/parserImpl.h similarity index 68% rename from source/nodes/src/nodesCode.c rename to source/libs/parser/inc/parserImpl.h index 7fe919ffe8..57012e2fd6 100644 --- a/source/nodes/src/nodesCode.c +++ b/source/libs/parser/inc/parserImpl.h @@ -14,11 +14,23 @@ */ #include "nodes.h" +#include "parser.h" -int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { +#ifndef _TD_AST_CREATE_FUNCS_H_ +#define _TD_AST_CREATE_FUNCS_H_ +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SQuery { + SNode* pRoot; +} SQuery; + +int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); + +#ifdef __cplusplus } +#endif -int32_t stringToNode(const char* pStr, SNode** pNode) { - -} +#endif /*_TD_AST_CREATE_FUNCS_H_*/ diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c new file mode 100644 index 0000000000..42c5257360 --- /dev/null +++ b/source/libs/parser/src/astCreateContext.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "ttoken.h" +#include "astCreateContext.h" + +void* acquireRaii(SAstCreateContext* pCxt, void* p) { + if (NULL == p) { + return NULL; + } + return p; +} + +void* releaseRaii(SAstCreateContext* pCxt, void* p) { + if (NULL == p) { + return NULL; + } + return p; +} + +int32_t createAstCreater(const SParseContext* pQueryCxt, SAstCreateContext* pCxt) { + +} + +int32_t destroyAstCreater(SAstCreateContext* pCxt) { + +} + diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c new file mode 100644 index 0000000000..337ae78d75 --- /dev/null +++ b/source/libs/parser/src/astCreateFuncs.c @@ -0,0 +1,91 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "astCreateFuncs.h" + +#include "astCreateContext.h" + +bool checkTableName(const SToken* pTableName) { + printf("%p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); + return pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; +} + +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { + +} + +SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { + +} + +SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { + +} + +SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { + +} + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { + +} + +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { + +} + +SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { + +} + +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { + +} + +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName) { + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL != pDbName) { + printf("DbName %p : %d, %d, %s\n", pDbName, pDbName->type, pDbName->n, pDbName->z); + strncpy(realTable->dbName, pDbName->z, pDbName->n); + } + printf("TableName %p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + return acquireRaii(pCxt, realTable); +} + +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { + SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + select->isDistinct = isDistinct; + if (NULL == pProjectionList) { + select->isStar = true; + } + select->pProjectionList = releaseRaii(pCxt, pProjectionList); + printf("pTable = %p, name = %s\n", pTable, ((SRealTableNode*)pTable)->table.tableName); + select->pFromTable = releaseRaii(pCxt, pTable); + return acquireRaii(pCxt, select); +} + +SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { + +} + +SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { + +} + +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { + +} diff --git a/source/nodes/src/nodesUtil.c b/source/libs/parser/src/astTranslate.c similarity index 84% rename from source/nodes/src/nodesUtil.c rename to source/libs/parser/src/astTranslate.c index fe5883d809..ff68c4b22c 100644 --- a/source/nodes/src/nodesUtil.c +++ b/source/libs/parser/src/astTranslate.c @@ -13,12 +13,6 @@ * along with this program. If not, see . */ -#include "nodes.h" +// int32_t doTranslate() { -bool isTimeorderQuery(const SNode* pQuery) { - -} - -bool isTimelineQuery(const SNode* pQuery) { - -} +// } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 0a2f2b20f2..a56a6524fc 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3644,6 +3644,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) return TSDB_CODE_SUCCESS; } +//TODO remove it int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); @@ -3651,21 +3652,17 @@ int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgL return code; } - int32_t vgroupNum = taosArrayGetSize(vgroupList); + size_t vgroupNum = taosArrayGetSize(vgroupList); - SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupNum); - + SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); vgList->numOfVgroups = vgroupNum; for (int32_t i = 0; i < vgroupNum; ++i) { SVgroupInfo *vg = taosArrayGet(vgroupList, i); - vgList->vgroups[i].vgId = vg->vgId; - vgList->vgroups[i].numOfEps = vg->numOfEps; - memcpy(vgList->vgroups[i].epAddr, vg->epAddr, sizeof(vgList->vgroups[i].epAddr)); + vgList->vgroups[i] = *vg; } *pVgList = vgList; - taosArrayDestroy(vgroupList); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 06a9a3d16e..3ae89bca0a 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -58,13 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - pEpSet->numOfEps = info->numOfEps; - pEpSet->inUse = info->inUse; - - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - pEpSet->port[i] = info->epAddr[i].port; - } + *pEpSet = info->epset; *outputLen = sizeof(SVShowTablesReq); *output = pShowReq; diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c new file mode 100644 index 0000000000..7bf29c45f8 --- /dev/null +++ b/source/libs/parser/src/new_sql.c @@ -0,0 +1,1718 @@ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +#include +#include +/************ Begin %include sections from the grammar ************************/ + +#include +#include +#include +#include +#include + +#include "nodes.h" +#include "ttoken.h" +#include "ttokendef.h" +#include "astCreateFuncs.h" + +#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** NewParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is NewParseTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** NewParseARG_SDECL A static variable declaration for the %extra_argument +** NewParseARG_PDECL A parameter declaration for the %extra_argument +** NewParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** NewParseARG_STORE Code to store %extra_argument into yypParser +** NewParseARG_FETCH Code to extract %extra_argument from yypParser +** NewParseCTX_* As NewParseARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define YYCODETYPE unsigned char +#define YYNOCODE 71 +#define YYACTIONTYPE unsigned char +#define NewParseTOKENTYPE SToken +typedef union { + int yyinit; + NewParseTOKENTYPE yy0; + bool yy9; + SNodeList* yy30; + SToken yy67; + ENullOrder yy68; + EOrder yy108; + SNode* yy130; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define NewParseARG_SDECL SAstCreateContext* pCxt ; +#define NewParseARG_PDECL , SAstCreateContext* pCxt +#define NewParseARG_PARAM ,pCxt +#define NewParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; +#define NewParseARG_STORE yypParser->pCxt =pCxt ; +#define NewParseCTX_SDECL +#define NewParseCTX_PDECL +#define NewParseCTX_PARAM +#define NewParseCTX_FETCH +#define NewParseCTX_STORE +#define YYNSTATE 63 +#define YYNRULE 68 +#define YYNTOKEN 35 +#define YY_MAX_SHIFT 62 +#define YY_MIN_SHIFTREDUCE 114 +#define YY_MAX_SHIFTREDUCE 181 +#define YY_ERROR_ACTION 182 +#define YY_ACCEPT_ACTION 183 +#define YY_NO_ACTION 184 +#define YY_MIN_REDUCE 185 +#define YY_MAX_REDUCE 252 +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (229) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 30, 50, 30, 30, 30, 30, 55, 50, 30, 30, + /* 10 */ 10, 9, 29, 62, 51, 195, 30, 19, 30, 30, + /* 20 */ 30, 30, 57, 19, 30, 30, 30, 19, 30, 30, + /* 30 */ 30, 30, 57, 19, 30, 30, 30, 50, 30, 30, + /* 40 */ 30, 30, 55, 50, 30, 30, 39, 222, 155, 156, + /* 50 */ 23, 196, 12, 11, 10, 9, 25, 223, 30, 58, + /* 60 */ 30, 30, 30, 30, 57, 58, 30, 30, 30, 59, + /* 70 */ 30, 30, 30, 30, 57, 59, 30, 30, 30, 37, + /* 80 */ 30, 30, 30, 30, 57, 37, 30, 30, 204, 38, + /* 90 */ 5, 183, 60, 166, 158, 159, 121, 53, 200, 201, + /* 100 */ 202, 203, 54, 203, 203, 203, 246, 18, 246, 246, + /* 110 */ 246, 246, 57, 120, 246, 246, 245, 17, 245, 245, + /* 120 */ 245, 245, 57, 46, 245, 245, 35, 27, 35, 35, + /* 130 */ 35, 35, 57, 31, 35, 35, 36, 47, 36, 36, + /* 140 */ 36, 36, 57, 128, 36, 36, 242, 117, 242, 242, + /* 150 */ 242, 242, 57, 42, 241, 242, 241, 241, 241, 241, + /* 160 */ 57, 52, 212, 241, 14, 13, 122, 1, 212, 212, + /* 170 */ 38, 5, 14, 13, 166, 40, 224, 20, 38, 5, + /* 180 */ 15, 210, 166, 211, 21, 22, 17, 210, 210, 43, + /* 190 */ 160, 6, 7, 45, 41, 26, 207, 48, 24, 208, + /* 200 */ 136, 44, 205, 8, 189, 3, 142, 2, 16, 127, + /* 210 */ 147, 146, 32, 150, 49, 149, 4, 33, 180, 206, + /* 220 */ 28, 117, 34, 186, 56, 162, 161, 185, 61, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + /* 10 */ 8, 9, 49, 10, 51, 52, 37, 38, 39, 40, + /* 20 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, + /* 30 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, + /* 40 */ 41, 42, 43, 44, 45, 46, 67, 68, 30, 31, + /* 50 */ 69, 52, 6, 7, 8, 9, 63, 68, 37, 38, + /* 60 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, + /* 70 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, + /* 80 */ 39, 40, 41, 42, 43, 44, 45, 46, 43, 12, + /* 90 */ 13, 35, 36, 16, 33, 34, 2, 12, 53, 54, + /* 100 */ 55, 56, 57, 58, 59, 60, 37, 22, 39, 40, + /* 110 */ 41, 42, 43, 19, 45, 46, 37, 61, 39, 40, + /* 120 */ 41, 42, 43, 15, 45, 46, 37, 63, 39, 40, + /* 130 */ 41, 42, 43, 32, 45, 46, 37, 29, 39, 40, + /* 140 */ 41, 42, 43, 8, 45, 46, 37, 12, 39, 40, + /* 150 */ 41, 42, 43, 1, 37, 46, 39, 40, 41, 42, + /* 160 */ 43, 36, 47, 46, 6, 7, 8, 48, 47, 47, + /* 170 */ 12, 13, 6, 7, 16, 23, 70, 62, 12, 13, + /* 180 */ 13, 66, 16, 62, 62, 18, 61, 66, 66, 15, + /* 190 */ 14, 15, 13, 25, 64, 64, 17, 28, 65, 65, + /* 200 */ 12, 27, 43, 21, 50, 15, 14, 24, 2, 12, + /* 210 */ 26, 26, 26, 26, 20, 26, 15, 26, 14, 17, + /* 220 */ 17, 12, 17, 0, 17, 14, 14, 0, 11, 71, + /* 230 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + /* 240 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + /* 250 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + /* 260 */ 71, 71, 71, 71, +}; +#define YY_SHIFT_COUNT (62) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (227) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 3, 158, 166, 166, 166, 166, 166, 166, 85, 166, + /* 10 */ 166, 166, 166, 77, 77, 167, 167, 167, 229, 18, + /* 20 */ 152, 152, 94, 101, 168, 169, 169, 168, 188, 182, + /* 30 */ 46, 61, 174, 108, 135, 2, 2, 176, 179, 190, + /* 40 */ 183, 192, 206, 184, 185, 186, 187, 189, 191, 197, + /* 50 */ 194, 201, 204, 202, 203, 205, 209, 207, 211, 212, + /* 60 */ 223, 227, 217, +}; +#define YY_REDUCE_COUNT (29) +#define YY_REDUCE_MIN (-37) +#define YY_REDUCE_MAX (159) +static const short yy_reduce_ofst[] = { + /* 0 */ 56, -37, -21, -11, -1, 21, 31, 41, 45, 69, + /* 10 */ 79, 89, 99, 109, 117, 115, 121, 122, 125, -19, + /* 20 */ -7, 64, 119, 106, 130, 133, 134, 131, 159, 154, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 209, 182, 182, 182, 182, 182, 182, 182, 182, 182, + /* 10 */ 182, 182, 182, 182, 182, 182, 182, 182, 209, 225, + /* 20 */ 214, 214, 190, 228, 216, 219, 219, 216, 182, 182, + /* 30 */ 239, 182, 182, 182, 182, 244, 243, 182, 187, 215, + /* 40 */ 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + /* 50 */ 197, 194, 182, 207, 182, 182, 182, 182, 182, 182, + /* 60 */ 182, 182, 182, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + NewParseARG_SDECL /* A place to hold %extra_argument */ + NewParseCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +#include +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "UNION", + /* 2 */ "ALL", + /* 3 */ "MINUS", + /* 4 */ "EXCEPT", + /* 5 */ "INTERSECT", + /* 6 */ "NK_PLUS", + /* 7 */ "NK_MINUS", + /* 8 */ "NK_STAR", + /* 9 */ "NK_SLASH", + /* 10 */ "SHOW", + /* 11 */ "DATABASES", + /* 12 */ "NK_ID", + /* 13 */ "NK_LP", + /* 14 */ "NK_RP", + /* 15 */ "NK_COMMA", + /* 16 */ "NK_LITERAL", + /* 17 */ "NK_DOT", + /* 18 */ "SELECT", + /* 19 */ "DISTINCT", + /* 20 */ "AS", + /* 21 */ "FROM", + /* 22 */ "NK_LR", + /* 23 */ "ORDER", + /* 24 */ "BY", + /* 25 */ "SLIMIT", + /* 26 */ "NK_INTEGER", + /* 27 */ "SOFFSET", + /* 28 */ "LIMIT", + /* 29 */ "OFFSET", + /* 30 */ "ASC", + /* 31 */ "DESC", + /* 32 */ "NULLS", + /* 33 */ "FIRST", + /* 34 */ "LAST", + /* 35 */ "cmd", + /* 36 */ "query_expression", + /* 37 */ "value_function", + /* 38 */ "value_expression", + /* 39 */ "value_expression_primary", + /* 40 */ "nonparenthesized_value_expression_primary", + /* 41 */ "literal", + /* 42 */ "column_reference", + /* 43 */ "table_name", + /* 44 */ "common_value_expression", + /* 45 */ "numeric_value_expression", + /* 46 */ "numeric_primary", + /* 47 */ "query_specification", + /* 48 */ "set_quantifier_opt", + /* 49 */ "select_list", + /* 50 */ "from_clause", + /* 51 */ "select_sublist", + /* 52 */ "select_item", + /* 53 */ "table_reference_list", + /* 54 */ "table_reference", + /* 55 */ "table_factor", + /* 56 */ "table_primary", + /* 57 */ "db_name", + /* 58 */ "derived_table", + /* 59 */ "table_subquery", + /* 60 */ "subquery", + /* 61 */ "with_clause_opt", + /* 62 */ "query_expression_body", + /* 63 */ "order_by_clause_opt", + /* 64 */ "slimit_clause_opt", + /* 65 */ "limit_clause_opt", + /* 66 */ "query_primary", + /* 67 */ "sort_specification_list", + /* 68 */ "sort_specification", + /* 69 */ "ordering_specification_opt", + /* 70 */ "null_ordering_opt", +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "cmd ::= SHOW DATABASES", + /* 1 */ "cmd ::= query_expression", + /* 2 */ "column_reference ::= NK_ID", + /* 3 */ "column_reference ::= table_name NK_DOT NK_ID", + /* 4 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause", + /* 5 */ "set_quantifier_opt ::=", + /* 6 */ "set_quantifier_opt ::= DISTINCT", + /* 7 */ "set_quantifier_opt ::= ALL", + /* 8 */ "select_list ::= NK_STAR", + /* 9 */ "select_list ::= select_sublist", + /* 10 */ "select_sublist ::= select_item", + /* 11 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 12 */ "select_item ::= value_expression", + /* 13 */ "select_item ::= value_expression AS NK_ID", + /* 14 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 15 */ "from_clause ::= FROM table_reference_list", + /* 16 */ "table_reference_list ::= table_reference", + /* 17 */ "table_reference ::= table_factor", + /* 18 */ "table_factor ::= table_primary", + /* 19 */ "table_primary ::= table_name", + /* 20 */ "table_primary ::= db_name NK_DOT table_name", + /* 21 */ "db_name ::= NK_ID", + /* 22 */ "table_name ::= NK_ID", + /* 23 */ "query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 24 */ "with_clause_opt ::=", + /* 25 */ "query_expression_body ::= query_primary", + /* 26 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 27 */ "query_primary ::= query_specification", + /* 28 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", + /* 29 */ "order_by_clause_opt ::=", + /* 30 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 31 */ "slimit_clause_opt ::=", + /* 32 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 33 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 34 */ "limit_clause_opt ::=", + /* 35 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 36 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 37 */ "sort_specification_list ::= sort_specification", + /* 38 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 39 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", + /* 40 */ "ordering_specification_opt ::=", + /* 41 */ "ordering_specification_opt ::= ASC", + /* 42 */ "ordering_specification_opt ::= DESC", + /* 43 */ "null_ordering_opt ::=", + /* 44 */ "null_ordering_opt ::= NULLS FIRST", + /* 45 */ "null_ordering_opt ::= NULLS LAST", + /* 46 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", + /* 47 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", + /* 48 */ "value_expression_primary ::= NK_LP value_expression NK_RP", + /* 49 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", + /* 50 */ "nonparenthesized_value_expression_primary ::= literal", + /* 51 */ "nonparenthesized_value_expression_primary ::= column_reference", + /* 52 */ "literal ::= NK_LITERAL", + /* 53 */ "value_expression ::= common_value_expression", + /* 54 */ "common_value_expression ::= numeric_value_expression", + /* 55 */ "numeric_value_expression ::= numeric_primary", + /* 56 */ "numeric_value_expression ::= NK_PLUS numeric_primary", + /* 57 */ "numeric_value_expression ::= NK_MINUS numeric_primary", + /* 58 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", + /* 59 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", + /* 60 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", + /* 61 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", + /* 62 */ "numeric_primary ::= value_expression_primary", + /* 63 */ "numeric_primary ::= value_function", + /* 64 */ "table_primary ::= derived_table", + /* 65 */ "derived_table ::= table_subquery", + /* 66 */ "subquery ::= NK_LR query_expression NK_RP", + /* 67 */ "table_subquery ::= subquery", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to NewParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + NewParseCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef NewParse_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to NewParse and NewParseFree. +*/ +void *NewParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) NewParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + NewParseCTX_STORE + NewParseInit(yypParser NewParseCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* NewParse_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + /* Default NON-TERMINAL Destructor */ + case 35: /* cmd */ + case 36: /* query_expression */ + case 37: /* value_function */ + case 38: /* value_expression */ + case 39: /* value_expression_primary */ + case 40: /* nonparenthesized_value_expression_primary */ + case 41: /* literal */ + case 42: /* column_reference */ + case 43: /* table_name */ + case 44: /* common_value_expression */ + case 45: /* numeric_value_expression */ + case 46: /* numeric_primary */ + case 47: /* query_specification */ + case 50: /* from_clause */ + case 52: /* select_item */ + case 53: /* table_reference_list */ + case 54: /* table_reference */ + case 55: /* table_factor */ + case 56: /* table_primary */ + case 57: /* db_name */ + case 58: /* derived_table */ + case 59: /* table_subquery */ + case 60: /* subquery */ + case 61: /* with_clause_opt */ + case 62: /* query_expression_body */ + case 64: /* slimit_clause_opt */ + case 65: /* limit_clause_opt */ + case 66: /* query_primary */ + case 68: /* sort_specification */ +{ + nodesDestroyNode((yypminor->yy130)); +} + break; + case 48: /* set_quantifier_opt */ +{ + +} + break; + case 49: /* select_list */ + case 51: /* select_sublist */ + case 63: /* order_by_clause_opt */ + case 67: /* sort_specification_list */ +{ + nodesDestroyNodeList((yypminor->yy30)); +} + break; + case 69: /* ordering_specification_opt */ +{ + +} + break; + case 70: /* null_ordering_opt */ +{ + +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +void NewParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef NewParse_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void NewParseFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + NewParseFinalize(p); + (*freeProc)(p); +} +#endif /* NewParse_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int NewParseStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int NewParseCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j0 + ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +/******** End %stack_overflow code ********************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument var */ + NewParseCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + NewParseTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + signed char nrhs; /* Negative of the number of RHS symbols in the rule */ +} yyRuleInfo[] = { + { 35, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 35, -1 }, /* (1) cmd ::= query_expression */ + { 42, -1 }, /* (2) column_reference ::= NK_ID */ + { 42, -3 }, /* (3) column_reference ::= table_name NK_DOT NK_ID */ + { 47, -4 }, /* (4) query_specification ::= SELECT set_quantifier_opt select_list from_clause */ + { 48, 0 }, /* (5) set_quantifier_opt ::= */ + { 48, -1 }, /* (6) set_quantifier_opt ::= DISTINCT */ + { 48, -1 }, /* (7) set_quantifier_opt ::= ALL */ + { 49, -1 }, /* (8) select_list ::= NK_STAR */ + { 49, -1 }, /* (9) select_list ::= select_sublist */ + { 51, -1 }, /* (10) select_sublist ::= select_item */ + { 51, -3 }, /* (11) select_sublist ::= select_sublist NK_COMMA select_item */ + { 52, -1 }, /* (12) select_item ::= value_expression */ + { 52, -3 }, /* (13) select_item ::= value_expression AS NK_ID */ + { 52, -3 }, /* (14) select_item ::= table_name NK_DOT NK_STAR */ + { 50, -2 }, /* (15) from_clause ::= FROM table_reference_list */ + { 53, -1 }, /* (16) table_reference_list ::= table_reference */ + { 54, -1 }, /* (17) table_reference ::= table_factor */ + { 55, -1 }, /* (18) table_factor ::= table_primary */ + { 56, -1 }, /* (19) table_primary ::= table_name */ + { 56, -3 }, /* (20) table_primary ::= db_name NK_DOT table_name */ + { 57, -1 }, /* (21) db_name ::= NK_ID */ + { 43, -1 }, /* (22) table_name ::= NK_ID */ + { 36, -5 }, /* (23) query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 61, 0 }, /* (24) with_clause_opt ::= */ + { 62, -1 }, /* (25) query_expression_body ::= query_primary */ + { 62, -4 }, /* (26) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 66, -1 }, /* (27) query_primary ::= query_specification */ + { 66, -6 }, /* (28) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ + { 63, 0 }, /* (29) order_by_clause_opt ::= */ + { 63, -3 }, /* (30) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 64, 0 }, /* (31) slimit_clause_opt ::= */ + { 64, -4 }, /* (32) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 64, -4 }, /* (33) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 65, 0 }, /* (34) limit_clause_opt ::= */ + { 65, -4 }, /* (35) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 65, -4 }, /* (36) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 67, -1 }, /* (37) sort_specification_list ::= sort_specification */ + { 67, -3 }, /* (38) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 68, -3 }, /* (39) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ + { 69, 0 }, /* (40) ordering_specification_opt ::= */ + { 69, -1 }, /* (41) ordering_specification_opt ::= ASC */ + { 69, -1 }, /* (42) ordering_specification_opt ::= DESC */ + { 70, 0 }, /* (43) null_ordering_opt ::= */ + { 70, -2 }, /* (44) null_ordering_opt ::= NULLS FIRST */ + { 70, -2 }, /* (45) null_ordering_opt ::= NULLS LAST */ + { 37, -4 }, /* (46) value_function ::= NK_ID NK_LP value_expression NK_RP */ + { 37, -6 }, /* (47) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ + { 39, -3 }, /* (48) value_expression_primary ::= NK_LP value_expression NK_RP */ + { 39, -1 }, /* (49) value_expression_primary ::= nonparenthesized_value_expression_primary */ + { 40, -1 }, /* (50) nonparenthesized_value_expression_primary ::= literal */ + { 40, -1 }, /* (51) nonparenthesized_value_expression_primary ::= column_reference */ + { 41, -1 }, /* (52) literal ::= NK_LITERAL */ + { 38, -1 }, /* (53) value_expression ::= common_value_expression */ + { 44, -1 }, /* (54) common_value_expression ::= numeric_value_expression */ + { 45, -1 }, /* (55) numeric_value_expression ::= numeric_primary */ + { 45, -2 }, /* (56) numeric_value_expression ::= NK_PLUS numeric_primary */ + { 45, -2 }, /* (57) numeric_value_expression ::= NK_MINUS numeric_primary */ + { 45, -3 }, /* (58) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ + { 45, -3 }, /* (59) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ + { 45, -3 }, /* (60) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ + { 45, -3 }, /* (61) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ + { 46, -1 }, /* (62) numeric_primary ::= value_expression_primary */ + { 46, -1 }, /* (63) numeric_primary ::= value_function */ + { 56, -1 }, /* (64) table_primary ::= derived_table */ + { 58, -1 }, /* (65) derived_table ::= table_subquery */ + { 60, -3 }, /* (66) subquery ::= NK_LR query_expression NK_RP */ + { 59, -1 }, /* (67) table_subquery ::= subquery */ +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + NewParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + NewParseCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + NewParseARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; +#ifndef NDEBUG + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + yysize = yyRuleInfo[yyruleno].nrhs; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s].\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno]); + } + } +#endif /* NDEBUG */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfo[yyruleno].nrhs==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } + yymsp = yypParser->yytos; + } +#endif + } + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* cmd ::= SHOW DATABASES */ +{ PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } + break; + case 1: /* cmd ::= query_expression */ +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy130; } + break; + case 2: /* column_reference ::= NK_ID */ +{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy130 = yylhsminor.yy130; + break; + case 3: /* column_reference ::= table_name NK_DOT NK_ID */ + case 14: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==14); +{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy130 = yylhsminor.yy130; + break; + case 4: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause */ +{ PARSER_TRACE; yymsp[-3].minor.yy130 = createSelectStmt(pCxt, yymsp[-2].minor.yy9, yymsp[-1].minor.yy30, yymsp[0].minor.yy130); } + break; + case 5: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy9 = false; } + break; + case 6: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy9 = true; } + break; + case 7: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy9 = false; } + break; + case 8: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy30 = NULL; } + break; + case 9: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy30 = yymsp[0].minor.yy30; } + yymsp[0].minor.yy30 = yylhsminor.yy30; + break; + case 10: /* select_sublist ::= select_item */ + case 37: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==37); +{ PARSER_TRACE; yylhsminor.yy30 = createNodeList(pCxt, yymsp[0].minor.yy130); } + yymsp[0].minor.yy30 = yylhsminor.yy30; + break; + case 11: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 38: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==38); +{ PARSER_TRACE; yylhsminor.yy30 = addNodeToList(pCxt, yymsp[-2].minor.yy30, yymsp[0].minor.yy130); } + yymsp[-2].minor.yy30 = yylhsminor.yy30; + break; + case 12: /* select_item ::= value_expression */ + case 16: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==16); + case 17: /* table_reference ::= table_factor */ yytestcase(yyruleno==17); + case 18: /* table_factor ::= table_primary */ yytestcase(yyruleno==18); + case 25: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==25); + case 27: /* query_primary ::= query_specification */ yytestcase(yyruleno==27); +{ PARSER_TRACE; yylhsminor.yy130 = yymsp[0].minor.yy130; } + yymsp[0].minor.yy130 = yylhsminor.yy130; + break; + case 13: /* select_item ::= value_expression AS NK_ID */ +{ PARSER_TRACE; yylhsminor.yy130 = setProjectionAlias(pCxt, yymsp[-2].minor.yy130, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy130 = yylhsminor.yy130; + break; + case 15: /* from_clause ::= FROM table_reference_list */ +{ PARSER_TRACE; yymsp[-1].minor.yy130 = yymsp[0].minor.yy130; } + break; + case 19: /* table_primary ::= table_name */ +{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy67); } + yymsp[0].minor.yy130 = yylhsminor.yy130; + break; + case 20: /* table_primary ::= db_name NK_DOT table_name */ +{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy67); } + yymsp[-2].minor.yy130 = yylhsminor.yy130; + break; + case 21: /* db_name ::= NK_ID */ + case 22: /* table_name ::= NK_ID */ yytestcase(yyruleno==22); +{ PARSER_TRACE; yylhsminor.yy67 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy67 = yylhsminor.yy67; + break; + case 23: /* query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ yy_destructor(yypParser,61,&yymsp[-4].minor); +{ + PARSER_TRACE; + addOrderByList(pCxt, yymsp[-3].minor.yy130, yymsp[-2].minor.yy30); + addSlimit(pCxt, yymsp[-3].minor.yy130, yymsp[-1].minor.yy130); + addLimit(pCxt, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); + yymsp[-4].minor.yy130 = yymsp[-3].minor.yy130; + } +} + break; + case 24: /* with_clause_opt ::= */ +{} + break; + case 26: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy130 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); } + yymsp[-3].minor.yy130 = yylhsminor.yy130; + break; + case 28: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy130 = yymsp[-4].minor.yy130;} + yy_destructor(yypParser,63,&yymsp[-3].minor); + yy_destructor(yypParser,65,&yymsp[-2].minor); + yy_destructor(yypParser,64,&yymsp[-1].minor); + break; + case 29: /* order_by_clause_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy30 = NULL; } + break; + case 30: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ +{ PARSER_TRACE; yymsp[-2].minor.yy30 = yymsp[0].minor.yy30; } + break; + case 31: /* slimit_clause_opt ::= */ + case 34: /* limit_clause_opt ::= */ yytestcase(yyruleno==34); +{ yymsp[1].minor.yy130 = NULL; } + break; + case 32: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 35: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==35); +{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 33: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 36: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==36); +{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + break; + case 39: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy130 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy130, yymsp[-1].minor.yy108, yymsp[0].minor.yy68); } + yymsp[-2].minor.yy130 = yylhsminor.yy130; + break; + case 40: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy108 = ORDER_ASC; } + break; + case 41: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_ASC; } + break; + case 42: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_DESC; } + break; + case 43: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy68 = NULL_ORDER_DEFAULT; } + break; + case 44: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_FIRST; } + break; + case 45: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_LAST; } + break; + case 46: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ + case 48: /* value_expression_primary ::= NK_LP value_expression NK_RP */ yytestcase(yyruleno==48); +{ +} + yy_destructor(yypParser,38,&yymsp[-1].minor); + break; + case 47: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ +{ +} + yy_destructor(yypParser,38,&yymsp[-3].minor); + yy_destructor(yypParser,38,&yymsp[-1].minor); + break; + case 49: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ +{ yy_destructor(yypParser,40,&yymsp[0].minor); +{ +} +} + break; + case 50: /* nonparenthesized_value_expression_primary ::= literal */ +{ yy_destructor(yypParser,41,&yymsp[0].minor); +{ +} +} + break; + case 51: /* nonparenthesized_value_expression_primary ::= column_reference */ +{ yy_destructor(yypParser,42,&yymsp[0].minor); +{ +} +} + break; + case 53: /* value_expression ::= common_value_expression */ +{ yy_destructor(yypParser,44,&yymsp[0].minor); +{ +} +} + break; + case 54: /* common_value_expression ::= numeric_value_expression */ +{ yy_destructor(yypParser,45,&yymsp[0].minor); +{ +} +} + break; + case 55: /* numeric_value_expression ::= numeric_primary */ +{ yy_destructor(yypParser,46,&yymsp[0].minor); +{ +} +} + break; + case 56: /* numeric_value_expression ::= NK_PLUS numeric_primary */ + case 57: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==57); +{ +} + yy_destructor(yypParser,46,&yymsp[0].minor); + break; + case 58: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ + case 59: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==59); + case 60: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==60); + case 61: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==61); +{ yy_destructor(yypParser,45,&yymsp[-2].minor); +{ +} + yy_destructor(yypParser,45,&yymsp[0].minor); +} + break; + case 62: /* numeric_primary ::= value_expression_primary */ +{ yy_destructor(yypParser,39,&yymsp[0].minor); +{ +} +} + break; + case 63: /* numeric_primary ::= value_function */ +{ yy_destructor(yypParser,37,&yymsp[0].minor); +{ +} +} + break; + case 64: /* table_primary ::= derived_table */ +{ yy_destructor(yypParser,58,&yymsp[0].minor); +{ +} +} + break; + case 65: /* derived_table ::= table_subquery */ +{ yy_destructor(yypParser,59,&yymsp[0].minor); +{ +} +} + break; + case 66: /* subquery ::= NK_LR query_expression NK_RP */ +{ +} + yy_destructor(yypParser,36,&yymsp[-1].minor); + break; + case 67: /* table_subquery ::= subquery */ +{ yy_destructor(yypParser,60,&yymsp[0].minor); +{ +} +} + break; + default: + /* (52) literal ::= NK_LITERAL */ yytestcase(yyruleno==52); + break; +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + NewParseTOKENTYPE yyminor /* The minor type of the error token */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ + + if(TOKEN.z) { + char msg[] = "syntax error near \"%s\""; + int32_t sqlLen = strlen(&TOKEN.z[0]); + + if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > pCxt->pQueryCxt->msgLen) { + char tmpstr[128] = {0}; + memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); + sprintf(pCxt->pQueryCxt->pMsg, msg, tmpstr); + } else { + sprintf(pCxt->pQueryCxt->pMsg, msg, &TOKEN.z[0]); + } + } else { + sprintf(pCxt->pQueryCxt->pMsg, "Incomplete SQL statement"); + } + pCxt->valid = false; +/************ End %syntax_error code ******************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ + printf("parsing complete!\n" ); +/*********** End %parse_accept code *******************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "NewParseAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +void NewParse( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + NewParseTOKENTYPE yyminor /* The value for the token */ + NewParseARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + NewParseCTX_FETCH + NewParseARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + do{ + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor NewParseCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos >= yypParser->yystack + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + }while( yypParser->yytos>yypParser->yystack ); +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int NewParseFallback(int iToken){ +#ifdef YYFALLBACK + if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ + return yyFallback[iToken]; + } +#else + (void)iToken; +#endif + return 0; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index c98f787f0b..3b79a2de92 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -249,7 +249,7 @@ void qDestroyQuery(SQueryNode* pQueryNode) { return; } - int32_t type = nodeType(pQueryNode); + int32_t type = queryNodeType(pQueryNode); if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c new file mode 100644 index 0000000000..8902da9274 --- /dev/null +++ b/source/libs/parser/src/parserImpl.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parserImpl.h" + +#include "ttoken.h" +#include "astCreateContext.h" + +typedef void* (*FMalloc)(size_t); +typedef void (*FFree)(void*); + +extern void* NewParseAlloc(FMalloc); +extern void NewParse(void*, int, SToken, void*); +extern void NewParseFree(void*, FFree); + +uint32_t toNewTokenId(uint32_t tokenId) { + switch (tokenId) { + case TK_UNION: + return NEW_TK_UNION; + case TK_ALL: + return NEW_TK_ALL; + case TK_MINUS: + return NEW_TK_NK_MINUS; + case TK_PLUS: + return NEW_TK_NK_PLUS; + case TK_STAR: + return NEW_TK_NK_STAR; + case TK_SLASH: + return NEW_TK_NK_SLASH; + case TK_SHOW: + return NEW_TK_SHOW; + case TK_DATABASES: + return NEW_TK_DATABASES; + case TK_ID: + return NEW_TK_NK_ID; + case TK_LP: + return NEW_TK_NK_LP; + case TK_RP: + return NEW_TK_NK_RP; + case TK_COMMA: + return NEW_TK_NK_COMMA; + case TK_DOT: + return NEW_TK_NK_DOT; + case TK_SELECT: + return NEW_TK_SELECT; + case TK_DISTINCT: + return NEW_TK_DISTINCT; + case TK_AS: + return NEW_TK_AS; + case TK_FROM: + return NEW_TK_FROM; + case TK_ORDER: + return NEW_TK_ORDER; + case TK_BY: + return NEW_TK_BY; + case TK_ASC: + return NEW_TK_ASC; + case TK_DESC: + return NEW_TK_DESC; + } + return tokenId; +} + +uint32_t getToken(const char* z, uint32_t* tokenId) { + uint32_t n = tGetToken(z, tokenId); + *tokenId = toNewTokenId(*tokenId); + return n; +} + +int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { + SAstCreateContext cxt = { .pQueryCxt = pParseCxt, .valid = true, .pRootNode = NULL }; + void *pParser = NewParseAlloc(malloc); + int32_t i = 0; + while (1) { + SToken t0 = {0}; + printf("===========================\n"); + if (cxt.pQueryCxt->pSql[i] == 0) { + NewParse(pParser, 0, t0, &cxt); + goto abort_parse; + } + printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); + t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); + t0.z = (char *)(cxt.pQueryCxt->pSql + i); + printf("token %p : %d %d [%s]\n", &t0, t0.type, t0.n, t0.z); + i += t0.n; + + switch (t0.type) { + case TK_SPACE: + case TK_COMMENT: { + break; + } + case TK_SEMI: { + NewParse(pParser, 0, t0, &cxt); + goto abort_parse; + } + + case TK_QUESTION: + case TK_ILLEGAL: { + snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z); + cxt.valid = false; + goto abort_parse; + } + + case TK_HEX: + case TK_OCT: + case TK_BIN: { + snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unsupported token: \"%s\"", t0.z); + cxt.valid = false; + goto abort_parse; + } + + default: + NewParse(pParser, t0.type, t0, &cxt); + if (!cxt.valid) { + goto abort_parse; + } + } + } + +abort_parse: + NewParseFree(pParser, free); + pQuery->pRoot = cxt.pRootNode; + return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; +} diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index a87e138ed0..71ee53dfea 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1426,35 +1426,6 @@ bool isQueryWithLimit(SQueryStmtInfo* pQueryInfo) { return false; } -SVgroupsInfo* vgroupInfoClone(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - - size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups; - SVgroupsInfo* pNew = malloc(size); - if (pNew == NULL) { - return NULL; - } - - pNew->numOfVgroups = vgroupList->numOfVgroups; - - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pNewVInfo = &pNew->vgroups[i]; - - SVgroupMsg* pvInfo = &vgroupList->vgroups[i]; - pNewVInfo->vgId = pvInfo->vgId; - pNewVInfo->numOfEps = pvInfo->numOfEps; - - for(int32_t j = 0; j < pvInfo->numOfEps; ++j) { - pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port; - tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN); - } - } - - return pNew; -} - void* vgroupInfoClear(SVgroupsInfo *vgroupList) { if (vgroupList == NULL) { return NULL; @@ -1505,19 +1476,6 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { return p; } -SVgroupsInfo* vgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { - assert(pVgroupsInfo != NULL); - - size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo* pInfo = calloc(1, size); - pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups; - for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) { - memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg)); - } - - return pInfo; -} - int32_t getNumOfOutput(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index e722b873b6..84164c8c76 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -15,5 +15,5 @@ TARGET_INCLUDE_DIRECTORIES( TARGET_LINK_LIBRARIES( parserTest - PUBLIC os util common parser catalog transport gtest function planner qcom + PUBLIC os util common nodes parser catalog transport gtest function planner qcom ) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index e2b8766700..00d64bd12a 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -15,6 +15,7 @@ #include "mockCatalogService.h" +#include "tep.h" #include #include #include @@ -39,7 +40,16 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + + SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; + + vgroup.epset.eps[0] = (SEp){"dnode_1", 6030}; + vgroup.epset.eps[1] = (SEp){"dnode_2", 6030}; + vgroup.epset.eps[2] = (SEp){"dnode_3", 6030}; + vgroup.epset.inUse = 0; + vgroup.epset.numOfEps = 3; + + meta_->vgs.emplace_back(vgroup); return *this; } @@ -112,9 +122,7 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo vgInfo->vgId = 1; - vgInfo->numOfEps = 1; - vgInfo->epAddr[0].port = 6030; - strcpy(vgInfo->epAddr[0].fqdn, "node1"); + addEpIntoEpSet(&vgInfo->epset, "node1", 6030); return 0; } @@ -133,9 +141,16 @@ public: meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema.reset(table.release()); meta_[db][tbname]->schema->uid = id_++; - meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + + SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; + addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030); + vgroup.epset.inUse = 0; + + meta_[db][tbname]->vgs.emplace_back(vgroup); // super table - meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][stbname]->vgs.emplace_back(vgroup); } void showTables() const { diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp new file mode 100644 index 0000000000..5223c9f02a --- /dev/null +++ b/source/libs/parser/test/newParserTest.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "parserImpl.h" + +using namespace std; +using namespace testing; + +class NewParserTest : public Test { +protected: + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.acctId = atoi(acctId_.c_str()); + cxt_.db = (char*) db_.c_str(); + strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + + } + + int32_t run() { + int32_t code = doParse(&cxt_, &query_); + if (code != TSDB_CODE_SUCCESS) { + cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return code; + } + cout << nodeType(query_.pRoot) << endl; + if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { + // SNode* pWhereCond; + // SNodeList* pPartitionByList; // SNode + // SNode* pWindowClause; + // SNodeList* pGroupByList; // SGroupingSetNode + // SNodeList* pOrderByList; // SOrderByExprNode + // SLimitNode limit; + // SLimitNode slimit; + + SSelectStmt* select = (SSelectStmt*)query_.pRoot; + string sql("SELECT "); + if (select->isDistinct) { + sql.append("DISTINCT "); + } + if (nullptr == select->pProjectionList) { + sql.append("* "); + } else { + nodeListToSql(select->pProjectionList, sql); + } + sql.append("FROM "); + tableToSql(select->pFromTable, sql); + cout << sql << endl; + } + // char* pStr = NULL; + // int32_t len = 0; + // code = nodesNodeToString(query_.pRoot, &pStr, &len); + // if (code != TSDB_CODE_SUCCESS) { + // cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + // return code; + // } + // cout << "node tree:\n" << pStr << endl; + return TSDB_CODE_SUCCESS; + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void tableToSql(const SNode* node, string& sql) { + const STableNode* table = (const STableNode*)node; + cout << "node : " << nodeType(node) << endl; + switch (nodeType(node)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* realTable = (SRealTableNode*)table; + if ('\0' != realTable->dbName[0]) { + sql.append(realTable->dbName); + sql.append("."); + } + sql.append(realTable->table.tableName); + break; + } + default: + break; + } + } + + void nodeListToSql(const SNodeList* nodelist, string& sql, const string& seq = ",") { + SNode* node = nullptr; + bool firstNode = true; + FOREACH(node, nodelist) { + if (!firstNode) { + sql.append(", "); + } + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: + sql.append(((SColumnNode*)node)->colName); + break; + } + } + } + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + } + + string acctId_; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + SQuery query_; +}; + +// SELECT * FROM t1 +TEST_F(NewParserTest, selectStar) { + setDatabase("root", "test"); + + bind("SELECT * FROM t1"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + + bind("SELECT * FROM test.t1"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index de62f4a2ef..a5b1fd6f6b 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -74,7 +74,7 @@ int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryP } int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { - switch (nodeType(pNode)) { + switch (queryNodeType(pNode)) { case TSDB_SQL_SELECT: { return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); } @@ -395,7 +395,7 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { } static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - int32_t type = nodeType(pQueryNode); + int32_t type = queryNodeType(pQueryNode); if (type == QNODE_MODIFY) { SDataPayloadInfo* pInfo = pQueryNode->pExtInfo; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 12c0d0780b..0be97f1ce0 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -251,24 +251,9 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { return subplan; } -static void vgroupInfoToEpSet(const SVgroupInfo* vg, SQueryNodeAddr* execNode) { - execNode->nodeId = vg->vgId; - execNode->inUse = vg->inUse; - execNode->numOfEps = vg->numOfEps; - for (int8_t i = 0; i < vg->numOfEps; ++i) { - execNode->epAddr[i] = vg->epAddr[i]; - } - return; -} - -static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) { - execNode->nodeId = vg->vgId; - execNode->inUse = 0; // todo - execNode->numOfEps = vg->numOfEps; - for (int8_t i = 0; i < vg->numOfEps; ++i) { - execNode->epAddr[i] = vg->epAddr[i]; - } - return; +static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { + pNodeAddr->nodeId = vg->vgId; + pNodeAddr->epset = vg->epset; } static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) { @@ -277,7 +262,8 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); subplan->msgType = TDMT_VND_QUERY; - vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); + + vgroupInfoToNodeAddr(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo); subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); @@ -297,11 +283,12 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType); } -static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, SSubplan* subplan) { - vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[0]), &subplan->execNode); - +// TODO: the SVgroupInfo index +static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) { + SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList; + vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode); int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; - return createUserTableScanNode(pPlanNode, pTable, type); + return createUserTableScanNode(pPlanNode, pTableInfo, type); } static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { @@ -374,7 +361,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); - vgroupInfoToEpSet(&blocks->vg, &subplan->execNode); + subplan->execNode.epset = blocks->vg.epset; subplan->pDataSink = createDataInserter(pCxt, blocks, NULL); subplan->pNode = NULL; subplan->type = QUERY_TYPE_MODIFY; @@ -461,7 +448,7 @@ static void destroyDataSinkNode(SDataSink* pSinkNode) { return; } - if (nodeType(pSinkNode) == DSINK_Dispatch) { + if (queryNodeType(pSinkNode) == DSINK_Dispatch) { SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode; tfree(pDdSink->sink.schema.pSchema); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 4fbec534f6..8d58b262aa 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -736,7 +736,7 @@ static const char* jkEpAddrFqdn = "Fqdn"; static const char* jkEpAddrPort = "Port"; static bool epAddrToJson(const void* obj, cJSON* json) { - const SEpAddr* ep = (const SEpAddr*)obj; + const SEp* ep = (const SEp*)obj; bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); if (res) { res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); @@ -745,7 +745,7 @@ static bool epAddrToJson(const void* obj, cJSON* json) { } static bool epAddrFromJson(const cJSON* json, void* obj) { - SEpAddr* ep = (SEpAddr*)obj; + SEp* ep = (SEp*)obj; copyString(json, jkEpAddrFqdn, ep->fqdn); ep->port = getNumber(json, jkEpAddrPort); return true; @@ -763,11 +763,11 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) { bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); if (res) { - res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->inUse); + res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse); } if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epAddr, sizeof(SEpAddr), pAddr->numOfEps); + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps); } return res; } @@ -776,11 +776,11 @@ static bool queryNodeAddrFromJson(const cJSON* json, void* obj) { SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj; pAddr->nodeId = getNumber(json, jkNodeAddrId); - pAddr->inUse = getNumber(json, jkNodeAddrInUse); + pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse); int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epAddr, sizeof(SEpAddr), &numOfEps); - pAddr->numOfEps = numOfEps; + bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps); + pAddr->epset.numOfEps = numOfEps; return res; } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index edf5fa5a81..f4bdf57572 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -124,12 +124,10 @@ private: } void copyStorageMeta(SVgroupsInfo** dst, const std::vector& src) { - *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size()); + *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * src.size()); (*dst)->numOfVgroups = src.size(); for (int32_t i = 0; i < src.size(); ++i) { - (*dst)->vgroups[i].vgId = src[i].vgId; - (*dst)->vgroups[i].numOfEps = src[i].numOfEps; - memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps); + (*dst)->vgroups[i] = src[i]; } } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 52e632ffbb..38ce005094 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -127,8 +127,8 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = ntohs(pRsp->vgroupInfo[i].epAddr[n].port); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) { + pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); } if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index d07e20d5c1..a19a02069b 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -137,6 +137,7 @@ typedef struct SQWorkerMgmt { SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; putReqToQueryQFp putToQueueFp; + sendReqToDnodeFp sendReqFp; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index c28e30333c..9b9b6a73ca 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1100,8 +1100,6 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8(&ctx->queryInQueue, 0); atomic_store_8(&ctx->queryContinue, 0); - DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { @@ -1313,12 +1311,13 @@ _return: QW_RET(code); } -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { - if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, + putReqToQueryQFp fp1, sendReqToDnodeFp fp2) { + if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); @@ -1361,7 +1360,8 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; mgmt->nodeObj = nodeObj; - mgmt->putToQueueFp = fp; + mgmt->putToQueueFp = fp1; + mgmt->sendReqFp = fp2; *qWorkerMgmt = mgmt; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 56c882f404..11783335b3 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -421,6 +421,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } +int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + qProcessFetchRsp(NULL, pMsg, NULL); + return TSDB_CODE_SUCCESS; +} + int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index b9d84b25aa..4c81123bf2 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -1081,7 +1081,7 @@ TEST(rcTest, shortExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1162,7 +1162,7 @@ TEST(rcTest, longExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1245,7 +1245,7 @@ TEST(rcTest, shortExeclongDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 934b222f64..92a95edf98 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -417,13 +417,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pTask->plan->execNode.numOfEps > 0) { + if (pTask->plan->execNode.epset.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.numOfEps); + SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps); return TSDB_CODE_SUCCESS; } @@ -446,7 +446,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } if (addNum <= 0) { - SCH_TASK_ELOG("no available execNode as candidate addr, nodeNum:%d", nodeNum); + SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -1050,31 +1050,19 @@ _return: SCH_RET(code); } -void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { - epSet->inUse = addr->inUse; - epSet->numOfEps = addr->numOfEps; - - for (int8_t i = 0; i < epSet->numOfEps; ++i) { - strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn)); - epSet->port[i] = addr->epAddr[i].port; - } -} - int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; - SEpSet epSet; - if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); isCandidateAddr = true; } - schConvertAddrToEpSet(addr, &epSet); - + SEpSet epSet = addr->epset; + switch (msgType) { case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { @@ -1218,8 +1206,6 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("subplanToString error, code:%x, msg:%p, len:%d", code, pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); } - -// printf("physical plan:%s\n", pTask->msg); } SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); @@ -1300,7 +1286,7 @@ void schDropJobAllTasks(SSchJob *pJob) { int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); - if (pNodeList && taosArrayGetSize(pNodeList) <= 0) { + if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 58714c51cc..6e44d99c15 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -29,7 +29,6 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wliteral-suffix" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -92,11 +91,11 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->id.templateId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003; scanPlan->type = QUERY_TYPE_SCAN; - scanPlan->execNode.numOfEps = 1; + scanPlan->execNode.nodeId = 1; - scanPlan->execNode.inUse = 0; - scanPlan->execNode.epAddr[0].port = 6030; - strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0"); + scanPlan->execNode.epset.inUse = 0; + addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030); + scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); @@ -108,7 +107,8 @@ void schtBuildQueryDag(SQueryDag *dag) { mergePlan->id.subplanId = 0x5555555555; mergePlan->type = QUERY_TYPE_MERGE; mergePlan->level = 0; - mergePlan->execNode.numOfEps = 0; + mergePlan->execNode.epset.numOfEps = 0; + mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pParents = NULL; mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); @@ -144,11 +144,11 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[0].id.subplanId = 0x0000000000000004; insertPlan[0].type = QUERY_TYPE_MODIFY; insertPlan[0].level = 0; - insertPlan[0].execNode.numOfEps = 1; + insertPlan[0].execNode.nodeId = 1; - insertPlan[0].execNode.inUse = 0; - insertPlan[0].execNode.epAddr[0].port = 6030; - strcpy(insertPlan[0].execNode.epAddr[0].fqdn, "ep0"); + insertPlan[0].execNode.epset.inUse = 0; + addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030); + insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; @@ -160,11 +160,11 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[1].id.subplanId = 0x0000000000000005; insertPlan[1].type = QUERY_TYPE_MODIFY; insertPlan[1].level = 0; - insertPlan[1].execNode.numOfEps = 1; + insertPlan[1].execNode.nodeId = 1; - insertPlan[1].execNode.inUse = 1; - insertPlan[1].execNode.epAddr[0].port = 6030; - strcpy(insertPlan[1].execNode.epAddr[0].fqdn, "ep1"); + insertPlan[1].execNode.epset.inUse = 0; + addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030); + insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; @@ -371,9 +371,9 @@ void* schtRunJobThread(void *aa) { while (!schtTestStop) { schtBuildQueryDag(&dag); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); @@ -523,9 +523,9 @@ TEST(queryTest, normalCase) { schtInitLogFile(); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); @@ -627,9 +627,9 @@ TEST(insertTest, normalCase) { schtInitLogFile(); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 61d781210c..5c214b75a1 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -13,7 +13,7 @@ target_link_libraries( PUBLIC util PUBLIC common ) -if (${BUILD_WITH_UV}) +if (${BUILD_WITH_UV_TRANS}) target_include_directories( transport PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" @@ -25,7 +25,7 @@ if (${BUILD_WITH_UV}) PUBLIC uv_a ) add_definitions(-DUSE_UV) -endif(${BUILD_WITH_UV}) +endif(${BUILD_WITH_UV_TRANS}) if (${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c7a23a2482..3f2aa1170e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -202,6 +202,8 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transConnCtxDestroy(STransConnCtx* ctx); void transFreeMsg(void* msg); + +// typedef struct SConnBuffer { char* buf; int len; @@ -209,4 +211,9 @@ typedef struct SConnBuffer { int left; } SConnBuffer; +int transInitBuffer(SConnBuffer* buf); +int transClearBuffer(SConnBuffer* buf); +int transDestroyBuffer(SConnBuffer* buf); +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); + #endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index b0927e9ecf..a286482fc1 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -814,9 +814,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SEpSet * pEpSet = &pContext->epSet; pConn = - rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); if (pConn == NULL || pConn->user[0] == 0) { - pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + pConn = rpcOpenConn(pRpc, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); } if (pConn) { @@ -1188,7 +1188,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.eps[pContext->epSet.inUse].port, pConn->connType); } else { rpcCloseConn(pConn); @@ -1202,9 +1202,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, pContext->epSet.inUse); for (int i = 0; i < pContext->epSet.numOfEps; ++i) { - pContext->epSet.port[i] = htons(pContext->epSet.port[i]); - tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.fqdn[i], - pContext->epSet.port[i]); + pContext->epSet.eps[i].port = htons(pContext->epSet.eps[i].port); + tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.eps[i].fqdn, + pContext->epSet.eps[i].port); } } rpcSendReqToServer(pRpc, pContext); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index bc9a9de318..91f9a8ead2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -30,7 +30,8 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->cfp = pInit->cfp; - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + // pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads; pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -55,7 +56,13 @@ void* rpcMallocCont(int contLen) { } return start + sizeof(STransMsgHead); } -void rpcFreeCont(void* cont) { return; } +void rpcFreeCont(void* cont) { + // impl + if (cont == NULL) { + return; + } + free((char*)cont - TRANS_MSG_OVERHEAD); +} void* rpcReallocCont(void* ptr, int contLen) { return NULL; } void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f265acf8c1..d64df9b0f3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -31,6 +31,7 @@ typedef struct SCliConn { char secured; uint64_t expireTime; int8_t notifyCount; // timers already notify to client + int32_t ref; } SCliConn; typedef struct SCliMsg { @@ -94,8 +95,10 @@ static void clientHandleExcept(SCliConn* conn); // handle req from app static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void clientMsgDestroy(SCliMsg* pMsg); -static void destroyTransConnCtx(STransConnCtx* ctx); +static void destroyUserdata(SRpcMsg* userdata); + +static void destroyCmsg(SCliMsg* cmsg); +static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); @@ -103,7 +106,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd); static void* clientThread(void* arg); static void clientHandleResp(SCliConn* conn) { - STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; + SCliMsg* pMsg = conn->data; + STransConnCtx* pCtx = pMsg->ctx; SRpcInfo* pRpc = pCtx->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); @@ -112,41 +116,53 @@ static void clientHandleResp(SCliConn* conn) { SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = transContFromHead(pHead); + rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; + tDebug("conn %p handle resp", conn); (pRpc->cfp)(NULL, &rpcMsg, NULL); conn->notifyCount += 1; + // buf's mem alread translated to rpcMsg.pCont + transClearBuffer(&conn->readBuf); + + uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); + SCliThrdObj* pThrd = conn->hostThrd; - tfree(conn->data); addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + destroyCmsg(pMsg); + conn->data = NULL; // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } - destroyTransConnCtx(pCtx); } static void clientHandleExcept(SCliConn* pConn) { + if (pConn->data == NULL) { + // handle conn except in conn pool + clientConnDestroy(pConn, true); + return; + } + tDebug("conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pRpc = pCtx->pTransInst; + destroyUserdata(&pMsg->msg); - transFreeMsg((pMsg->msg.pCont)); - pMsg->msg.pCont = NULL; + STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = -1; // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, NULL); - tfree(pConn->data); + (pCtx->pTransInst->cfp)(NULL, &rpcMsg, NULL); pConn->notifyCount += 1; - destroyTransConnCtx(pCtx); + + destroyCmsg(pMsg); + pConn->data = NULL; + // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); } @@ -213,14 +229,20 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { } queue* h = QUEUE_HEAD(&plist->conn); QUEUE_REMOVE(h); - return QUEUE_DATA(h, SCliConn, conn); + + SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); + QUEUE_INIT(&conn->conn); + return conn; } static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + tDebug("conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); conn->notifyCount = 0; @@ -237,40 +259,18 @@ static bool clientReadComplete(SConnBuffer* data) { if (msgLen > data->len) { data->left = msgLen - data->len; return false; - } else { + } else if (msgLen == data->len) { + data->left = 0; return true; } } else { return false; } } -static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - // impl later - static const int CAPACITY = 512; - +static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } - } - buf->base = pBuf->buf + pBuf->len; - buf->len = pBuf->cap - pBuf->len; - } + transAllocBuffer(pBuf, buf); } static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // impl later @@ -279,6 +279,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { + uv_read_stop((uv_stream_t*)conn->stream); tDebug("conn %p read complete", conn); clientHandleResp(conn); } else { @@ -288,10 +289,12 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } assert(nread <= 0); if (nread == 0) { - tError("conn %p closed", conn); + // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb + // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under + // read(2). return; } - if (nread < 0) { + if (nread < 0 || nread == UV_EOF) { tError("conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } @@ -300,43 +303,47 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } static void clientConnDestroy(SCliConn* conn, bool clear) { - tDebug("conn %p destroy", conn); - if (clear) { - uv_close((uv_handle_t*)conn->stream, NULL); + // + conn->ref--; + if (conn->ref == 0) { + tDebug("conn %p remove from conn pool", conn); + QUEUE_REMOVE(&conn->conn); + tDebug("conn %p remove from conn pool successfully", conn); + if (clear) { + uv_close((uv_handle_t*)conn->stream, clientDestroy); + } } - free(conn->stream); - free(conn->readBuf.buf); - free(conn->writeReq); - free(conn); } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - // QUEUE_REMOVE(&conn->conn); - clientConnDestroy(conn, false); + // transDestroyBuffer(&conn->readBuf); + + free(conn->stream); + free(conn->writeReq); + tDebug("conn %p destroy successfully", conn); + free(conn); + + // clientConnDestroy(conn, false); } static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; - - SCliMsg* pMsg = pConn->data; - transFreeMsg((pMsg->msg.pCont)); - pMsg->msg.pCont = NULL; - if (status == 0) { tDebug("conn %p data already was written out", pConn); + SCliMsg* pMsg = pConn->data; + if (pMsg == NULL) { + destroy + // handle + return; + } + destroyUserdata(&pMsg->msg); } else { tError("conn %p failed to write: %s", pConn, uv_err_name(status)); clientHandleExcept(pConn); return; } SCliThrdObj* pThrd = pConn->hostThrd; - // if (pConn->stream == NULL) { - // pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); - // uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); - // pConn->stream->data = pConn; - //} - uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); - // impl later + uv_read_start((uv_stream_t*)pConn->stream, clientAllocBufferCb, clientReadCb); } static void clientWrite(SCliConn* pConn) { @@ -381,14 +388,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("conn %p get from conn pool", conn); conn->data = pMsg; conn->writeReq->data = conn; - - conn->readBuf.len = 0; - memset(conn->readBuf.buf, 0, conn->readBuf.cap); - conn->readBuf.left = -1; + transDestroyBuffer(&conn->readBuf); clientWrite(conn); } else { SCliConn* conn = calloc(1, sizeof(SCliConn)); - + conn->ref++; // read/write stream handle conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); @@ -397,6 +401,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); conn->writeReq->data = conn; + QUEUE_INIT(&conn->conn); conn->connReq.data = conn; @@ -459,8 +464,20 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return cli; } -static void clientMsgDestroy(SCliMsg* pMsg) { - // impl later + +static void destroyUserdata(SRpcMsg* userdata) { + if (userdata->pCont == NULL) { + return; + } + transFreeMsg(userdata->pCont); + userdata->pCont = NULL; +} +static void destroyCmsg(SCliMsg* pMsg) { + if (pMsg == NULL) { + return; + } + transDestroyConnCtx(pMsg->ctx); + destroyUserdata(&pMsg->msg); free(pMsg); } static SCliThrdObj* createThrdObj() { @@ -493,7 +510,7 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { free(pThrd); } -static void destroyTransConnCtx(STransConnCtx* ctx) { +static void transDestroyConnCtx(STransConnCtx* ctx) { if (ctx != NULL) { free(ctx->ip); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5bece11bec..ca39f85eb3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -198,4 +198,51 @@ void transFreeMsg(void* msg) { } free((char*)msg - sizeof(STransMsgHead)); } + +int transInitBuffer(SConnBuffer* buf) { + transClearBuffer(buf); + return 0; +} +int transClearBuffer(SConnBuffer* buf) { + memset(buf, 0, sizeof(*buf)); + return 0; +} +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { + /* + * formate of data buffer: + * |<--------------------------data from socket------------------------------->| + * |<------STransMsgHead------->|<-------------------other data--------------->| + */ + static const int CAPACITY = 1024; + + SConnBuffer* p = connBuf; + if (p->cap == 0) { + p->buf = (char*)calloc(CAPACITY, sizeof(char)); + p->len = 0; + p->cap = CAPACITY; + p->left = -1; + + uvBuf->base = p->buf; + uvBuf->len = CAPACITY; + } else { + if (p->len >= p->cap) { + if (p->left == -1) { + p->cap *= 2; + p->buf = realloc(p->buf, p->cap); + } else if (p->len + p->left > p->cap) { + p->cap = p->len + p->left; + p->buf = realloc(p->buf, p->len + p->left); + } + } + uvBuf->base = p->buf + p->len; + uvBuf->len = p->cap - p->len; + } + return 0; +} +int transDestroyBuffer(SConnBuffer* buf) { + if (buf->cap > 0) { + tfree(buf->buf); + } + transClearBuffer(buf); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c70b1a5b28..a5ee1f1c63 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,7 +17,7 @@ #include "transComm.h" -typedef struct SConn { +typedef struct SSrvConn { uv_tcp_t* pTcp; uv_write_t* pWriter; uv_timer_t* pTimer; @@ -26,13 +26,14 @@ typedef struct SConn { queue queue; int ref; int persist; // persist connection or not - SConnBuffer connBuf; // read buf, + SConnBuffer readBuf; // read buf, int inType; void* pTransInst; // rpc init void* ahandle; // void* hostThrd; + void* pSrvMsg; - SRpcMsg sendMsg; + // SRpcMsg sendMsg; // del later char secured; int spi; @@ -40,7 +41,13 @@ typedef struct SConn { char user[TSDB_UNI_LEN]; // user ID for the link char secret[TSDB_PASSWORD_LEN]; char ckey[TSDB_PASSWORD_LEN]; // ciphering key -} SConn; +} SSrvConn; + +typedef struct SSrvMsg { + SSrvConn* pConn; + SRpcMsg msg; + queue q; +} SSrvMsg; typedef struct SWorkThrdObj { pthread_t thread; @@ -48,8 +55,8 @@ typedef struct SWorkThrdObj { int fd; uv_loop_t* loop; uv_async_t* workerAsync; // - queue conn; - pthread_mutex_t connMtx; + queue msg; + pthread_mutex_t msgMtx; void* pTransInst; } SWorkThrdObj; @@ -68,9 +75,9 @@ typedef struct SServerObj { static const char* notify = "a"; // refactor later -static int transAddAuthPart(SConn* pConn, char* msg, int msgLen); +static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); -static int uvAuthMsg(SConn* pConn, char* msg, int msgLen); +static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -82,12 +89,13 @@ static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); -static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); - +static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); +static void uvStartSendResp(SSrvMsg* msg); +static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet -static bool readComplete(SConnBuffer* buf); -static SConn* createConn(); -static void destroyConn(SConn* conn, bool clear /*clear handle or not*/); +static bool readComplete(SConnBuffer* buf); +static SSrvConn* createConn(); +static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void uvDestroyConn(uv_handle_t* handle); @@ -105,31 +113,9 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b * |<--------------------------data from socket------------------------------->| * |<------STransMsgHead------->|<-------------------other data--------------->| */ - static const int CAPACITY = 1024; - - SConn* conn = handle->data; - SConnBuffer* pBuf = &conn->connBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); - } - } - buf->base = pBuf->buf + pBuf->len; - buf->len = pBuf->cap - pBuf->len; - } + SSrvConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; + transAllocBuffer(pBuf, buf); } // check data read from socket completely or not @@ -159,7 +145,7 @@ static bool readComplete(SConnBuffer* data) { // // impl later // STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; // SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; -// SConn* pConn = pRecv->thandle; +// SSrvConn* pConn = pRecv->thandle; // tDump(pRecv->msg, pRecv->msgLen); // terrno = 0; // // SRpcReqContext* pContest; @@ -167,7 +153,7 @@ static bool readComplete(SConnBuffer* data) { // // do auth and check //} -static int uvAuthMsg(SConn* pConn, char* msg, int len) { +static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; int code = 0; @@ -222,14 +208,14 @@ static int uvAuthMsg(SConn* pConn, char* msg, int len) { // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { - SConn* conn = handle->data; + SSrvConn* conn = handle->data; tDebug("%p timeout since no activity", conn); } -static void uvHandleReq(SConn* pConn) { +static void uvHandleReq(SSrvConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; - SConnBuffer* pBuf = &pConn->connBuf; + SConnBuffer* pBuf = &pConn->readBuf; p->msg = pBuf->buf; p->msgLen = pBuf->len; p->ip = 0; @@ -255,7 +241,6 @@ static void uvHandleReq(SConn* pConn) { pHead->code = htonl(pHead->code); int32_t dlen = 0; - SRpcMsg rpcMsg; if (transDecompressMsg(NULL, 0, NULL)) { // add compress later // pHead = rpcDecompressRpcMsg(pHead); @@ -264,6 +249,8 @@ static void uvHandleReq(SConn* pConn) { // impl later // } + + SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; @@ -271,6 +258,7 @@ static void uvHandleReq(SConn* pConn) { rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; + transClearBuffer(&pConn->readBuf); pConn->ref++; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); @@ -280,8 +268,8 @@ static void uvHandleReq(SConn* pConn) { void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // opt - SConn* conn = cli->data; - SConnBuffer* pBuf = &conn->connBuf; + SSrvConn* conn = cli->data; + SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); @@ -294,11 +282,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } if (nread == 0) { - tDebug("conn %p except read", conn); - // destroyConn(conn, true); return; } - if (nread != UV_EOF) { + if (nread < 0 || nread != UV_EOF) { + if (conn->ref > 1) { + conn->ref++; // ref > 1 signed that write is in progress + } tDebug("conn %p read error: %s", conn, uv_err_name(nread)); destroyConn(conn, true); } @@ -310,25 +299,23 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b void uvOnTimeoutCb(uv_timer_t* handle) { // opt - SConn* pConn = handle->data; + SSrvConn* pConn = handle->data; tDebug("conn %p time out", pConn); } void uvOnWriteCb(uv_write_t* req, int status) { - SConn* conn = req->data; + SSrvConn* conn = req->data; - SConnBuffer* buf = &conn->connBuf; - buf->len = 0; - memset(buf->buf, 0, buf->cap); - buf->left = -1; - - SRpcMsg* pMsg = &conn->sendMsg; - transFreeMsg(pMsg->pCont); + SSrvMsg* smsg = conn->pSrvMsg; + destroySmsg(smsg); + conn->pSrvMsg = NULL; + transClearBuffer(&conn->readBuf); if (status == 0) { tDebug("conn %p data already was written on stream", conn); } else { tDebug("conn %p failed to write data, %s", conn, uv_err_name(status)); + // destroyConn(conn, true); } // opt @@ -341,16 +328,16 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } } -static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { +static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; - tDebug("conn %p prepare to send resp", conn); - SRpcMsg* pMsg = &conn->sendMsg; + tDebug("conn %p prepare to send resp", smsg->pConn); + SRpcMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->msgType = conn->inType + 1; + pHead->msgType = smsg->pConn->inType + 1; // add more info char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); @@ -361,28 +348,53 @@ static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { wb->base = msg; wb->len = len; } +static void uvStartSendResp(SSrvMsg* smsg) { + // impl + uv_buf_t wb; + uvPrepareSendData(smsg, &wb); + + SSrvConn* pConn = smsg->pConn; + uv_timer_stop(pConn->pTimer); + + pConn->pSrvMsg = smsg; + // conn->pWriter->data = smsg; + uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb); + + // SRpcMsg* rpcMsg = smsg->msg; + + return; +} +static void destroySmsg(SSrvMsg* smsg) { + if (smsg == NULL) { + return; + } + transFreeMsg(smsg->msg.pCont); + free(smsg); +} void uvWorkerAsyncCb(uv_async_t* handle) { SWorkThrdObj* pThrd = handle->data; - SConn* conn = NULL; + SSrvConn* conn = NULL; queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&pThrd->connMtx); - QUEUE_MOVE(&pThrd->conn, &wq); - pthread_mutex_unlock(&pThrd->connMtx); + pthread_mutex_lock(&pThrd->msgMtx); + QUEUE_MOVE(&pThrd->msg, &wq); + pthread_mutex_unlock(&pThrd->msgMtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); QUEUE_REMOVE(head); - SConn* conn = QUEUE_DATA(head, SConn, queue); - if (conn == NULL) { - tError("except occurred, do nothing"); - return; - } - uv_buf_t wb; - uvPrepareSendData(conn, &wb); - uv_timer_stop(conn->pTimer); - uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); + SSrvMsg* msg = QUEUE_DATA(head, SSrvMsg, q); + if (msg == NULL) { + tError("except occurred, continue"); + continue; + } + uvStartSendResp(msg); + // uv_buf_t wb; + // uvPrepareSendData(msg, &wb); + // uv_timer_stop(conn->pTimer); + + // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } @@ -435,7 +447,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConn* pConn = createConn(); + SSrvConn* pConn = createConn(); pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ @@ -484,8 +496,8 @@ static bool addHandleToWorkloop(void* arg) { pThrd->pipe->data = pThrd; - QUEUE_INIT(&pThrd->conn); - pthread_mutex_init(&pThrd->connMtx, NULL); + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); pThrd->workerAsync = malloc(sizeof(uv_async_t)); uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); @@ -523,34 +535,39 @@ void* workerThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SConn* createConn() { - SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); +static SSrvConn* createConn() { + SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + tDebug("conn %p created", pConn); ++pConn->ref; return pConn; } -static void destroyConn(SConn* conn, bool clear) { +static void destroyConn(SSrvConn* conn, bool clear) { if (conn == NULL) { return; } - if (--conn->ref == 0) { + tDebug("conn %p try to destroy", conn); + if (--conn->ref > 0) { return; } + transDestroyBuffer(&conn->readBuf); + destroySmsg(conn->pSrvMsg); + conn->pSrvMsg = NULL; + if (clear) { - uv_close((uv_handle_t*)conn->pTcp, NULL); + uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); } +} +static void uvDestroyConn(uv_handle_t* handle) { + SSrvConn* conn = handle->data; + tDebug("conn %p destroy", conn); uv_timer_stop(conn->pTimer); free(conn->pTimer); - free(conn->pTcp); - free(conn->connBuf.buf); + // free(conn->pTcp); free(conn->pWriter); free(conn); } -static void uvDestroyConn(uv_handle_t* handle) { - SConn* conn = handle->data; - destroyConn(conn, false); -} -static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { +static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { STransMsgHead* pHead = (STransMsgHead*)msg; if (pConn->spi && pConn->secured == 0) { @@ -632,6 +649,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { pthread_join(pThrd->thread, NULL); // free(srv->pipe[i]); free(pThrd->loop); + pthread_mutex_destroy(&pThrd->msgMtx); free(pThrd); } void taosCloseServer(void* arg) { @@ -648,17 +666,20 @@ void taosCloseServer(void* arg) { } void rpcSendResponse(const SRpcMsg* pMsg) { - SConn* pConn = pMsg->handle; + SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; - // opt later - pConn->sendMsg = *pMsg; - pthread_mutex_lock(&pThrd->connMtx); - QUEUE_PUSH(&pThrd->conn, &pConn->queue); - pthread_mutex_unlock(&pThrd->connMtx); + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->pConn = pConn; + srvMsg->msg = *pMsg; + + pthread_mutex_lock(&pThrd->msgMtx); + QUEUE_PUSH(&pThrd->msg, &srvMsg->q); + pthread_mutex_unlock(&pThrd->msgMtx); + tDebug("conn %p start to send resp", pConn); - uv_async_send(pConn->pWorkerAsync); + uv_async_send(pThrd->workerAsync); } #endif diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 84814f39fc..6767e704b5 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" @@ -64,6 +65,7 @@ static void *sendRequest(void *param) { // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); tDebug("recv response succefully"); + // usleep(100000000); } @@ -86,12 +88,9 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - epSet.numOfEps = 1; epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); + addEpIntoEpSet(&epSet, serverIp, 7000); + addEpIntoEpSet(&epSet, "192.168.0.1", 7000); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -109,9 +108,9 @@ int main(int argc, char *argv[]) { for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - epSet.port[0] = atoi(argv[++i]); + epSet.eps[0].port = atoi(argv[++i]); } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { - tstrncpy(epSet.fqdn[0], argv[++i], sizeof(epSet.fqdn[0])); + tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn)); } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { rpcInit.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { @@ -135,7 +134,7 @@ int main(int argc, char *argv[]) { } else { printf("\nusage: %s [options] \n", argv[0]); printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); - printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]); + printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port); printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 12d8a01819..d1a587f4e5 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -165,6 +165,7 @@ int main(int argc, char *argv[]) { tError("failed to start RPC server"); return -1; } + // sleep(5); tInfo("RPC server is running, ctrl-c to exit"); @@ -172,7 +173,6 @@ int main(int argc, char *argv[]) { dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); } - qhandle = taosOpenQueue(); qset = taosOpenQset(); taosAddIntoQset(qset, qhandle, NULL); diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt index 4d2dd97c87..bcf759e04f 100644 --- a/source/libs/wal/CMakeLists.txt +++ b/source/libs/wal/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries( PUBLIC cjson PUBLIC os PUBLIC util + PUBLIC common ) if(${BUILD_TEST}) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2bc328b4e2..a4b34dee37 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -257,7 +257,7 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { return 0; } -int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen) { +int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { if (pWal == NULL) return -1; int code = 0; diff --git a/source/nodes/src/nodesClone.c b/source/nodes/src/nodesCloneFuncs.c similarity index 93% rename from source/nodes/src/nodesClone.c rename to source/nodes/src/nodesCloneFuncs.c index 04f6df5623..2d0a6483ae 100644 --- a/source/nodes/src/nodesClone.c +++ b/source/nodes/src/nodesCloneFuncs.c @@ -15,6 +15,6 @@ #include "nodes.h" -void cloneNode(const SNode* pNode) { +void nodesCloneNode(const SNode* pNode) { } diff --git a/source/nodes/src/nodesCodeFuncs.c b/source/nodes/src/nodesCodeFuncs.c new file mode 100644 index 0000000000..959e10fb82 --- /dev/null +++ b/source/nodes/src/nodesCodeFuncs.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_IS_NULL_CONDITION: + case QUERY_NODE_FUNCTION: + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: + case QUERY_NODE_STATE_WINDOW: + case QUERY_NODE_SESSION_WINDOW: + case QUERY_NODE_INTERVAL_WINDOW: + case QUERY_NODE_SET_OPERATOR: + case QUERY_NODE_SELECT_STMT: + case QUERY_NODE_SHOW_STMT: + default: + break; + } +} + +int32_t nodesStringToNode(const char* pStr, SNode** pNode) { + +} diff --git a/source/nodes/src/nodesEqual.c b/source/nodes/src/nodesEqualFuncs.c similarity index 86% rename from source/nodes/src/nodesEqual.c rename to source/nodes/src/nodesEqualFuncs.c index bef025fbea..f2752fe7f9 100644 --- a/source/nodes/src/nodesEqual.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -32,17 +32,17 @@ #define COMPARE_NODE_FIELD(fldname) \ do { \ - if (!nodeEqual(a->fldname, b->fldname)) \ + if (!nodesEqualNode(a->fldname, b->fldname)) \ return false; \ } while (0) -#define COMPARE_ARRAY_FIELD(fldname) \ +#define COMPARE_NODE_LIST_FIELD(fldname) \ do { \ - if (!nodeArrayEqual(a->fldname, b->fldname)) \ + if (!nodeNodeListEqual(a->fldname, b->fldname)) \ return false; \ } while (0) -static bool nodeArrayEqual(const SArray* a, const SArray* b) { +static bool nodeNodeListEqual(const SNodeList* a, const SNodeList* b) { if (a == b) { return true; } @@ -51,13 +51,13 @@ static bool nodeArrayEqual(const SArray* a, const SArray* b) { return false; } - if (taosArrayGetSize(a) != taosArrayGetSize(b)) { + if (LIST_LENGTH(a) != LIST_LENGTH(b)) { return false; } - size_t size = taosArrayGetSize(a); - for (size_t i = 0; i < size; ++i) { - if (!nodeEqual((SNode*)taosArrayGetP(a, i), (SNode*)taosArrayGetP(b, i))) { + SNode* na, *nb; + FORBOTH(na, a, nb, b) { + if (!nodesEqualNode(na, nb)) { return false; } } @@ -85,7 +85,7 @@ static bool operatorNodeEqual(const SOperatorNode* a, const SOperatorNode* b) { static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicConditionNode* b) { COMPARE_SCALAR_FIELD(condType); - COMPARE_ARRAY_FIELD(pParameterList); + COMPARE_NODE_LIST_FIELD(pParameterList); return true; } @@ -97,11 +97,11 @@ static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCond static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); - COMPARE_ARRAY_FIELD(pParameterList); + COMPARE_NODE_LIST_FIELD(pParameterList); return true; } -bool nodeEqual(const SNode* a, const SNode* b) { +bool nodesEqualNode(const SNode* a, const SNode* b) { if (a == b) { return true; } @@ -132,6 +132,7 @@ bool nodeEqual(const SNode* a, const SNode* b) { case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_GROUPING_SET: case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: return false; // todo default: break; diff --git a/source/nodes/src/nodesTraverse.c b/source/nodes/src/nodesTraverseFuncs.c similarity index 57% rename from source/nodes/src/nodesTraverse.c rename to source/nodes/src/nodesTraverseFuncs.c index eac8288099..ffe40073e6 100644 --- a/source/nodes/src/nodesTraverse.c +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -17,17 +17,17 @@ typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); -bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext) { - size_t size = taosArrayGetSize(pArray); - for (size_t i = 0; i < size; ++i) { - if (!nodeTreeWalker((SNode*)taosArrayGetP(pArray, i), walker, pContext)) { - return false; - } +bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { + SNode* node; + FOREACH(node, pNodeList) { + if (!nodesWalkNode(node, walker, pContext)) { + return false; } - return true; + } + return true; } -bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { +bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { if (NULL == pNode) { return true; } @@ -39,38 +39,39 @@ bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: // these node types with no subnodes return true; case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; - if (!nodeTreeWalker(pOpNode->pLeft, walker, pContext)) { + if (!nodesWalkNode(pOpNode->pLeft, walker, pContext)) { return false; } - return nodeTreeWalker(pOpNode->pRight, walker, pContext); + return nodesWalkNode(pOpNode->pRight, walker, pContext); } case QUERY_NODE_LOGIC_CONDITION: - return nodeArrayWalker(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); + return nodesWalkNodeList(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); case QUERY_NODE_IS_NULL_CONDITION: - return nodeTreeWalker(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); + return nodesWalkNode(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); case QUERY_NODE_FUNCTION: - return nodeArrayWalker(((SFunctionNode*)pNode)->pParameterList, walker, pContext); + return nodesWalkNodeList(((SFunctionNode*)pNode)->pParameterList, walker, pContext); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: return true; // todo case QUERY_NODE_JOIN_TABLE: { SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; - if (!nodeTreeWalker(pJoinTableNode->pLeft, walker, pContext)) { + if (!nodesWalkNode(pJoinTableNode->pLeft, walker, pContext)) { return false; } - if (!nodeTreeWalker(pJoinTableNode->pRight, walker, pContext)) { + if (!nodesWalkNode(pJoinTableNode->pRight, walker, pContext)) { return false; } - return nodeTreeWalker(pJoinTableNode->pOnCond, walker, pContext); + return nodesWalkNode(pJoinTableNode->pOnCond, walker, pContext); } case QUERY_NODE_GROUPING_SET: - return nodeArrayWalker(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); + return nodesWalkNodeList(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); case QUERY_NODE_ORDER_BY_EXPR: - return nodeTreeWalker(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); + return nodesWalkNode(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); default: break; } @@ -78,6 +79,6 @@ bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { return false; } -bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { +bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) { } diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c new file mode 100644 index 0000000000..535bd8e171 --- /dev/null +++ b/source/nodes/src/nodesUtilFuncs.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" +#include "nodesShowStmts.h" + +bool nodesIsTimeorderQuery(const SNode* pQuery) { + +} + +bool nodesIsTimelineQuery(const SNode* pQuery) { + +} + +static SNode* makeNode(ENodeType type, size_t size) { + SNode* p = calloc(1, size); + setNodeType(p, type); + return p; +} + +SNode* nodesMakeNode(ENodeType type) { + switch (type) { + case QUERY_NODE_COLUMN: + return makeNode(type, sizeof(SColumnNode)); + case QUERY_NODE_VALUE: + return makeNode(type, sizeof(SValueNode)); + case QUERY_NODE_OPERATOR: + return makeNode(type, sizeof(SOperatorNode)); + case QUERY_NODE_LOGIC_CONDITION: + return makeNode(type, sizeof(SLogicConditionNode)); + case QUERY_NODE_IS_NULL_CONDITION: + return makeNode(type, sizeof(SIsNullCondNode)); + case QUERY_NODE_FUNCTION: + return makeNode(type, sizeof(SFunctionNode)); + case QUERY_NODE_REAL_TABLE: + return makeNode(type, sizeof(SRealTableNode)); + case QUERY_NODE_TEMP_TABLE: + return makeNode(type, sizeof(STempTableNode)); + case QUERY_NODE_JOIN_TABLE: + return makeNode(type, sizeof(SJoinTableNode)); + case QUERY_NODE_GROUPING_SET: + return makeNode(type, sizeof(SGroupingSetNode)); + case QUERY_NODE_ORDER_BY_EXPR: + return makeNode(type, sizeof(SOrderByExprNode)); + case QUERY_NODE_LIMIT: + return makeNode(type, sizeof(SLimitNode)); + case QUERY_NODE_STATE_WINDOW: + return makeNode(type, sizeof(SStateWindowNode)); + case QUERY_NODE_SESSION_WINDOW: + return makeNode(type, sizeof(SSessionWindowNode)); + case QUERY_NODE_INTERVAL_WINDOW: + return makeNode(type, sizeof(SIntervalWindowNode)); + case QUERY_NODE_SET_OPERATOR: + return makeNode(type, sizeof(SSetOperator)); + case QUERY_NODE_SELECT_STMT: + return makeNode(type, sizeof(SSelectStmt)); + case QUERY_NODE_SHOW_STMT: + return makeNode(type, sizeof(SShowStmt)); + default: + break; + } + return NULL; +} + +void nodesDestroyNode(SNode* pNode) { + +} + +void nodesDestroyNodeList(SNodeList* pList) { + +} diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 97440f8dae..ca4e5a6f30 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -181,7 +181,7 @@ static void *tFWorkerThreadFp(SQWorker *worker) { } STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tFWorkerThreadFp); + return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); } void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); } diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index e689bb3261..6061b6ece1 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -178,9 +178,9 @@ if $rows != 3 then endi sql select * from st -#if $rows != 15 then -# return -1 -#endi +if $rows != 15 then + return -1 +endi print =============== drop dnode sql drop dnode 2; diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index f47630b737..776b09b813 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -139,9 +139,9 @@ endi print =============== query data frpm st sql select * from st -#if $rows != 21 then -# return -1 -#endi +if $rows != 21 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -200,8 +200,8 @@ endi print =============== query data frpm st sql select * from st -#if $rows != 21 then -# return -1 -#endi +if $rows != 21 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file