diff --git a/example/src/tmq.c b/example/src/tmq.c index e19e6b5aee..99e0c443dd 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -44,35 +44,35 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + /*pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");*/ /*if (taos_errno(pRes) != 0) {*/ /*printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));*/ /*return -1;*/ /*}*/ - taos_free_result(pRes); + /*taos_free_result(pRes);*/ - pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); + /*pRes = taos_query(pConn, "create table if not exists tu using st1 tags(1)");*/ + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ - pRes = taos_query(pConn, "create table tu2 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); + /*pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");*/ + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ const char* sql = "select * from st1"; pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); - /*if (taos_errno(pRes) != 0) {*/ - /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ - /*return -1;*/ - /*}*/ - /*taos_free_result(pRes);*/ + if (taos_errno(pRes) != 0) { + printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); taos_close(pConn); return 0; } @@ -116,7 +116,7 @@ void basic_consume_loop(tmq_t *tmq, int32_t cnt = 0; /*clock_t startTime = clock();*/ while (running) { - tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0); + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { cnt++; msg_process(tmqmessage); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b386fb0e0d..13e73e6871 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1739,11 +1739,10 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { return buf; } -typedef struct SMqSetCVgReq { +typedef struct { int64_t leftForVer; int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; + int64_t consumerId; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; char* sql; @@ -1752,59 +1751,78 @@ typedef struct SMqSetCVgReq { char* qmsg; } SMqSetCVgReq; -static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { - int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pMsg->sId); - tlen += taosEncodeFixedU64(buf, pMsg->queryId); - tlen += taosEncodeFixedU64(buf, pMsg->taskId); - tlen += taosEncodeFixedU32(buf, pMsg->sqlLen); - tlen += taosEncodeFixedU32(buf, pMsg->phyLen); - //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); - return tlen; -} - -static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { - buf = taosDecodeFixedU64(buf, &pMsg->sId); - buf = taosDecodeFixedU64(buf, &pMsg->queryId); - buf = taosDecodeFixedU64(buf, &pMsg->taskId); - buf = taosDecodeFixedU32(buf, &pMsg->sqlLen); - buf = taosDecodeFixedU32(buf, &pMsg->phyLen); - //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); - return buf; -} - static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); tlen += taosEncodeString(buf, pReq->qmsg); - //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->topicName); buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); buf = taosDecodeString(buf, &pReq->qmsg); + return buf; +} + +typedef struct { + int64_t leftForVer; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; + //char topicName[TSDB_TOPIC_FNAME_LEN]; + //char cgroup[TSDB_CONSUMER_GROUP_LEN]; + //char* sql; + //char* logicalPlan; + //char* physicalPlan; + //char* qmsg; +} SMqMVRebReq; + +static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); + tlen += taosEncodeFixedI32(buf, pReq->vgId); + tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); + //tlen += taosEncodeString(buf, pReq->topicName); + //tlen += taosEncodeString(buf, pReq->cgroup); + //tlen += taosEncodeString(buf, pReq->sql); + //tlen += taosEncodeString(buf, pReq->logicalPlan); + //tlen += taosEncodeString(buf, pReq->physicalPlan); + //tlen += taosEncodeString(buf, pReq->qmsg); + //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); + buf = taosDecodeFixedI32(buf, &pReq->vgId); + buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); + //buf = taosDecodeStringTo(buf, pReq->topicName); + //buf = taosDecodeStringTo(buf, pReq->cgroup); + //buf = taosDecodeString(buf, &pReq->sql); + //buf = taosDecodeString(buf, &pReq->logicalPlan); + //buf = taosDecodeString(buf, &pReq->physicalPlan); + //buf = taosDecodeString(buf, &pReq->qmsg); //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } -typedef struct SMqSetCVgRsp { +typedef struct { SMsgHead header; int32_t vgId; int64_t consumerId; @@ -1812,6 +1830,14 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; +typedef struct { + SMsgHead header; + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqMVRebRsp; + typedef struct { uint32_t nCols; SSchema *pSchema; @@ -1912,7 +1938,7 @@ typedef struct { typedef struct { int64_t consumerId; - int64_t epoch; + int32_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; @@ -1969,7 +1995,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->epoch); + tlen += taosEncodeFixedI32(buf, pRsp->epoch); tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -1982,7 +2008,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->epoch); + buf = taosDecodeFixedI32(buf, &pRsp->epoch); buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); @@ -2002,4 +2028,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } #endif -#endif /*_TD_COMMON_TAOS_MSG_H_*/ +#endif /*_TD_COMMON_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 98d2b2c519..cf890f5cbb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -164,6 +164,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ab1298785a..b6054c02b4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,7 +30,7 @@ extern "C" { #include "tmsg.h" #include "transport.h" -struct SCatalog; +typedef struct SCatalog SCatalog; enum { CTG_DBG_DB_NUM = 1, @@ -64,6 +64,7 @@ typedef struct SCatalogCfg { typedef struct SSTableMetaVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; @@ -84,7 +85,7 @@ int32_t catalogInit(SCatalogCfg *cfg); * @param catalogHandle (output, NO need to free it) * @return error code */ -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); /** * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. @@ -92,9 +93,9 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); * @param pCatalog (input, NO more usage) * @return error code */ -void catalogFreeHandle(struct SCatalog* pCatalog); +void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version); /** * Get a DB's all vgroup info. @@ -106,13 +107,13 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); +int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); +int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); /** * Get a table's meta data. @@ -123,7 +124,7 @@ int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, c * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -134,13 +135,13 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg); +int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); /** - * Force renew a table's local cached meta data. + * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -148,10 +149,10 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); + int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); /** - * Force renew a table's local cached meta data and get the new one. + * Force refresh a table's local cached meta data and get the new one. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -160,7 +161,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); + int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); @@ -173,7 +174,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); +int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); /** * Get a table's vgroup from its name's hash value. @@ -184,7 +185,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, * @param vgInfo (output, vgroup info) * @return error code */ -int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); +int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); /** @@ -196,14 +197,14 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter * @param pRsp (output, response data) * @return error code */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); +int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); /** diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 549f36a898..1b70c78af9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -80,16 +80,16 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; -typedef struct SDBVgroupInfo { +typedef struct SDBVgInfo { int32_t vgVersion; int8_t hashMethod; SHashObj *vgHash; //key:vgId, value:SVgroupInfo -} SDBVgroupInfo; +} SDBVgInfo; typedef struct SUseDbOutput { char db[TSDB_DB_FNAME_LEN]; uint64_t dbId; - SDBVgroupInfo *dbVgroup; + SDBVgInfo *dbVgroup; } SUseDbOutput; enum { diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 283604508f..00ba1120e7 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 TAOS Data, Inc. + * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -23,7 +23,7 @@ extern "C" { #include #include "taosdef.h" -typedef int32_t SyncNodeId; +typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; typedef int64_t SyncIndex; typedef uint64_t SyncTerm; @@ -46,109 +46,113 @@ typedef struct { } SNodeInfo; typedef struct { - int32_t selfIndex; - int32_t replica; + int32_t replicaNum; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; -} SSyncCluster; +} SSyncCfg; typedef struct { - int32_t selfIndex; - int32_t replica; - SNodeInfo node[TSDB_MAX_REPLICA]; + int32_t replicaNum; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; ESyncState role[TSDB_MAX_REPLICA]; } SNodesRole; +// abstract definition of snapshot +typedef struct SSnapshot { + void* data; + SyncIndex lastApplyIndex; +} SSnapshot; + typedef struct SSyncFSM { - void* pData; + void* data; - // apply committed log, bufs will be free by sync module - int32_t (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData); + // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // cluster commit callback - int32_t (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData); + // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm return snapshot in ppBuf, bufs will be free by sync module - // TODO: getSnapshot SHOULD be async? - int32_t (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int32_t* objId, bool* isLast); + // when log entry is updated by a new one, FpRollBackCb is called + // user can do something to roll back. for example, delete data from tsdb, or just ignore it + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm apply snapshot with pBuf data - int32_t (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int32_t objId, bool isLast); + // user should implement this function, use "data" to take snapshot into "snapshot" + int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); - // call when restore snapshot and log done - int32_t (*onRestoreDone)(struct SSyncFSM* fsm); - - void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf); - - void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole); + // user should implement this function, restore "data" from "snapshot" + int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot); } SSyncFSM; +// abstract definition of log store in raft +// SWal implements it typedef struct SSyncLogStore { - void* pData; + void* data; - // write log with given index - int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf); + // append one log entry + int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); - /** - * read log from given index(included) with limit, return the actual num in nBuf, - * pBuf will be free in sync module - **/ - int32_t (*logRead)(struct SSyncLogStore* logStore, SyncIndex index, int limit, - SSyncBuffer* pBuf, int* nBuf); + // get one log entry, user need to free pBuf->data + int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); - // mark log with given index has been commtted - int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index); + // update log store commit index with "index" + int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); - // prune log before given index(not included) - int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index); + // truncate log with index, entries after the given index (>index) will be deleted + int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index); - // rollback log after given index(included) - int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index); + // return commit index of log + SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + + // return index of last entry + SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore); + + // return term of last entry + SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore); - // return last index of log - SyncIndex (*logLastIndex)(struct SSyncLogStore* logStore); } SSyncLogStore; -typedef struct SStateManager { - void* pData; +// raft need to persist two variables in storage: currentTerm, voteFor +typedef struct SStateMgr { + void* data; - // save serialized server state data, buffer will be free by Sync - int32_t (*saveServerState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); + int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); - // read serialized server state data, buffer will be free by Sync - int32_t (*readServerState)(struct SStateManager* stateMng, char** ppBuffer, int* n); + int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor); + int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor); - // save serialized cluster state data, buffer will be free by Sync - void (*saveClusterState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); + int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); - // read serialized cluster state data, buffer will be free by Sync - int32_t (*readClusterState)(struct SStateManager* stateMng, char** ppBuffer, int* n); -} SStateManager; +} SStateMgr; typedef struct { SyncGroupId vgId; - SyncIndex appliedIndex; - SSyncCluster syncCfg; - SSyncFSM fsm; + SSyncCfg syncCfg; SSyncLogStore logStore; - SStateManager stateManager; + SStateMgr stateManager; + SSyncFSM syncFsm; + } SSyncInfo; -struct SSyncNode; -typedef struct SSyncNode SSyncNode; +// will be defined in syncInt.h, here just for complie +typedef struct SSyncNode { +} SSyncNode; int32_t syncInit(); void syncCleanUp(); -SSyncNode* syncStart(const SSyncInfo*); -void syncReconfig(const SSyncNode*, const SSyncCluster*); -void syncStop(const SSyncNode*); +int64_t syncStart(const SSyncInfo*); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg*); -int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak); +// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); -int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode); - -int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode); +ESyncState syncGetMyRole(int64_t rid); +void syncGetNodesRole(int64_t rid, SNodesRole*); extern int32_t sDebugFlag; diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index b3ef5ba45c..b182063808 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -31,16 +31,7 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) // only be use in FOREACH -#define ERASE_NODE(list) \ - if (NULL == cell->pPrev) { \ - (list)->pHead = cell->pNext; \ - } else { \ - cell->pPrev->pNext = cell->pNext; \ - cell->pNext->pPrev = cell->pPrev; \ - } \ - SListCell* tmp = cell; \ - cell = cell->pNext; \ - tfree(tmp); +#define ERASE_NODE(list) cell = nodesListErase(list, cell); #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) @@ -343,18 +334,22 @@ void nodesDestroyNode(SNode* pNode); SNodeList* nodesMakeList(); SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SNode* nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); -typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); +typedef enum EDealRes { + DEAL_RES_CONTINUE = 1, + DEAL_RES_IGNORE_CHILD, + DEAL_RES_ERROR, +} EDealRes; +typedef EDealRes (*FQueryNodeWalker)(SNode* pNode, void* pContext); void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); -bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); - bool nodesEqualNode(const SNode* a, const SNode* b); void nodesCloneNode(const SNode* pNode); @@ -362,6 +357,8 @@ void nodesCloneNode(const SNode* pNode); int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); +bool nodesIsExprNode(const SNode* pNode); + bool nodesIsArithmeticOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2451b2b904..0a822927ad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -247,7 +247,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) -// mnode-topic +// mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) #define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1) #define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2) @@ -256,7 +256,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) -#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) +#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) @@ -454,6 +456,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) //Inconsistent datatypes #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression +#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression +#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression +#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function #ifdef __cplusplus } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9e5cdee516..6737467d17 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -44,7 +44,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { - SDBVgroupInfo vgInfo = {0}; + SDBVgInfo vgInfo = {0}; vgInfo.vgVersion = rsp->vgVersion; vgInfo.hashMethod = rsp->hashMethod; vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -60,12 +60,9 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - } - - code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo); - if (code) { - taosHashCleanup(vgInfo.vgHash); - } + } + + catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); } if (code) { @@ -86,13 +83,14 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo rsp->numOfColumns = ntohl(rsp->numOfColumns); rsp->suid = be64toh(rsp->suid); + rsp->dbId = be64toh(rsp->dbId); if (rsp->numOfColumns < 0) { schemaNum = 0; tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); - catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid); } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index a820d68e50..8b1faaef4e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -58,7 +58,7 @@ struct tmq_t { char clientId[256]; SRWLatch lock; int64_t consumerId; - int64_t epoch; + int32_t epoch; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -113,10 +113,10 @@ typedef struct SMqConsumeCbParam { } SMqConsumeCbParam; typedef struct SMqCommitCbParam { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; + tmq_t* tmq; + SMqClientVg* pVg; + int32_t async; + tsem_t rspSem; } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -163,8 +163,8 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqCommitCbParam* pParam = (SMqCommitCbParam*) param; - tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; + tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } @@ -221,13 +221,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tNameExtractFullName(&name, topicFname); tscDebug("subscribe topic: %s", topicFname); SMqClientTopic topic = { - .nextVgIdx = 0, - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL - }; + .nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL}; topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); /*SMqClientTopic topic = {*/ @@ -461,10 +455,10 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { void tmqShowMsg(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - static bool noPrintSchema; - char pBuf[128]; + static bool noPrintSchema; + char pBuf[128]; SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; - int32_t colNum = pRsp->schemas->nCols; + int32_t colNum = pRsp->schemas->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -506,7 +500,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; SMqClientVg* pVg = pParam->pVg; if (code != 0) { - /*printf("msg discard\n");*/ + printf("msg discard\n"); tsem_post(&pParam->rspSem); return 0; } @@ -517,7 +511,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return -1; } tDecodeSMqConsumeRsp(pMsg->pData, pRsp); - /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ + /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->numOfTopics == 0) { /*printf("no data\n");*/ free(pRsp); @@ -584,7 +578,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { +int32_t tmqAskEp(tmq_t* tmq, bool wait) { int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { @@ -592,6 +586,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { goto END; } buf->consumerId = htobe64(tmq->consumerId); + buf->epoch = htonl(tmq->epoch); strcpy(buf->cgroup, tmq->groupId); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); @@ -602,7 +597,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; - SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); goto END; @@ -652,7 +647,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_message = NULL; int64_t status = atomic_load_64(&tmq->status); - tmqAsyncAskEp(tmq, status == 0); + tmqAskEp(tmq, status == 0); if (blocking_time < 0) blocking_time = 1; if (blocking_time > 1000) blocking_time = 1000; @@ -670,48 +665,58 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); - if (pReq == NULL) { - usleep(blocking_time * 1000); - return NULL; + int32_t beginVgIdx = pTopic->nextVgIdx; + while (1) { + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); + if (pReq == NULL) { + ASSERT(false); + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + ASSERT(false); + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; + tsem_init(¶m->rspSem, 0, 0); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + /*printf("req offset: %ld\n", pReq->offset);*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; + + tsem_wait(¶m->rspSem); + tsem_destroy(¶m->rspSem); + free(param); + + if (tmq_message == NULL) { + if (beginVgIdx == pTopic->nextVgIdx) { + usleep(blocking_time * 1000); + } else { + continue; + } + } + + return tmq_message; } - SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); - if (param == NULL) { - usleep(blocking_time * 1000); - return NULL; - } - param->tmq = tmq; - param->retMsg = &tmq_message; - param->pVg = pVg; - tsem_init(¶m->rspSem, 0, 0); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = param; - sendInfo->fp = tmqPollCb; - - /*printf("req offset: %ld\n", pReq->offset);*/ - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tmq->pollCnt++; - - tsem_wait(¶m->rspSem); - tsem_destroy(¶m->rspSem); - free(param); - - if (tmq_message == NULL) { - usleep(blocking_time * 1000); - } - - return tmq_message; - /*tsem_wait(&pRequest->body.rspSem);*/ /*if (body != NULL) {*/ @@ -726,21 +731,20 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - if (tmq_topic_vgroup_list != NULL) { - //TODO + // TODO } - //TODO: change semaphore to gate + // TODO: change semaphore to gate for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); - + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { continue; } @@ -771,9 +775,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { free(tmq_message); } -tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { - return TMQ_RESP_ERR__SUCCESS; -} +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } const char* tmq_err2str(tmq_resp_err_t err) { if (err == TMQ_RESP_ERR__SUCCESS) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 9b3ab40226..239fe9ca4d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -116,6 +116,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; // Requests handled by VNODE @@ -149,6 +150,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 78eab3151f..3eeb28e532 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; + SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); @@ -594,6 +594,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { vnodeCfg.pDnode = pDnode; vnodeCfg.pTfs = pDnode->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9090d5d97c..73c0446e3f 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -424,6 +424,7 @@ typedef struct { int32_t status; int32_t vgNum; SArray* consumers; // SArray + SArray* lostConsumers; // SArray SArray* unassignedVg; // SArray } SMqSubscribeObj; @@ -432,10 +433,17 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { if (pSub == NULL) { return NULL; } + pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer)); if (pSub->consumers == NULL) { goto _err; } + + pSub->lostConsumers = taosArrayInit(0, sizeof(SMqSubConsumer)); + if (pSub->lostConsumers == NULL) { + goto _err; + } + pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); if (pSub->unassignedVg == NULL) { goto _err; @@ -448,8 +456,9 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { return pSub; _err: - tfree(pSub->unassignedVg); tfree(pSub->consumers); + tfree(pSub->lostConsumers); + tfree(pSub->unassignedVg); tfree(pSub); return NULL; } @@ -468,6 +477,13 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); } + sz = taosArrayGetSize(pSub->lostConsumers); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->lostConsumers, i); + tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); + } + sz = taosArrayGetSize(pSub->unassignedVg); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -496,6 +512,17 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) taosArrayPush(pSub->consumers, &subConsumer); } + buf = taosDecodeFixedI32(buf, &sz); + pSub->lostConsumers = taosArrayInit(sz, sizeof(SMqSubConsumer)); + if (pSub->lostConsumers == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubConsumer subConsumer = {0}; + buf = tDecodeSMqSubConsumer(buf, &subConsumer); + taosArrayPush(pSub->lostConsumers, &subConsumer); + } + buf = taosDecodeFixedI32(buf, &sz); pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->unassignedVg == NULL) { @@ -545,7 +572,7 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* currentTopics; // SArray SArray* recentRemovedTopics; // SArray - int64_t epoch; + int32_t epoch; // stat int64_t pollCnt; // status @@ -561,8 +588,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); tlen += taosEncodeFixedI64(buf, pConsumer->connId); - tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeFixedI32(buf, pConsumer->epoch); tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); + tlen += taosEncodeFixedI32(buf, pConsumer->status); tlen += taosEncodeString(buf, pConsumer->cgroup); sz = taosArrayGetSize(pConsumer->currentTopics); @@ -585,8 +613,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); buf = taosDecodeFixedI64(buf, &pConsumer->connId); - buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeFixedI32(buf, &pConsumer->epoch); buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); + buf = taosDecodeFixedI32(buf, &pConsumer->status); buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4b78f6eb64..21ebea258d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -59,9 +59,10 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pConsumer->recentRemovedTopics = taosArrayInit(0, sizeof(char*)); pConsumer->epoch = 1; pConsumer->consumerId = consumerId; - pConsumer->status = MQ_CONSUMER_STATUS__INIT; + atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT); strcpy(pConsumer->cgroup, cgroup); taosInitRWLatch(&pConsumer->lock); return pConsumer; @@ -169,7 +170,7 @@ SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) { SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); if (pConsumer == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; } return pConsumer; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9b0bad9827..344eab38b9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1523,4 +1523,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 4fab193a83..2a3e0008a2 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -55,7 +55,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pSub); + const SMqConsumerEp *pConsumerEp); static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp); @@ -72,6 +72,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg); @@ -105,13 +106,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj } static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { - SMqSetCVgReq req = { - .vgId = pConsumerEp->vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, + SMqMVRebReq req = { + .vgId = pConsumerEp->vgId, + .oldConsumerId = pConsumerEp->oldConsumerId, + .newConsumerId = pConsumerEp->consumerId, }; - int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -123,7 +124,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume pMsgHead->vgId = htonl(pConsumerEp->vgId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqSetCVgReq(&abuf, &req); + tEncodeSMqMVRebReq(&abuf, &req); *pBuf = buf; *pLen = tlen; @@ -132,6 +133,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume } static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) { + ASSERT(pConsumerEp->oldConsumerId != -1); int32_t vgId = pConsumerEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); @@ -145,7 +147,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.pCont = buf; action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_SET_CONN; + action.msgType = TDMT_VND_MQ_REB; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -158,8 +160,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { SMqSetCVgReq req = {0}; - req.oldConsumerId = pConsumerEp->consumerId; - req.newConsumerId = -1; + req.consumerId = pConsumerEp->consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); @@ -208,6 +209,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); if (pConsumer == NULL) { @@ -216,10 +218,19 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + // TODO + int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); + mTrace("try to get sub ep, old val: %d", hbStatus); + atomic_store_32(&pConsumer->hbStatus, 0); + /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ + /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ + /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ + strcpy(rsp.cgroup, pReq->cgroup); rsp.consumerId = consumerId; rsp.epoch = pConsumer->epoch; - if (pReq->epoch != rsp.epoch) { + if (epoch != rsp.epoch) { + mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch); SArray *pTopics = pConsumer->currentTopics; int sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -258,6 +269,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { void *abuf = buf; tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tDeleteSMqCMGetSubEpRsp(&rsp); + mndReleaseConsumer(pMnode, pConsumer); pMsg->pCont = buf; pMsg->contLen = tlen; return 0; @@ -299,7 +311,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) break; - int32_t hbStatus = atomic_fetch_add_32(&pConsumer->hbStatus, 1); + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) { int32_t old = atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST); @@ -312,10 +324,6 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId); } - /*pRebMsg->consumerId = pConsumer->consumerId;*/ - /*SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = - * sizeof(SMqDoRebalanceMsg)};*/ - /*pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);*/ } } int32_t status = atomic_load_32(&pConsumer->status); @@ -373,9 +381,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { mInfo("mq remove lost consumer %ld", lostConsumerId); for (int j = 0; j < taosArrayGetSize(pSub->consumers); j++) { - SMqConsumerEp *pConsumerEp = taosArrayGet(pSub->consumers, j); - if (pConsumerEp->consumerId == lostConsumerId) { - taosArrayPush(pSub->unassignedVg, pConsumerEp); + SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); + if (pSubConsumer->consumerId == lostConsumerId) { + taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo); + taosArrayPush(pSub->lostConsumers, pSubConsumer); taosArrayRemove(pSub->consumers, j); break; } @@ -389,73 +398,90 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { int32_t vgEachConsumer = vgNum / consumerNum; int32_t imbalanceVg = vgNum % consumerNum; int32_t imbalanceSolved = 0; - SArray *unassignedVgStash = taosArrayInit(0, sizeof(SMqConsumerEp)); - SArray *unassignedConsumerIdx = taosArrayInit(0, sizeof(int32_t)); // iterate all consumers, set unassignedVgStash for (int i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; - if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; - else vgThisConsumerAfterRb = vgEachConsumer; + int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int vgThisConsumerAfterRb; + if (i < imbalanceVg) + vgThisConsumerAfterRb = vgEachConsumer + 1; + else + vgThisConsumerAfterRb = vgEachConsumer; - mInfo("mq consumer:%ld ,connectted vgroup change from %d %d", pSubConsumer->consumerId, vgThisConsumerBeforeRb, vgThisConsumerAfterRb); + mInfo("mq consumer:%ld, connectted vgroup number change from %d to %d", pSubConsumer->consumerId, + vgThisConsumerBeforeRb, vgThisConsumerAfterRb); - while(taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { + while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo); ASSERT(pConsumerEp != NULL); ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId); - taosArrayPush(unassignedVgStash, pConsumerEp); + taosArrayPush(pSub->unassignedVg, pConsumerEp); } - SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); - int32_t status = atomic_load_32(&pRebConsumer->status); - if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || - (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || - (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST) - ) { - pRebConsumer->epoch++; - if (vgThisConsumerAfterRb != 0) { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); - } else { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); - } - - mInfo("mq consumer:%ld , status change from %d %d", pRebConsumer->consumerId, status, pRebConsumer->status); - - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pConsumerRaw); + SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); + int32_t status = atomic_load_32(&pRebConsumer->status); + if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || + (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || + (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) { + pRebConsumer->epoch++; + if (vgThisConsumerAfterRb != 0) { + atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); + } else { + atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); } - mndReleaseConsumer(pMnode, pRebConsumer); + + mInfo("mq consumer:%ld, status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); + + SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); + sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pConsumerRaw); + } + mndReleaseConsumer(pMnode, pRebConsumer); } - //assign to vgroup - if (taosArrayGetSize(unassignedVgStash) != 0) { + // assign to vgroup + if (taosArrayGetSize(pSub->unassignedVg) != 0) { for (int i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; - if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; - else vgThisConsumerAfterRb = vgEachConsumer; + int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int vgThisConsumerAfterRb; + if (i < imbalanceVg) + vgThisConsumerAfterRb = vgEachConsumer + 1; + else + vgThisConsumerAfterRb = vgEachConsumer; - while(taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerBeforeRb) { - SMqConsumerEp* pConsumerEp = taosArrayPop(unassignedVgStash); + while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) { + SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); ASSERT(pConsumerEp != NULL); - ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId); - pConsumerEp->oldConsumerId = pConsumerEp->consumerId; pConsumerEp->consumerId = pSubConsumer->consumerId; + taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); - mInfo("mq consumer:%ld , assign vgroup %d, previously assigned to consumer %ld", pSubConsumer->consumerId, pConsumerEp->vgId, pConsumerEp->oldConsumerId); + if (pConsumerEp->oldConsumerId == -1) { + char *topic; + char *cgroup; + mndSplitSubscribeKey(pSub->key, &topic, &cgroup); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic, + pConsumerEp->consumerId); + + mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); + mndReleaseTopic(pMnode, pTopic); + free(topic); + free(cgroup); + } else { + mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId, + pConsumerEp->oldConsumerId, pConsumerEp->consumerId); + + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + } } } } - ASSERT(taosArrayGetSize(unassignedVgStash) == 0); + ASSERT(taosArrayGetSize(pSub->unassignedVg) == 0); // TODO: log rebalance statistics SSdbRaw *pSubRaw = mndSubActionEncode(pSub); @@ -466,10 +492,12 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { } if (mndTransPrepare(pMnode, pTrans) != 0) { mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + taosHashCleanup(pReq->rebSubHash); mndTransDrop(pTrans); return -1; } + taosHashCleanup(pReq->rebSubHash); mndTransDrop(pTrans); return 0; } @@ -695,11 +723,6 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql); return -1; } - /*if (pArray && taosArrayGetSize(pArray) != 1) {*/ - /*terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;*/ - /*mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray));*/ - /*return -1;*/ - /*}*/ SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; @@ -721,13 +744,13 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp) { + ASSERT(pConsumerEp->oldConsumerId == -1); int32_t vgId = pConsumerEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, + .consumerId = pConsumerEp->consumerId, .sql = pTopic->sql, .logicalPlan = pTopic->logicalPlan, .physicalPlan = pTopic->physicalPlan, @@ -880,7 +903,7 @@ SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const c SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); free(key); if (pSub == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } return pSub; } @@ -889,7 +912,7 @@ SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key) { SSdb *pSdb = pMnode->pSdb; SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); if (pSub == NULL) { - /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } return pSub; } @@ -1018,8 +1041,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { pConsumerEp->oldConsumerId = pConsumerEp->consumerId; pConsumerEp->consumerId = consumerId; taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp); - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - atomic_store_32(&pConsumer->hbStatus, MQ_CONSUMER_STATUS__ACTIVE); + if (pConsumerEp->oldConsumerId == -1) { + mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); + } else { + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + } + // do not set status active to trigger rebalance + /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ } SSdbRaw *pRaw = mndSubActionEncode(pSub); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index ca851748d6..211cfbd1f0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -389,7 +389,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER) { + if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 2aceaeb016..9d396e0771 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -35,155 +35,14 @@ extern "C" { #endif -#define TQ_BUFFER_SIZE 8 - -typedef struct STqRspHandle { - void* handle; - void* ahandle; -} STqRspHandle; - -typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; - -typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; - STqReadHandle* pReadHandle; - SSubQueryMsg* pQueryMsg; -} STqTaskItem; - -// new version -typedef struct STqBuffer { - int64_t firstOffset; - int64_t lastOffset; - STqTaskItem output[TQ_BUFFER_SIZE]; -} STqBuffer; - -typedef struct STqTopicHandle { - char topicName[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - int64_t committedOffset; - int64_t currentOffset; - STqBuffer buffer; - SWalReadHandle* pReadhandle; -} STqTopicHandle; - -typedef struct STqConsumerHandle { - int64_t consumerId; - int64_t epoch; - char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray -} STqConsumerHandle; +typedef struct STQ STQ; +// memory allocator provided by vnode typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; } STqMemRef; -typedef struct STqSerializedHead { - int16_t ver; - int16_t action; - int32_t checksum; - int64_t ssize; - char content[]; -} STqSerializedHead; - -typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); -typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); -typedef void (*FTqDelete)(void*); - -#define TQ_BUCKET_MASK 0xFF -#define TQ_BUCKET_SIZE 256 - -#define TQ_PAGE_SIZE 4096 -// key + offset + size -#define TQ_IDX_SIZE 24 -// 4096 / 24 -#define TQ_MAX_IDX_ONE_PAGE 170 -// 24 * 170 -#define TQ_IDX_PAGE_BODY_SIZE 4080 -// 4096 - 4080 -#define TQ_IDX_PAGE_HEAD_SIZE 16 - -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 -#define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 - -#define TQ_SVER 0 - -// TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 - -#define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 - -static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } - -static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } - -static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; - -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE - -typedef struct STqMetaHandle { - int64_t key; - int64_t offset; - int64_t serializedSize; - void* valueInUse; - void* valueInTxn; -} STqMetaHandle; - -typedef struct STqMetaList { - STqMetaHandle handle; - struct STqMetaList* next; - // struct STqMetaList* inTxnPrev; - // struct STqMetaList* inTxnNext; - struct STqMetaList* unpersistPrev; - struct STqMetaList* unpersistNext; -} STqMetaList; - -typedef struct STqMetaStore { - STqMetaList* bucket[TQ_BUCKET_SIZE]; - // a table head - STqMetaList* unpersistHead; - // topics that are not connectted - STqMetaList* unconnectTopic; - - // TODO:temporaral use, to be replaced by unified tfile - int fileFd; - // TODO:temporaral use, to be replaced by unified tfile - int idxFd; - - char* dirPath; - int32_t tqConfigFlag; - FTqSerialize pSerializer; - FTqDeserialize pDeserializer; - FTqDelete pDeleter; -} STqMetaStore; - -typedef struct STQ { - // the collection of groups - // the handle of meta kvstore - char* path; - STqCfg* tqConfig; - STqMemRef tqMemRef; - STqMetaStore* tqMeta; - SWal* pWal; - SMeta* pMeta; -} STQ; - -typedef struct STqMgmt { - int8_t inited; - tmr_h timer; -} STqMgmt; - -static STqMgmt tqMgmt; - // init once int tqInit(); void tqCleanUp(); @@ -192,12 +51,13 @@ void tqCleanUp(); STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); -// void* will be replace by a msg type +// required by vnode int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); +int32_t tqProcessRebReq(STQ* pTq, char* msg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7549772613..520a4d026a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -42,6 +42,7 @@ typedef struct STqCfg { typedef struct SVnodeCfg { int32_t vgId; + uint64_t dbId; SDnode *pDnode; STfs *pTfs; uint64_t wsize; diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index cefe9eff72..13c0150cd2 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -63,8 +63,145 @@ extern int32_t tqDebugFlag; } \ } -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); +#define TQ_BUFFER_SIZE 8 + +#define TQ_BUCKET_MASK 0xFF +#define TQ_BUCKET_SIZE 256 + +#define TQ_PAGE_SIZE 4096 +// key + offset + size +#define TQ_IDX_SIZE 24 +// 4096 / 24 +#define TQ_MAX_IDX_ONE_PAGE 170 +// 24 * 170 +#define TQ_IDX_PAGE_BODY_SIZE 4080 +// 4096 - 4080 +#define TQ_IDX_PAGE_HEAD_SIZE 16 + +#define TQ_ACTION_CONST 0 +#define TQ_ACTION_INUSE 1 +#define TQ_ACTION_INUSE_CONT 2 +#define TQ_ACTION_INTXN 3 + +#define TQ_SVER 0 + +// TODO: inplace mode is not implemented +#define TQ_UPDATE_INPLACE 0 +#define TQ_UPDATE_APPEND 1 + +#define TQ_DUP_INTXN_REWRITE 0 +#define TQ_DUP_INTXN_REJECT 2 + +static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } + +static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } + +static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; + +#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE + +typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; + +typedef struct { + int16_t ver; + int16_t action; + int32_t checksum; + int64_t ssize; + char content[]; +} STqSerializedHead; + +typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); +typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); +typedef void (*FTqDelete)(void*); +typedef struct STqMetaHandle { + int64_t key; + int64_t offset; + int64_t serializedSize; + void* valueInUse; + void* valueInTxn; +} STqMetaHandle; + +typedef struct STqMetaList { + STqMetaHandle handle; + struct STqMetaList* next; + // struct STqMetaList* inTxnPrev; + // struct STqMetaList* inTxnNext; + struct STqMetaList* unpersistPrev; + struct STqMetaList* unpersistNext; +} STqMetaList; + +typedef struct { + STqMetaList* bucket[TQ_BUCKET_SIZE]; + // a table head + STqMetaList* unpersistHead; + // topics that are not connectted + STqMetaList* unconnectTopic; + + // TODO:temporaral use, to be replaced by unified tfile + int fileFd; + // TODO:temporaral use, to be replaced by unified tfile + int idxFd; + + char* dirPath; + int32_t tqConfigFlag; + FTqSerialize pSerializer; + FTqDeserialize pDeserializer; + FTqDelete pDeleter; +} STqMetaStore; + +struct STQ { + // the collection of groups + // the handle of meta kvstore + char* path; + STqCfg* tqConfig; + STqMemRef tqMemRef; + STqMetaStore* tqMeta; + SWal* pWal; + SMeta* pMeta; +}; + +typedef struct { + int8_t inited; + tmr_h timer; +} STqMgmt; + +static STqMgmt tqMgmt; + +typedef struct { + int8_t status; + int64_t offset; + qTaskInfo_t task; + STqReadHandle* pReadHandle; +} STqTaskItem; + +// new version +typedef struct { + int64_t firstOffset; + int64_t lastOffset; + STqTaskItem output[TQ_BUFFER_SIZE]; +} STqBuffer; + +typedef struct { + char topicName[TSDB_TOPIC_FNAME_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + char* qmsg; + int64_t committedOffset; + int64_t currentOffset; + STqBuffer buffer; + SWalReadHandle* pReadhandle; +} STqTopicHandle; + +typedef struct { + int64_t consumerId; + int64_t epoch; + char cgroup[TSDB_TOPIC_FNAME_LEN]; + SArray* topics; // SArray +} STqConsumerHandle; + +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f65c0d9893..8ffedfebe4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -134,7 +134,6 @@ const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHan return NULL; } - int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t reqId = pReq->reqId; @@ -144,6 +143,8 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; + /*printf("vg %d get consume req\n", pReq->head.vgId);*/ + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); if (pConsumer == NULL) { pMsg->pCont = NULL; @@ -158,6 +159,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); // TODO: support multiple topic in one req if (strcmp(pTopic->topicName, pReq->topic) != 0) { + /*ASSERT(false);*/ continue; } @@ -171,7 +173,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - pTopic->committedOffset = pReq->offset-1; + pTopic->committedOffset = pReq->offset - 1; } rsp.committedOffset = pTopic->committedOffset; @@ -181,6 +183,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (fetchOffset <= pTopic->committedOffset) { fetchOffset = pTopic->committedOffset + 1; } + /*printf("vg %d fetch Offset %ld\n", pReq->head.vgId, fetchOffset);*/ int8_t pos; int8_t skip = 0; SWalHead* pHead; @@ -231,7 +234,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { break; } } - //TODO copy + // TODO copy rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; @@ -266,11 +269,11 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); if (rsp.pBlockData) { - taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock); + taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); rsp.pBlockData = NULL; /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ - /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ - /*tDeleteSSDataBlock(pBlock);*/ + /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ + /*tDeleteSSDataBlock(pBlock);*/ /*}*/ /*taosArrayDestroy(rsp.pBlockData);*/ } @@ -281,30 +284,36 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } +int32_t tqProcessRebReq(STQ* pTq, char* msg) { + SMqMVRebReq req = {0}; + tDecodeSMqMVRebReq(msg, &req); + + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); + ASSERT(pConsumer); + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandlePurge(pTq->tqMeta, req.oldConsumerId); + terrno = TSDB_CODE_SUCCESS; + return 0; +} + int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { SMqSetCVgReq req = {0}; tDecodeSMqSetCVgReq(msg, &req); - - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); + + /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ + STqConsumerHandle* pConsumer = calloc(1, sizeof(STqConsumerHandle)); if (pConsumer == NULL) { - pConsumer = calloc(sizeof(STqConsumerHandle), 1); - if (pConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - } else { - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - tqHandlePurge(pTq->tqMeta, req.oldConsumerId); - terrno = TSDB_CODE_SUCCESS; - return 0; + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; } + strcpy(pConsumer->cgroup, req.cgroup); pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); - pConsumer->consumerId = req.newConsumerId; + pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); + STqTopicHandle* pTopic = calloc(1, sizeof(STqTopicHandle)); if (pTopic == NULL) { free(pConsumer); return -1; @@ -313,6 +322,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->sql = req.sql; pTopic->logicalPlan = req.logicalPlan; pTopic->physicalPlan = req.physicalPlan; + pTopic->qmsg = req.qmsg; pTopic->committedOffset = -1; pTopic->currentOffset = -1; @@ -324,13 +334,13 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta }; + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } taosArrayPush(pConsumer->topics, pTopic); - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.consumerId); terrno = TSDB_CODE_SUCCESS; return 0; } @@ -416,7 +426,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t numOfCols = pHandle->pSchema->numOfCols; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); - //TODO: stable case + // TODO: stable case if (colNumNeed > pSchemaWrapper->nCols) { colNumNeed = pSchemaWrapper->nCols; } @@ -432,7 +442,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } - SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; + SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; SColumnInfoData colInfo = {0}; int sz = numOfRows * pColSchema->bytes; colInfo.info.bytes = pColSchema->bytes; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d220966ba6..121be98572 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -289,7 +289,6 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { return 0; } -// TODO: wrap in tfile int32_t tqStorePersist(STqMetaStore* pMeta) { STqIdxPageBuf idxBuf; int64_t* bufPtr = (int64_t*)idxBuf.buffer; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 6bbf3b959d..c748907d6c 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -29,6 +29,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { cfg.vgId = pVnodeCfg->vgId; cfg.pDnode = pVnodeCfg->pDnode; cfg.pTfs = pVnodeCfg->pTfs; + cfg.dbId = pVnodeCfg->dbId; } // Validate options diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index a45a9d5a72..f541834bec 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -93,6 +93,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (pTbCfg->type == META_CHILD_TABLE) { pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); if (pStbCfg == NULL) { + code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; } @@ -116,9 +117,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols); pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { + code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; } + pTbMetaMsg->dbId = htobe64(pVnode->config.dbId); memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 28487821e6..4d048bc6e2 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -130,6 +130,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } break; + case TDMT_VND_MQ_REB: { + if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) { + + } + } break; default: ASSERT(0); break; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8f24374387..5857437f9d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,7 +27,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 +#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 @@ -47,9 +47,19 @@ enum { CTG_RENT_STABLE, }; +enum { + CTG_ACT_UPDATE_VG = 0, + CTG_ACT_UPDATE_TBL, + CTG_ACT_REMOVE_DB, + CTG_ACT_REMOVE_STB, + CTG_ACT_REMOVE_TBL, + CTG_ACT_MAX +}; + typedef struct SCtgDebug { bool lockDebug; bool cacheDebug; + bool apiDebug; uint32_t showCachePeriodSec; } SCtgDebug; @@ -65,7 +75,7 @@ typedef struct SCtgDBCache { SRWLatch vgLock; uint64_t dbId; int8_t deleted; - SDBVgroupInfo *vgInfo; + SDBVgInfo *vgInfo; SCtgTbMetaCache tbCache; } SCtgDBCache; @@ -85,7 +95,6 @@ typedef struct SCtgRentMgmt { typedef struct SCatalog { uint64_t clusterId; - SRWLatch dbLock; SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -96,7 +105,8 @@ typedef struct SCtgApiStat { } SCtgApiStat; typedef struct SCtgRuntimeStat { - + uint64_t qNum; + uint64_t qDoneNum; } SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -109,15 +119,70 @@ typedef struct SCatalogStat { SCtgCacheStat cache; } SCatalogStat; +typedef struct SCtgUpdateVgMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgInfo* dbInfo; +} SCtgUpdateVgMsg; + +typedef struct SCtgUpdateTblMsg { + SCatalog* pCtg; + STableMetaOutput* output; +} SCtgUpdateTblMsg; + +typedef struct SCtgRemoveDBMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; +} SCtgRemoveDBMsg; + +typedef struct SCtgRemoveStbMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; + uint64_t suid; +} SCtgRemoveStbMsg; + +typedef struct SCtgMetaAction { + int32_t act; + void *data; +} SCtgMetaAction; + +typedef struct SCtgQNode { + SCtgMetaAction action; + struct SCtgQNode *next; +} SCtgQNode; + typedef struct SCatalogMgmt { bool exit; SRWLatch lock; + SRWLatch qlock; + SCtgQNode *head; + SCtgQNode *tail; + tsem_t sem; + uint64_t qRemainNum; + pthread_t updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; } SCatalogMgmt; typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); +typedef int32_t (*ctgActFunc)(SCtgMetaAction *); + +typedef struct SCtgAction { + int32_t actId; + char name[32]; + ctgActFunc func; +} SCtgAction; + +#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1) +#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1) + +#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) +#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -130,20 +195,26 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) #define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) -#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgError(param, ...) qError("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCatalog, __VA_ARGS__) +#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST) + +#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) #define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) #define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiDebug) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 +#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock)) + #define CTG_LOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ assert(atomic_load_32((_lock)) >= 0); \ @@ -181,8 +252,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) -#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0) +#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6ecff87a89..f1c2395de4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -18,9 +18,75 @@ #include "tname.h" #include "catalogInt.h" -SCatalogMgmt ctgMgmt = {0}; +int32_t ctgActUpdateVg(SCtgMetaAction *action); +int32_t ctgActUpdateTbl(SCtgMetaAction *action); +int32_t ctgActRemoveDB(SCtgMetaAction *action); +int32_t ctgActRemoveStb(SCtgMetaAction *action); +int32_t ctgActRemoveTbl(SCtgMetaAction *action); +SCatalogMgmt gCtgMgmt = {0}; SCtgDebug gCTGDebug = {0}; +SCtgAction gCtgAction[CTG_ACT_MAX] = {{ + CTG_ACT_UPDATE_VG, + "update vgInfo", + ctgActUpdateVg + }, + { + CTG_ACT_UPDATE_TBL, + "update tbMeta", + ctgActUpdateTbl + }, + { + CTG_ACT_REMOVE_DB, + "remove DB", + ctgActRemoveDB + }, + { + CTG_ACT_REMOVE_STB, + "remove stbMeta", + ctgActRemoveStb + }, + { + CTG_ACT_REMOVE_TBL, + "remove tbMeta", + ctgActRemoveTbl + } +}; + +int32_t ctgDbgEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockDebug = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheDebug = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiDebug = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgDbgGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; @@ -44,25 +110,25 @@ int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { return num; } -int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { - if (NULL == pCatalog || NULL == pCatalog->dbCache) { +int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { return 0; } switch (type) { case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCatalog->dbCache); + return (int32_t)taosHashGetSize(pCtg->dbCache); case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->dbRent); + return ctgDbgGetRentNum(&pCtg->dbRent); case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->stbRent); + return ctgDbgGetRentNum(&pCtg->stbRent); default: break; } SCtgDBCache *dbCache = NULL; int32_t num = 0; - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { dbCache = (SCtgDBCache *)pIter; switch (type) { @@ -76,7 +142,7 @@ int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { ctgError("invalid type:%d", type); break; } - pIter = taosHashIterate(pCatalog->dbCache, pIter); + pIter = taosHashIterate(pCtg->dbCache, pIter); } return num; @@ -108,79 +174,57 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { -void ctgDbgShowClusterCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void ctgDbgShowClusterCache(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCatalog->clusterId, pCatalog); - CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_RENT_NUM)); + CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); + CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - ctgDbgShowDBCache(pCatalog->dbCache); -} - -int32_t ctgInitDBCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog->dbCache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { - taosHashCleanup(cache); - } - } - - return TSDB_CODE_SUCCESS; + ctgDbgShowDBCache(pCtg->dbCache); } -int32_t ctgInitTbMetaCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.metaCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == metaCache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.metaCache, NULL, metaCache)) { - taosHashCleanup(metaCache); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgInitStbCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.stbCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = gCtgMgmt.head; - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } + SCtgQNode *node = gCtgMgmt.head->next; + gCtgMgmt.head = gCtgMgmt.head->next; - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { - taosHashCleanup(cache); - } + CTG_QUEUE_SUB(); + + tfree(orig); + + *action = &node->action; +} + + +int32_t ctgPushAction(SCtgMetaAction *action) { + SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); } + + node->action = *action; + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.qlock); + gCtgMgmt.tail->next = node; + gCtgMgmt.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.qlock); + + CTG_QUEUE_ADD(); + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + + tsem_post(&gCtgMgmt.sem); return TSDB_CODE_SUCCESS; } - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { return; @@ -214,93 +258,183 @@ void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + tfree(vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - atomic_store_8(&dbCache->deleted, 1); - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - } + ctgFreeVgInfo (dbCache->vgInfo); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); ctgFreeTableMetaCache(&dbCache->tbCache); } -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stbRent); - if (pCatalog->dbCache) { - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; + atomic_store_8(&dbCache->deleted, 1); + ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCatalog->dbCache, pIter); + + pIter = taosHashIterate(pCtg->dbCache, pIter); } - taosHashCleanup(pCatalog->dbCache); + taosHashCleanup(pCtg->dbCache); } - free(pCatalog); + free(pCtg); } -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbFName, SCtgDBCache **dbCache, bool *inCache) { - if (NULL == pCatalog->dbCache) { + +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgLock); + + if (dbCache->deleted) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + + if (NULL == dbCache->vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + *inCache = false; + ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + return TSDB_CODE_SUCCESS; + } + + *inCache = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->deleted) { + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + return TSDB_CODE_SUCCESS; +} + +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); +} + +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); +} + +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); +} + + +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + if (NULL == pCtg->dbCache) { + *pCache = NULL; *inCache = false; ctgWarn("empty db cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *cache = NULL; - - while (true) { - cache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - - if (NULL == cache) { - *inCache = false; - ctgWarn("not in db vgroup cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &cache->vgLock); - if (NULL == cache->vgInfo) { - CTG_UNLOCK(CTG_READ, &cache->vgLock); - taosHashRelease(pCatalog->dbCache, cache); - ctgWarn("db cache vgInfo is NULL, dbFName:%s", dbFName); - - continue; - } - - break; + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + *pCache = NULL; + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + ctgAcquireVgInfo(pCtg, dbCache, inCache); + if (!(*inCache)) { + ctgReleaseDBCache(pCtg, dbCache); + + *pCache = NULL; + return TSDB_CODE_SUCCESS; } - *dbCache = cache; + *pCache = dbCache; *inCache = true; - ctgDebug("Got db vgroup from cache, dbFName:%s", dbFName); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { char *msg = NULL; int32_t msgLen = 0; - ctgDebug("try to get db vgroup from mnode, db:%s", input->db); + ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen); if (code) { @@ -318,6 +452,11 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (CTG_DB_NOT_EXIST(rpcRsp.code)) { + ctgDebug("db not exist in mnode, dbFName:%s", input->db); + return rpcRsp.code; + } + ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -328,31 +467,32 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp CTG_ERR_RET(code); } + ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db); + return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("db not exist in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == tbMeta) { - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -361,7 +501,7 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha *exist = 1; - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -369,29 +509,22 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha } -int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); *pTableMeta = NULL; - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - if (NULL == dbCache->tbCache.metaCache) { - *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -402,8 +535,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if (NULL == *pTableMeta) { *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -412,8 +545,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -422,8 +555,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; @@ -431,17 +564,17 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); tfree(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -450,50 +583,53 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { + if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbName); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - ctgInfo("db not in cache, dbFName:%s", dbName); return TSDB_CODE_SUCCESS; } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == pTableMeta) { - ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); - taosHashRelease(pCatalog->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); return TSDB_CODE_SUCCESS; } *tbType = atomic_load_8(&pTableMeta->tableType); - taosHashRelease(dbCache->tbCache.metaCache, dbCache); - taosHashRelease(pCatalog->dbCache, dbCache); + taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -539,15 +675,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -607,12 +743,12 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray** vgroupList) { +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosHashGetSize(vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -620,23 +756,23 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + void *pIter = taosHashIterate(vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgHash, pIter); + pIter = taosHashIterate(vgHash, pIter); vgInfo = NULL; } - *vgroupList = vgList; - vgList = NULL; + *pList = vgList; - ctgDebug("Got vg list from DB, vgNum:%d", vgNum); + ctgDebug("Got vgList from cache, vgNum:%d", vgNum); return TSDB_CODE_SUCCESS; @@ -649,7 +785,7 @@ _return: CTG_RET(code); } -int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); @@ -693,7 +829,6 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn CTG_RET(code); } -#if 1 int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; @@ -713,29 +848,6 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { return 0; } } -#else - -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { - return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { - return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -#endif int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; @@ -932,38 +1044,55 @@ int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t si return TSDB_CODE_SUCCESS; } -int32_t ctgAddDBCache(struct SCatalog *pCatalog, const char *dbFName, SCtgDBCache *dbCache) { +int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t code = 0; - code = taosHashPut(pCatalog->dbCache, dbFName, strlen(dbFName), dbCache, sizeof(SCtgDBCache)); + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.metaCache) { + ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.stbCache) { + ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); if (code) { if (HASH_NODE_EXIST(code)) { ctgDebug("db already in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + goto _return; } ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SDbVgVersion vgVersion = {.dbId = dbCache->dbId, .vgVersion = dbCache->vgInfo ? dbCache->vgInfo->vgVersion : -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("dbCache added, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbCache->dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; _return: - ctgFreeDbCache(dbCache); + ctgFreeDbCache(&newDBCache); CTG_RET(code); } -void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache) { +void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { void *pIter = taosHashIterate(cache->stbCache, NULL); @@ -971,7 +1100,7 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCatalog->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -979,232 +1108,256 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache } } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - - ctgFreeTableMetaCache(cache); } -int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, SCtgDBCache *dbCache, const char* dbFName) { - if (taosHashRemove(pCatalog->dbCache, dbFName, strlen(dbFName))) { +int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + uint64_t dbId = dbCache->dbId; + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + atomic_store_8(&dbCache->deleted, 1); + + ctgRemoveStbRent(pCtg, &dbCache->tbCache); + + ctgFreeDbCache(dbCache); + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - atomic_store_8(&dbCache->deleted, 1); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - ctgInfo("cleanup db vgInfo, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - } - - tfree(dbCache->vgInfo); - } - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgRemoveAndFreeTableMeta(pCatalog, &dbCache->tbCache); - - ctgInfo("db removed from cache, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); - - CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); - - ctgDebug("db removed from rent, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCache(struct SCatalog* pCatalog, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { +int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - - CTG_LOCK(CTG_WRITE, &pCatalog->dbLock); + ctgGetDBCache(pCtg, dbFName, &dbCache); - while (true) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (dbCache) { - // TODO OPEN IT + if (dbCache) { + // TODO OPEN IT #if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } #else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - code = ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName); - taosHashRelease(pCatalog->dbCache, dbCache); - dbCache = NULL; - if (code) { - if (TSDB_CODE_CTG_DB_DROPPED == code) { - continue; - } - - CTG_ERR_JRET(code); - } + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; } - SCtgDBCache newDBCache = {0}; - newDBCache.dbId = dbId; + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } - CTG_ERR_JRET(ctgAddDBCache(pCatalog, dbFName, &newDBCache)); + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { + int32_t code = 0; + SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } -_return: + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + if (dbCache->vgInfo) { + if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(dbCache->vgInfo); + } + + dbCache->vgInfo = dbInfo; + + *pDbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWReleaseVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); CTG_RET(code); } +int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { + SCtgTbMetaCache *tbCache = &dbCache->tbCache; -int32_t ctgUpdateTbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - CTG_LOCK(CTG_READ, &tbCache->metaLock); + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + origType = orig->tableType; + origSuid = orig->suid; + + if (origType == TSDB_SUPER_TABLE && ((!isStb) || origSuid != meta->suid)) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + } + } + + if (isStb) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + } + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + if (isStb) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgUpdateStbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - bool newAdded = false; - int32_t code = 0; - SSTableMetaVersion metaRent = {.suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *orig = taosHashAcquire(tbCache->metaCache, tbName, strlen(tbName)); - if (orig) { - if (orig->suid != meta->suid) { - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, db:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } - - ctgMetaRentRemove(&pCatalog->stbRent, orig->suid, ctgSTableVersionCompare); - } - - taosHashRelease(tbCache->metaCache, orig); + if (!isStb) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, tbCache, dbFName, tbName, meta, metaSize)); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPutExt(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (isStb && origSuid == meta->suid) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - ctgDebug("update stable to cache, suid:%"PRIx64, meta->suid); - - if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); + return TSDB_CODE_SUCCESS; +} + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { + *dst = malloc(sizeof(SDBVgInfo)); + if (NULL == *dst) { + qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + memcpy(*dst, src, sizeof(SDBVgInfo)); + + size_t hashSize = taosHashGetSize(src->vgHash); + (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == (*dst)->vgHash) { + qError("taosHashInit %d failed", (int32_t)hashSize); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t *vgId = NULL; + void *pIter = taosHashIterate(src->vgHash, NULL); + while (pIter) { + taosHashGetKey(pIter, (void **)&vgId, NULL); + + if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { + qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); + taosHashCancelIterate(src->vgHash, pIter); + taosHashCleanup((*dst)->vgHash); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(src->vgHash, pIter); + } + + return TSDB_CODE_SUCCESS; - -_return: - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_RET(code); } -int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - CTG_ERR_RET(ctgInitDBCache(pCatalog)); - - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, output->dbFName, output->dbId, &dbCache)); - - CTG_ERR_JRET(ctgInitTbMetaCache(pCatalog, dbCache)); - CTG_ERR_JRET(ctgInitStbCache(pCatalog, dbCache)); - - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); - } - - if (CTG_IS_META_CTABLE(output->metaType)) { - goto _return; - } - - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); - - if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { - CTG_ERR_JRET(ctgUpdateStbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } else { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } - -_return: - - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - - CTG_RET(code); -} - -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } - - ctgDebug("failed to get DB vgroupInfo from cache, dbName:%s, load it from mnode, update:%d", dbFName, forceUpdate); } SUseDbOutput DbOut = {0}; @@ -1213,221 +1366,490 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - while (true) { - CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbFName, DbOut.dbId, DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); - if (!inCache) { - ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbFName); - continue; + CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(DbOut.dbVgroup); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = DbOut.dbId; + msg->dbInfo = DbOut.dbVgroup; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(*pInfo); + tfree(msg); + + *pInfo = DbOut.dbVgroup; + + CTG_RET(code); +} + + +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { + *pOutput = malloc(sizeof(STableMetaOutput)); + if (NULL == *pOutput) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*pOutput, output, sizeof(STableMetaOutput)); + + if (output->tbMeta) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + (*pOutput)->tbMeta = malloc(metaSize); + if (NULL == (*pOutput)->tbMeta) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + tfree(*pOutput); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - break; + memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); } return TSDB_CODE_SUCCESS; } -int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { - *removed = false; - - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, stbName, strlen(stbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - taosHashRelease(pCatalog->dbCache, dbCache); - - *removed = true; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); - - STableMetaOutput voutput = {0}; - STableMetaOutput moutput = {0}; - STableMetaOutput *output = &voutput; + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + SCtgUpdateTblMsg *msg = NULL; + STableMetaOutput moutput = {0}; + STableMetaOutput *output = malloc(sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); - if (CTG_IS_META_NULL(moutput.metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); - } else { - output = &moutput; + if (CTG_IS_META_NULL(output->metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); - if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); - voutput.metaType = moutput.metaType; + tfree(output->tbMeta); - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; - moutput.tbMeta = NULL; - } else if (CTG_IS_META_BOTH(voutput.metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { - SET_META_TYPE_NULL(voutput.metaType); + SET_META_TYPE_NULL(output->metaType); } - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; + tfree(output->tbMeta); + output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(voutput.tbMeta); + tfree(output->tbMeta); - SET_META_TYPE_CTABLE(voutput.metaType); + SET_META_TYPE_CTABLE(output->metaType); } } } if (CTG_IS_META_NULL(output->metaType)) { - ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName)); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output)); + if (CTG_IS_META_TABLE(output->metaType)) { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + } else { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + } + + if (pOutput) { + CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); + } + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; _return: - tfree(voutput.tbMeta); - tfree(moutput.tbMeta); + tfree(output->tbMeta); + tfree(output); + tfree(msg); CTG_RET(code); } -int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } + + tfree(*pTableMeta); } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); CTG_SET_STABLE(isSTable, tbType); } - CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable)); + STableMetaOutput *output = NULL; - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; + } + + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + tfree(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + // HANDLE ONLY CHILD TABLE META + + SName stbName = *pTableName; + strcpy(stbName.tname, output->tbName); + + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); if (0 == exist) { - ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", tNameGetTableName(pTableName)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); + } + + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + +_return: + + tfree(output); + + if (*pTableMeta) { + ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); + } + + CTG_RET(code); +} + + + +int32_t ctgActUpdateVg(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateVgMsg *msg = action->data; + + CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + +_return: + + tfree(msg->dbInfo); + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveDB(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveDBMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; } + if (dbCache->dbId != msg->dbId) { + ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + goto _return; + } + + CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); + +_return: + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActUpdateTbl(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + STableMetaOutput* output = msg->output; + SCtgDBCache *dbCache = NULL; + + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + } + + if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + } + +_return: + + if (output) { + tfree(output->tbMeta); + tfree(output); + } + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActRemoveStb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveStbMsg *msg = action->data; + bool removed = false; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + +_return: + + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveTbl(SCtgMetaAction *action) { + +} + + + +void* ctgUpdateThreadFunc(void* param) { + setThreadName("catalog"); + + qInfo("catalog update thread started"); + + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); + + while (true) { + tsem_wait(&gCtgMgmt.sem); + + if (atomic_load_8(&gCtgMgmt.exit)) { + break; + } + + SCtgMetaAction *action = NULL; + ctgPopAction(&action); + + qDebug("process %s action", gCtgAction[action->act].name); + + (*gCtgAction[action->act].func)(action); + + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + } + + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + + qInfo("catalog update thread stopped"); + + return NULL; +} + + +int32_t ctgStartUpdateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + + pthread_attr_destroy(&thAttr); return TSDB_CODE_SUCCESS; } int32_t catalogInit(SCatalogCfg *cfg) { - if (ctgMgmt.pCluster) { + if (gCtgMgmt.pCluster) { qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8(&ctgMgmt.exit, false); + atomic_store_8(&gCtgMgmt.exit, false); if (cfg) { - memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); + memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); - if (ctgMgmt.cfg.maxDBCacheNum == 0) { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + if (gCtgMgmt.cfg.maxDBCacheNum == 0) { + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; } - if (ctgMgmt.cfg.maxTblCacheNum == 0) { - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; + if (gCtgMgmt.cfg.maxTblCacheNum == 0) { + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; } - if (ctgMgmt.cfg.dbRentSec == 0) { - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.dbRentSec == 0) { + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stbRentSec == 0) { - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.stbRentSec == 0) { + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if (NULL == ctgMgmt.pCluster) { + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); + CTG_ERR_RET(ctgStartUpdateThread()); + + tsem_init(&gCtgMgmt.sem, 0, 0); + + gCtgMgmt.head = calloc(1, sizeof(SCtgQNode)); + if (NULL == gCtgMgmt.head) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + gCtgMgmt.tail = gCtgMgmt.head; + + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == ctgMgmt.pCluster) { + if (NULL == gCtgMgmt.pCluster) { qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1436,7 +1858,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; @@ -1452,10 +1874,22 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == clusterCtg->dbCache) { + qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { ctgFreeHandle(clusterCtg); @@ -1482,97 +1916,82 @@ _return: CTG_RET(code); } -void catalogFreeHandle(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void catalogFreeHandle(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - if (taosHashRemove(ctgMgmt.pCluster, &pCatalog->clusterId, sizeof(pCatalog->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCatalog->clusterId); + if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); return; } - uint64_t clusterId = pCatalog->clusterId; + uint64_t clusterId = pCtg->clusterId; - ctgFreeHandle(pCatalog); + ctgFreeHandle(pCtg); ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { - if (NULL == pCatalog || NULL == dbName || NULL == version) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version) { CTG_API_ENTER(); - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg || NULL == dbFName || NULL == version) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCtg->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbName:%s", dbName); + ctgInfo("empty db cache, dbFName:%s", dbFName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == db) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_LOCK(CTG_READ, &db->vgLock); - - if (NULL == db->vgInfo) { - CTG_UNLOCK(CTG_READ, &db->vgLock); + bool inCache = false; + ctgAcquireVgInfo(pCtg, dbCache, &inCache); + if (!inCache) { + ctgReleaseDBCache(pCtg, dbCache); *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = db->vgInfo->vgVersion; - CTG_UNLOCK(CTG_READ, &db->vgLock); - - taosHashRelease(pCatalog->dbCache, db); + *version = dbCache->vgInfo->vgVersion; - ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { - if (NULL == pCatalog || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { CTG_API_ENTER(); - SCtgDBCache* dbCache = NULL; - SVgroupInfo *vgInfo = NULL; + if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache)); - - int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); - vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit %d failed", vgNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache, &vgInfo)); + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; } - void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - - if (NULL == taosArrayPush(vgList, vgInfo)) { - ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); - vgInfo = NULL; - } + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, &vgList)); *vgroupList = vgList; vgList = NULL; @@ -1580,248 +1999,268 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); } - if (vgList) { - taosArrayDestroy(vgList); - vgList = NULL; + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId, SDBVgroupInfo* dbInfo) { - int32_t code = 0; - +int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); + + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName || NULL == dbInfo) { + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_ERR_JRET(ctgInitDBCache(pCatalog)); - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, dbFName, dbId, &dbCache)); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->deleted) { - ctgInfo("db is dropping, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_ERR_JRET(TSDB_CODE_CTG_DB_DROPPED); - } - - if (NULL == dbCache->vgInfo) { - dbCache->vgInfo = dbInfo; - } else { - if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { - ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, current:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - - goto _return; - } - - if (dbCache->vgInfo->vgHash) { - ctgDebug("cleanup db vgHash, dbFName:%s", dbFName); - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - dbCache->vgInfo = dbInfo; - } - + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + msg->dbInfo = dbInfo; dbInfo = NULL; - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + action.data = msg; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); + CTG_ERR_JRET(ctgPushAction(&action)); - ctgDebug("dbCache updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(code); + _return: - if (dbCache) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - if (dbInfo) { taosHashCleanup(dbInfo->vgHash); - dbInfo->vgHash = NULL; tfree(dbInfo); } + + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { + CTG_API_ENTER(); + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - if (dbCache->dbId != dbId) { - ctgInfo("db id already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbFName, dbCache->dbId, dbId); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); - CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName)); - _return: - - taosHashRelease(pCatalog->dbCache, dbCache); + + tfree(action.data); CTG_API_LEAVE(code); } -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { + CTG_API_ENTER(); + int32_t code = 0; - bool removed = false; - if (NULL == pCatalog || NULL == dbName || NULL == stbName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - - CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); - if (!removed) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; + SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - - ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_JRET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); - - ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + _return: - + + tfree(action.data); + CTG_API_LEAVE(code); } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); } -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); } -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg) { - STableMetaOutput output = {0}; +int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == rspMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); + } + int32_t code = 0; - CTG_API_ENTER(); + strcpy(output->dbFName, rspMsg->dbFName); + strcpy(output->tbName, rspMsg->tbName); - strcpy(output.dbFName, rspMsg->dbFName); - strcpy(output.tbName, rspMsg->tbName); + output->dbId = rspMsg->dbId; - SET_META_TYPE_TABLE(output.metaType); + SET_META_TYPE_TABLE(output->metaType); - CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output.tbMeta)); + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + CTG_API_LEAVE(code); + _return: - tfree(output.tbMeta); + tfree(output->tbMeta); + tfree(output); + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); } -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - - CTG_API_ENTER(); STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; - *pVgroupList = NULL; + *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); - // TODO REMOEV THIS .... + SHashObj *vgHash = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, false, &dbCache, &vgInfo)); + + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; + } + + /* TODO REMOEV THIS .... if (0 == tbMeta->vgId) { SVgroupInfo vgroup = {0}; - catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pTableName, &vgroup); + catalogGetTableHashVgroup(pCtg, pRpc, pMgmtEps, pTableName, &vgroup); tbMeta->vgId = vgroup.vgId; } - // TODO REMOVE THIS .... + // TODO REMOVE THIS ....*/ if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1837,16 +2276,22 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - *pVgroupList = vgList; + *pVgList = vgList; vgList = NULL; } _return: - tfree(tbMeta); if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + tfree(tbMeta); + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } if (vgList) { @@ -1858,37 +2303,42 @@ _return: } -int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + SCtgDBCache* dbCache = NULL; + int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + int32_t code = 0; pRsp->pTableMeta = NULL; @@ -1909,7 +2359,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); @@ -1937,64 +2387,72 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + //TODO CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num) { - if (NULL == pCatalog || NULL == stables || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + if (NULL == pCtg || NULL == stables || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { - if (NULL == pCatalog || NULL == dbs || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == dbs || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } void catalogDestroy(void) { - if (NULL == ctgMgmt.pCluster || atomic_load_8(&ctgMgmt.exit)) { + qInfo("start to destroy catalog"); + + if (NULL == gCtgMgmt.pCluster || atomic_load_8(&gCtgMgmt.exit)) { return; } - atomic_store_8(&ctgMgmt.exit, true); + atomic_store_8(&gCtgMgmt.exit, true); - CTG_LOCK(CTG_WRITE, &ctgMgmt.lock); + tsem_post(&gCtgMgmt.sem); - SCatalog *pCatalog = NULL; - void *pIter = taosHashIterate(ctgMgmt.pCluster, NULL); - while (pIter) { - pCatalog = *(SCatalog **)pIter; - - if (pCatalog) { - catalogFreeHandle(pCatalog); - } - - pIter = taosHashIterate(ctgMgmt.pCluster, pIter); + while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { + usleep(1); } - taosHashCleanup(ctgMgmt.pCluster); - ctgMgmt.pCluster = NULL; + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); - CTG_UNLOCK(CTG_WRITE, &ctgMgmt.lock); + SCatalog *pCtg = NULL; + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + catalogFreeHandle(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + taosHashCleanup(gCtgMgmt.pCluster); + gCtgMgmt.pCluster = NULL; + + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); qInfo("catalog destroyed"); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 7742d8ee22..263293426e 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -33,18 +33,31 @@ #include "tep.h" #include "trpc.h" #include "tvariant.h" +#include "catalogInt.h" namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); -extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); -extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type); +extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); +extern "C" int32_t ctgDbgEnableDebug(char *option); +extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); -void ctgTestSetPrepareTableMeta(); -void ctgTestSetPrepareCTableMeta(); -void ctgTestSetPrepareSTableMeta(); -void ctgTestSetPrepareMultiSTableMeta(); +void ctgTestSetRspTableMeta(); +void ctgTestSetRspCTableMeta(); +void ctgTestSetRspSTableMeta(); +void ctgTestSetRspMultiSTableMeta(); + +extern "C" SCatalogMgmt gCtgMgmt; + +enum { + CTGT_RSP_VGINFO = 1, + CTGT_RSP_TBMETA, + CTGT_RSP_CTBMETA, + CTGT_RSP_STBMETA, + CTGT_RSP_MSTBMETA, +}; bool ctgTestStop = false; bool ctgTestEnableSleep = false; @@ -68,6 +81,9 @@ char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; +int32_t ctgTestRspFunc[10] = {0}; +int32_t ctgTestRspIdx = 0; + void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SCreateDbReq createReq = {0}; strcpy(createReq.db, "1.db1"); @@ -113,6 +129,8 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + ctgDbgEnableDebug("api"); + char temp[128] = {0}; sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { @@ -179,11 +197,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { +void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -250,7 +268,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { return; } -void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp usedbRsp = {0}; strcpy(usedbRsp.db, ctgTestDbname); usedbRsp.vgVersion = ctgTestVgVersion; @@ -290,7 +308,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM pRsp->pCont = pReq; } -void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -326,7 +344,7 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } -void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -369,7 +387,7 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -412,7 +430,7 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; @@ -458,151 +476,193 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, return; } -void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareTableMeta(); +void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + switch (ctgTestRspFunc[ctgTestRspIdx]) { + case CTGT_RSP_VGINFO: + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_TBMETA: + ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_CTBMETA: + ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_STBMETA: + ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_MSTBMETA: + ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + default: + break; + } + + ctgTestRspIdx++; return; } -void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareCTableMeta(); +void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareSTableMeta(); + ctgTestSetRspCTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareMultiSTableMeta(); + ctgTestSetRspSTableMeta(); return; } -void ctgTestSetPrepareDbVgroups() { +void ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspMultiSTableMeta(); + + return; +} + +void ctgTestSetRspDbVgroups() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); + stub.set(rpcSendRecv, ctgTestRspDbVgroups); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroups); + stub.set(f.second, ctgTestRspDbVgroups); } } } -void ctgTestSetPrepareTableMeta() { +void ctgTestSetRspTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareTableMeta); + stub.set(rpcSendRecv, ctgTestRspTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareTableMeta); + stub.set(f.second, ctgTestRspTableMeta); } } } -void ctgTestSetPrepareCTableMeta() { +void ctgTestSetRspCTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); + stub.set(rpcSendRecv, ctgTestRspCTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareCTableMeta); + stub.set(f.second, ctgTestRspCTableMeta); } } } -void ctgTestSetPrepareSTableMeta() { +void ctgTestSetRspSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); + stub.set(rpcSendRecv, ctgTestRspSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareSTableMeta); + stub.set(f.second, ctgTestRspSTableMeta); } } } -void ctgTestSetPrepareMultiSTableMeta() { +void ctgTestSetRspMultiSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta); + stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareMultiSTableMeta); + stub.set(f.second, ctgTestRspMultiSTableMeta); } } } -void ctgTestSetPrepareDbVgroupsAndNormalMeta() { +void ctgTestSetRspByIdx() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(rpcSendRecv, ctgTestRspByIdx); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(f.second, ctgTestRspByIdx); } } } -void ctgTestSetPrepareDbVgroupsAndChildMeta() { + +void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } } } -void ctgTestSetPrepareDbVgroupsAndSuperMeta() { +void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } } } -void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() { +void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); + } + } +} + +void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto &f : result) { + stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } } } @@ -617,7 +677,7 @@ void *ctgTestGetDbVgroupThread(void *param) { int32_t n = 0; while (!ctgTestStop) { - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); if (code) { assert(0); } @@ -640,12 +700,12 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetSameDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); if (code) { assert(0); } @@ -664,12 +724,12 @@ void *ctgTestSetSameDbVgroupThread(void *param) { void *ctgTestSetDiffDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); if (code) { assert(0); } @@ -719,14 +779,24 @@ void *ctgTestGetCtableMetaThread(void *param) { void *ctgTestSetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; int32_t n = 0; - STableMetaOutput output = {0}; + STableMetaOutput *output = NULL; - ctgTestBuildCTableMetaOutput(&output); + SCtgMetaAction action = {0}; + + action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - code = ctgUpdateTableMetaCache(pCtg, &output); + output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + ctgTestBuildCTableMetaOutput(output); + + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); + msg->pCtg = pCtg; + msg->output = output; + action.data = msg; + + code = ctgActUpdateTbl(&action); if (code) { assert(0); } @@ -739,8 +809,6 @@ void *ctgTestSetCtableMetaThread(void *param) { } } - tfree(output.tbMeta); - return NULL; } @@ -751,7 +819,7 @@ TEST(tableMeta, normalTable) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -772,7 +840,11 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - ctgTestSetPrepareTableMeta(); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(10000); + } + + ctgTestSetRspTableMeta(); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -786,6 +858,16 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -834,6 +916,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(allStbNum, 0); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, childTableCase) { @@ -843,7 +926,7 @@ TEST(tableMeta, childTableCase) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -870,6 +953,16 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -932,6 +1025,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, superTableCase) { @@ -939,7 +1033,7 @@ TEST(tableMeta, superTableCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -968,7 +1062,17 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - ctgTestSetPrepareCTableMeta(); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + + ctgTestSetRspCTableMeta(); tableMeta = NULL; @@ -985,8 +1089,18 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (2 != n) { + usleep(10000); + } else { + break; + } + } + + tableMeta = NULL; - code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); @@ -1034,6 +1148,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, rmStbMeta) { @@ -1043,7 +1158,7 @@ TEST(tableMeta, rmStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1072,9 +1187,30 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); ASSERT_EQ(code, 0); + while (true) { + int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + if (n || m) { + usleep(10000); + } else { + break; + } + } + + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); @@ -1082,6 +1218,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, updateStbMeta) { @@ -1091,7 +1228,7 @@ TEST(tableMeta, updateStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1120,6 +1257,16 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } + } + + tfree(tableMeta); STableMetaRsp rsp = {0}; @@ -1128,6 +1275,16 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(10000); + } else { + break; + } + } + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); @@ -1150,6 +1307,7 @@ TEST(tableMeta, updateStbMeta) { tfree(tableMeta); catalogDestroy(); + memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } TEST(tableDistVgroup, normalTable) { @@ -1158,7 +1316,15 @@ TEST(tableDistVgroup, normalTable) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1174,7 +1340,7 @@ TEST(tableDistVgroup, normalTable) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1182,6 +1348,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, childTableCase) { @@ -1190,7 +1357,16 @@ TEST(tableDistVgroup, childTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1206,7 +1382,7 @@ TEST(tableDistVgroup, childTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1214,6 +1390,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 4); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, superTableCase) { @@ -1222,7 +1399,18 @@ TEST(tableDistVgroup, superTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); @@ -1237,7 +1425,7 @@ TEST(tableDistVgroup, superTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1251,6 +1439,7 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(dbVgroup, getSetDbVgroupCase) { @@ -1258,12 +1447,19 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; SArray *vgList = NULL; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + + + ctgTestSetRspByIdx(); + initQueryModuleMsgHandle(); @@ -1279,16 +1475,21 @@ TEST(dbVgroup, getSetDbVgroupCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(10000); + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1297,15 +1498,26 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(10000); + } else { + break; + } + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); ASSERT_EQ(vgInfo.epset.numOfEps, 2); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1314,6 +1526,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmSameDbVgroup) { @@ -1321,13 +1534,13 @@ TEST(multiThread, getSetRmSameDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1365,6 +1578,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmDiffDbVgroup) { @@ -1372,13 +1586,13 @@ TEST(multiThread, getSetRmDiffDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1416,6 +1630,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, ctableMeta) { @@ -1423,13 +1638,13 @@ TEST(multiThread, ctableMeta) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1466,6 +1681,7 @@ TEST(multiThread, ctableMeta) { sleep(2); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(rentTest, allRent) { @@ -1473,7 +1689,7 @@ TEST(rentTest, allRent) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; @@ -1482,7 +1698,7 @@ TEST(rentTest, allRent) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); + ctgTestSetRspDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); @@ -1512,6 +1728,10 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + usleep(10000); + } + code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); printf("%d - expired dbNum:%d\n", i, num); @@ -1538,6 +1758,7 @@ TEST(rentTest, allRent) { } catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } int main(int argc, char **argv) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 2138e58ccc..78d2feaa83 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -47,10 +47,11 @@ int32_t fmGetHandle(FuncMgtHandle* pHandle) { int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { SFuncMgtService* pService = (SFuncMgtService*)handle; - pFuncId = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); - if (NULL == pFuncId) { + void* pVal = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); + if (NULL == pVal) { return TSDB_CODE_FAILED; } + *pFuncId = *(int32_t*)pVal; if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index 82315a5ba4..5b97a0e0c6 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -55,6 +55,7 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToke SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 763f4ee38b..eef04b17c3 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -238,6 +238,7 @@ joined_table(A) ::= %type join_type { EJoinType } %destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= . { PARSER_TRACE; A = JOIN_TYPE_INNER; } join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } /************************************************ query_specification *************************************************/ @@ -315,7 +316,12 @@ fill_mode(A) ::= NEXT. %type group_by_clause_opt { SNodeList* } %destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } +group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { PARSER_TRACE; A = B; } + +%type group_by_list { SNodeList* } +%destructor group_by_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); } +group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); } having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index bfcebfd8b8..ce2903f6aa 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -242,6 +242,9 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order CHECK_OUT_OF_MEM(orderByExpr); orderByExpr->pExpr = pExpr; orderByExpr->order = order; + if (NULL_ORDER_DEFAULT == nullOrder) { + nullOrder = (ORDER_ASC == order ? NULL_ORDER_FIRST : NULL_ORDER_LAST); + } orderByExpr->nullOrder = nullOrder; return (SNode*)orderByExpr; } @@ -279,6 +282,15 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { return (SNode*)fill; } +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { + SGroupingSetNode* groupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET); + CHECK_OUT_OF_MEM(groupingSet); + groupingSet->groupingSetType = GP_TYPE_NORMAL; + groupingSet->pParameterList = nodesMakeList(); + nodesListAppend(groupingSet->pParameterList, pNode); + return (SNode*)groupingSet; +} + SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); return pNode; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index a56a6524fc..ffc9f4a3f6 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3647,7 +3647,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) //TODO remove it int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; - int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); + int32_t code = catalogGetTableDistVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6283b56754..b4125b0a24 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -50,7 +50,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); - int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + int32_t code = catalogGetDBVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); if (code != TSDB_CODE_SUCCESS) { terrno = code; return code; diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index 4d03de14d4..b7a82e5d97 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -109,21 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 124 +#define YYNOCODE 125 #define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - EOrder yy10; - EFillMode yy14; - SNode* yy168; - ENullOrder yy177; - SNodeList* yy192; - bool yy209; - EOperatorType yy228; - EJoinType yy229; - SToken yy241; + EOperatorType yy40; + EFillMode yy44; + SToken yy79; + ENullOrder yy107; + EJoinType yy162; + SNodeList* yy174; + EOrder yy188; + SNode* yy212; + bool yy237; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -138,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 144 -#define YYNRULE 135 +#define YYNSTATE 147 +#define YYNRULE 138 #define YYNTOKEN 72 -#define YY_MAX_SHIFT 143 -#define YY_MIN_SHIFTREDUCE 240 -#define YY_MAX_SHIFTREDUCE 374 -#define YY_ERROR_ACTION 375 -#define YY_ACCEPT_ACTION 376 -#define YY_NO_ACTION 377 -#define YY_MIN_REDUCE 378 -#define YY_MAX_REDUCE 512 +#define YY_MAX_SHIFT 146 +#define YY_MIN_SHIFTREDUCE 243 +#define YY_MAX_SHIFTREDUCE 380 +#define YY_ERROR_ACTION 381 +#define YY_ACCEPT_ACTION 382 +#define YY_NO_ACTION 383 +#define YY_MIN_REDUCE 384 +#define YY_MAX_REDUCE 521 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -215,200 +215,208 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (692) +#define YY_ACTTAB_COUNT (737) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 387, 385, 89, 22, 64, 388, 385, 462, 29, 27, - /* 10 */ 25, 24, 23, 395, 385, 138, 410, 138, 410, 139, - /* 20 */ 71, 109, 78, 396, 262, 399, 21, 84, 93, 143, - /* 30 */ 280, 281, 282, 283, 284, 285, 286, 288, 289, 290, - /* 40 */ 29, 27, 25, 24, 23, 395, 385, 138, 410, 138, - /* 50 */ 410, 139, 318, 113, 81, 396, 97, 399, 21, 84, - /* 60 */ 93, 51, 280, 281, 282, 283, 284, 285, 286, 288, - /* 70 */ 289, 290, 128, 395, 385, 114, 447, 138, 410, 139, - /* 80 */ 125, 9, 38, 396, 51, 399, 435, 255, 28, 26, - /* 90 */ 83, 431, 491, 444, 317, 242, 243, 244, 245, 140, - /* 100 */ 248, 491, 98, 1, 343, 490, 67, 10, 106, 489, - /* 110 */ 326, 395, 385, 60, 50, 138, 410, 139, 489, 123, - /* 120 */ 39, 396, 253, 399, 435, 51, 125, 9, 91, 431, - /* 130 */ 105, 341, 342, 344, 345, 395, 385, 73, 466, 138, - /* 140 */ 410, 139, 4, 314, 39, 396, 411, 399, 435, 51, - /* 150 */ 447, 447, 91, 431, 242, 243, 244, 245, 140, 248, - /* 160 */ 395, 385, 487, 8, 138, 410, 139, 443, 442, 39, - /* 170 */ 396, 40, 399, 435, 340, 18, 107, 91, 431, 7, - /* 180 */ 6, 29, 27, 25, 24, 23, 135, 451, 128, 395, - /* 190 */ 385, 7, 6, 138, 410, 139, 28, 26, 77, 396, - /* 200 */ 473, 399, 298, 242, 243, 244, 245, 140, 248, 104, - /* 210 */ 98, 1, 28, 26, 392, 10, 390, 491, 52, 242, - /* 220 */ 243, 244, 245, 140, 248, 248, 98, 5, 102, 54, - /* 230 */ 50, 136, 57, 19, 489, 395, 385, 131, 103, 138, - /* 240 */ 410, 139, 472, 287, 39, 396, 291, 399, 435, 51, - /* 250 */ 373, 374, 434, 431, 25, 24, 23, 29, 27, 25, - /* 260 */ 24, 23, 30, 395, 385, 292, 56, 138, 410, 139, - /* 270 */ 90, 256, 39, 396, 132, 399, 435, 28, 26, 319, - /* 280 */ 118, 431, 3, 453, 242, 243, 244, 245, 140, 248, - /* 290 */ 70, 98, 5, 117, 395, 385, 130, 128, 126, 410, - /* 300 */ 139, 122, 43, 44, 396, 20, 399, 119, 59, 277, - /* 310 */ 41, 29, 27, 25, 24, 23, 101, 133, 47, 65, - /* 320 */ 440, 121, 30, 120, 69, 259, 491, 115, 110, 108, - /* 330 */ 99, 459, 29, 27, 25, 24, 23, 2, 314, 50, - /* 340 */ 28, 26, 127, 489, 61, 16, 417, 242, 243, 244, - /* 350 */ 245, 140, 248, 42, 98, 5, 28, 26, 29, 27, - /* 360 */ 25, 24, 23, 242, 243, 244, 245, 140, 248, 252, - /* 370 */ 98, 1, 395, 385, 262, 255, 138, 410, 139, 31, - /* 380 */ 448, 39, 396, 256, 399, 435, 28, 26, 62, 463, - /* 390 */ 432, 370, 371, 242, 243, 244, 245, 140, 248, 94, - /* 400 */ 98, 5, 506, 137, 488, 395, 385, 253, 134, 138, - /* 410 */ 410, 139, 13, 72, 45, 396, 30, 399, 395, 385, - /* 420 */ 14, 53, 126, 410, 139, 337, 55, 44, 396, 34, - /* 430 */ 399, 395, 385, 339, 46, 138, 410, 139, 58, 35, - /* 440 */ 81, 396, 100, 399, 111, 376, 141, 333, 332, 112, - /* 450 */ 390, 129, 504, 395, 385, 460, 6, 138, 410, 139, - /* 460 */ 36, 15, 81, 396, 116, 399, 312, 278, 395, 385, - /* 470 */ 311, 33, 138, 410, 139, 491, 66, 81, 396, 92, - /* 480 */ 399, 395, 385, 32, 389, 138, 410, 139, 50, 49, - /* 490 */ 45, 396, 489, 399, 395, 385, 260, 364, 138, 410, - /* 500 */ 139, 17, 11, 75, 396, 37, 399, 395, 385, 359, - /* 510 */ 358, 138, 410, 139, 95, 363, 79, 396, 362, 399, - /* 520 */ 29, 27, 25, 24, 23, 96, 395, 385, 505, 12, - /* 530 */ 138, 410, 139, 74, 379, 76, 396, 246, 399, 395, - /* 540 */ 385, 378, 142, 138, 410, 139, 377, 377, 80, 396, - /* 550 */ 377, 399, 395, 385, 377, 377, 138, 410, 139, 377, - /* 560 */ 377, 407, 396, 377, 399, 395, 385, 377, 377, 138, - /* 570 */ 410, 139, 377, 377, 406, 396, 377, 399, 395, 385, - /* 580 */ 377, 377, 138, 410, 139, 377, 377, 405, 396, 377, - /* 590 */ 399, 395, 385, 377, 377, 138, 410, 139, 377, 377, - /* 600 */ 87, 396, 377, 399, 377, 395, 385, 377, 377, 138, - /* 610 */ 410, 139, 377, 377, 86, 396, 377, 399, 395, 385, - /* 620 */ 377, 377, 138, 410, 139, 377, 377, 88, 396, 377, - /* 630 */ 399, 395, 385, 377, 377, 138, 410, 139, 377, 377, - /* 640 */ 85, 396, 377, 399, 395, 385, 377, 377, 138, 410, - /* 650 */ 139, 377, 377, 82, 396, 377, 399, 122, 43, 377, - /* 660 */ 377, 377, 122, 43, 377, 377, 41, 377, 377, 122, - /* 670 */ 43, 41, 377, 377, 124, 63, 440, 441, 41, 445, - /* 680 */ 48, 440, 441, 377, 445, 377, 377, 68, 440, 441, - /* 690 */ 377, 445, + /* 0 */ 393, 391, 92, 23, 66, 394, 391, 469, 30, 28, + /* 10 */ 26, 25, 24, 401, 391, 141, 416, 141, 416, 142, + /* 20 */ 72, 112, 46, 402, 265, 405, 22, 87, 96, 117, + /* 30 */ 283, 284, 285, 286, 287, 288, 289, 291, 292, 293, + /* 40 */ 30, 28, 26, 25, 24, 401, 391, 141, 416, 141, + /* 50 */ 416, 142, 146, 116, 46, 402, 54, 405, 22, 87, + /* 60 */ 96, 514, 283, 284, 285, 286, 287, 288, 289, 291, + /* 70 */ 292, 293, 131, 401, 391, 128, 9, 141, 416, 142, + /* 80 */ 4, 318, 39, 402, 54, 405, 441, 109, 29, 27, + /* 90 */ 86, 437, 132, 513, 63, 245, 246, 247, 248, 143, + /* 100 */ 251, 500, 101, 1, 118, 113, 111, 10, 245, 246, + /* 110 */ 247, 248, 143, 251, 500, 53, 453, 401, 391, 498, + /* 120 */ 258, 141, 416, 142, 126, 54, 40, 402, 499, 405, + /* 130 */ 441, 453, 498, 450, 94, 437, 453, 401, 391, 69, + /* 140 */ 330, 141, 416, 142, 473, 417, 40, 402, 449, 405, + /* 150 */ 441, 134, 256, 448, 94, 437, 41, 401, 391, 344, + /* 160 */ 8, 141, 416, 142, 496, 74, 40, 402, 347, 405, + /* 170 */ 441, 458, 110, 318, 94, 437, 131, 401, 391, 107, + /* 180 */ 322, 141, 416, 142, 457, 57, 78, 402, 60, 405, + /* 190 */ 128, 9, 29, 27, 108, 345, 346, 348, 349, 245, + /* 200 */ 246, 247, 248, 143, 251, 500, 101, 1, 19, 31, + /* 210 */ 133, 10, 295, 54, 30, 28, 26, 25, 24, 53, + /* 220 */ 29, 27, 321, 498, 26, 25, 24, 245, 246, 247, + /* 230 */ 248, 143, 251, 480, 101, 5, 401, 391, 7, 6, + /* 240 */ 141, 416, 142, 7, 6, 40, 402, 251, 405, 441, + /* 250 */ 122, 138, 280, 440, 437, 55, 398, 54, 396, 401, + /* 260 */ 391, 301, 105, 141, 416, 142, 379, 380, 40, 402, + /* 270 */ 106, 405, 441, 29, 27, 323, 121, 437, 479, 59, + /* 280 */ 245, 246, 247, 248, 143, 251, 93, 101, 5, 30, + /* 290 */ 28, 26, 25, 24, 401, 391, 139, 131, 129, 416, + /* 300 */ 142, 125, 44, 45, 402, 265, 405, 3, 460, 135, + /* 310 */ 42, 120, 20, 30, 28, 26, 25, 24, 62, 67, + /* 320 */ 446, 124, 290, 123, 70, 294, 500, 259, 104, 21, + /* 330 */ 102, 466, 48, 64, 2, 30, 28, 26, 25, 24, + /* 340 */ 53, 16, 31, 423, 498, 262, 71, 318, 255, 29, + /* 350 */ 27, 130, 136, 43, 258, 454, 245, 246, 247, 248, + /* 360 */ 143, 251, 32, 101, 5, 29, 27, 30, 28, 26, + /* 370 */ 25, 24, 245, 246, 247, 248, 143, 251, 65, 101, + /* 380 */ 1, 401, 391, 259, 470, 141, 416, 142, 97, 140, + /* 390 */ 40, 402, 515, 405, 441, 29, 27, 497, 137, 438, + /* 400 */ 73, 256, 245, 246, 247, 248, 143, 251, 56, 101, + /* 410 */ 5, 13, 30, 28, 26, 25, 24, 31, 14, 401, + /* 420 */ 391, 341, 58, 141, 416, 142, 376, 377, 84, 402, + /* 430 */ 100, 405, 401, 391, 35, 343, 129, 416, 142, 47, + /* 440 */ 61, 45, 402, 337, 405, 36, 114, 401, 391, 382, + /* 450 */ 144, 141, 416, 142, 336, 37, 84, 402, 103, 405, + /* 460 */ 115, 396, 401, 391, 18, 315, 141, 416, 142, 467, + /* 470 */ 6, 80, 402, 15, 405, 281, 314, 401, 391, 500, + /* 480 */ 68, 141, 416, 142, 33, 34, 84, 402, 95, 405, + /* 490 */ 395, 52, 17, 53, 263, 401, 391, 498, 370, 141, + /* 500 */ 416, 142, 11, 119, 79, 402, 38, 405, 401, 391, + /* 510 */ 365, 75, 141, 416, 142, 364, 98, 81, 402, 369, + /* 520 */ 405, 368, 401, 391, 385, 99, 141, 416, 142, 249, + /* 530 */ 12, 76, 402, 384, 405, 145, 383, 401, 391, 383, + /* 540 */ 383, 141, 416, 142, 383, 383, 82, 402, 383, 405, + /* 550 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 560 */ 383, 77, 402, 383, 405, 383, 383, 383, 383, 383, + /* 570 */ 401, 391, 383, 383, 141, 416, 142, 383, 383, 83, + /* 580 */ 402, 383, 405, 401, 391, 383, 383, 141, 416, 142, + /* 590 */ 383, 383, 413, 402, 383, 405, 383, 401, 391, 383, + /* 600 */ 383, 141, 416, 142, 383, 383, 412, 402, 383, 405, + /* 610 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 620 */ 383, 411, 402, 383, 405, 383, 383, 401, 391, 383, + /* 630 */ 383, 141, 416, 142, 383, 383, 90, 402, 383, 405, + /* 640 */ 383, 383, 383, 383, 383, 401, 391, 383, 383, 141, + /* 650 */ 416, 142, 383, 383, 89, 402, 383, 405, 401, 391, + /* 660 */ 383, 383, 141, 416, 142, 383, 383, 91, 402, 383, + /* 670 */ 405, 383, 401, 391, 383, 383, 141, 416, 142, 383, + /* 680 */ 383, 88, 402, 383, 405, 125, 44, 401, 391, 383, + /* 690 */ 383, 141, 416, 142, 42, 383, 85, 402, 383, 405, + /* 700 */ 125, 44, 127, 49, 446, 447, 383, 451, 383, 42, + /* 710 */ 383, 383, 383, 383, 125, 44, 383, 383, 50, 446, + /* 720 */ 447, 383, 451, 42, 383, 383, 383, 383, 383, 383, + /* 730 */ 383, 383, 51, 446, 447, 383, 451, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 74, 75, 76, 88, 89, 74, 75, 82, 8, 9, /* 10 */ 10, 11, 12, 74, 75, 78, 79, 78, 79, 80, - /* 20 */ 122, 84, 83, 84, 24, 86, 26, 27, 28, 13, + /* 20 */ 123, 84, 83, 84, 24, 86, 26, 27, 28, 22, /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, /* 40 */ 8, 9, 10, 11, 12, 74, 75, 78, 79, 78, - /* 50 */ 79, 80, 4, 84, 83, 84, 85, 86, 26, 27, - /* 60 */ 28, 45, 30, 31, 32, 33, 34, 35, 36, 37, - /* 70 */ 38, 39, 73, 74, 75, 22, 81, 78, 79, 80, - /* 80 */ 22, 23, 83, 84, 45, 86, 87, 22, 8, 9, - /* 90 */ 91, 92, 102, 98, 46, 15, 16, 17, 18, 19, - /* 100 */ 20, 102, 22, 23, 29, 115, 41, 27, 114, 119, - /* 110 */ 10, 74, 75, 107, 115, 78, 79, 80, 119, 100, - /* 120 */ 83, 84, 22, 86, 87, 45, 22, 23, 91, 92, - /* 130 */ 55, 56, 57, 58, 59, 74, 75, 116, 101, 78, - /* 140 */ 79, 80, 43, 44, 83, 84, 79, 86, 87, 45, - /* 150 */ 81, 81, 91, 92, 15, 16, 17, 18, 19, 20, - /* 160 */ 74, 75, 101, 103, 78, 79, 80, 98, 98, 83, - /* 170 */ 84, 21, 86, 87, 24, 2, 54, 91, 92, 1, - /* 180 */ 2, 8, 9, 10, 11, 12, 21, 101, 73, 74, - /* 190 */ 75, 1, 2, 78, 79, 80, 8, 9, 83, 84, - /* 200 */ 113, 86, 24, 15, 16, 17, 18, 19, 20, 53, - /* 210 */ 22, 23, 8, 9, 23, 27, 25, 102, 112, 15, - /* 220 */ 16, 17, 18, 19, 20, 20, 22, 23, 75, 21, - /* 230 */ 115, 66, 24, 26, 119, 74, 75, 3, 75, 78, - /* 240 */ 79, 80, 113, 36, 83, 84, 39, 86, 87, 45, - /* 250 */ 70, 71, 91, 92, 10, 11, 12, 8, 9, 10, - /* 260 */ 11, 12, 21, 74, 75, 24, 112, 78, 79, 80, - /* 270 */ 75, 22, 83, 84, 21, 86, 87, 8, 9, 10, - /* 280 */ 91, 92, 61, 109, 15, 16, 17, 18, 19, 20, - /* 290 */ 41, 22, 23, 60, 74, 75, 62, 73, 78, 79, - /* 300 */ 80, 77, 78, 83, 84, 2, 86, 27, 108, 29, - /* 310 */ 86, 8, 9, 10, 11, 12, 48, 64, 106, 95, - /* 320 */ 96, 97, 21, 99, 104, 24, 102, 50, 51, 52, - /* 330 */ 110, 111, 8, 9, 10, 11, 12, 47, 44, 115, - /* 340 */ 8, 9, 10, 119, 105, 23, 90, 15, 16, 17, - /* 350 */ 18, 19, 20, 78, 22, 23, 8, 9, 8, 9, - /* 360 */ 10, 11, 12, 15, 16, 17, 18, 19, 20, 22, - /* 370 */ 22, 23, 74, 75, 24, 22, 78, 79, 80, 40, - /* 380 */ 81, 83, 84, 22, 86, 87, 8, 9, 93, 82, - /* 390 */ 92, 67, 68, 15, 16, 17, 18, 19, 20, 69, - /* 400 */ 22, 23, 123, 65, 118, 74, 75, 22, 63, 78, - /* 410 */ 79, 80, 21, 117, 83, 84, 21, 86, 74, 75, - /* 420 */ 49, 24, 78, 79, 80, 24, 23, 83, 84, 21, - /* 430 */ 86, 74, 75, 24, 23, 78, 79, 80, 23, 23, - /* 440 */ 83, 84, 85, 86, 15, 72, 73, 24, 24, 21, - /* 450 */ 25, 120, 121, 74, 75, 111, 2, 78, 79, 80, - /* 460 */ 23, 49, 83, 84, 85, 86, 24, 29, 74, 75, - /* 470 */ 24, 21, 78, 79, 80, 102, 25, 83, 84, 85, - /* 480 */ 86, 74, 75, 42, 25, 78, 79, 80, 115, 25, - /* 490 */ 83, 84, 119, 86, 74, 75, 24, 24, 78, 79, - /* 500 */ 80, 21, 49, 83, 84, 4, 86, 74, 75, 15, - /* 510 */ 15, 78, 79, 80, 15, 15, 83, 84, 15, 86, - /* 520 */ 8, 9, 10, 11, 12, 15, 74, 75, 121, 23, - /* 530 */ 78, 79, 80, 25, 0, 83, 84, 17, 86, 74, - /* 540 */ 75, 0, 14, 78, 79, 80, 124, 124, 83, 84, - /* 550 */ 124, 86, 74, 75, 124, 124, 78, 79, 80, 124, - /* 560 */ 124, 83, 84, 124, 86, 74, 75, 124, 124, 78, - /* 570 */ 79, 80, 124, 124, 83, 84, 124, 86, 74, 75, - /* 580 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, - /* 590 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, - /* 600 */ 83, 84, 124, 86, 124, 74, 75, 124, 124, 78, - /* 610 */ 79, 80, 124, 124, 83, 84, 124, 86, 74, 75, - /* 620 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, - /* 630 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, - /* 640 */ 83, 84, 124, 86, 74, 75, 124, 124, 78, 79, - /* 650 */ 80, 124, 124, 83, 84, 124, 86, 77, 78, 124, - /* 660 */ 124, 124, 77, 78, 124, 124, 86, 124, 124, 77, - /* 670 */ 78, 86, 124, 124, 94, 95, 96, 97, 86, 99, - /* 680 */ 95, 96, 97, 124, 99, 124, 124, 95, 96, 97, - /* 690 */ 124, 99, + /* 50 */ 79, 80, 13, 84, 83, 84, 45, 86, 26, 27, + /* 60 */ 28, 122, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 73, 74, 75, 22, 23, 78, 79, 80, + /* 80 */ 43, 44, 83, 84, 45, 86, 87, 114, 8, 9, + /* 90 */ 91, 92, 121, 122, 107, 15, 16, 17, 18, 19, + /* 100 */ 20, 102, 22, 23, 50, 51, 52, 27, 15, 16, + /* 110 */ 17, 18, 19, 20, 102, 116, 81, 74, 75, 120, + /* 120 */ 22, 78, 79, 80, 100, 45, 83, 84, 116, 86, + /* 130 */ 87, 81, 120, 98, 91, 92, 81, 74, 75, 41, + /* 140 */ 10, 78, 79, 80, 101, 79, 83, 84, 98, 86, + /* 150 */ 87, 3, 22, 98, 91, 92, 21, 74, 75, 24, + /* 160 */ 103, 78, 79, 80, 101, 117, 83, 84, 29, 86, + /* 170 */ 87, 42, 54, 44, 91, 92, 73, 74, 75, 53, + /* 180 */ 4, 78, 79, 80, 101, 21, 83, 84, 24, 86, + /* 190 */ 22, 23, 8, 9, 55, 56, 57, 58, 59, 15, + /* 200 */ 16, 17, 18, 19, 20, 102, 22, 23, 2, 21, + /* 210 */ 62, 27, 24, 45, 8, 9, 10, 11, 12, 116, + /* 220 */ 8, 9, 46, 120, 10, 11, 12, 15, 16, 17, + /* 230 */ 18, 19, 20, 113, 22, 23, 74, 75, 1, 2, + /* 240 */ 78, 79, 80, 1, 2, 83, 84, 20, 86, 87, + /* 250 */ 27, 21, 29, 91, 92, 112, 23, 45, 25, 74, + /* 260 */ 75, 24, 75, 78, 79, 80, 70, 71, 83, 84, + /* 270 */ 75, 86, 87, 8, 9, 10, 91, 92, 113, 112, + /* 280 */ 15, 16, 17, 18, 19, 20, 75, 22, 23, 8, + /* 290 */ 9, 10, 11, 12, 74, 75, 66, 73, 78, 79, + /* 300 */ 80, 77, 78, 83, 84, 24, 86, 61, 109, 21, + /* 310 */ 86, 60, 26, 8, 9, 10, 11, 12, 108, 95, + /* 320 */ 96, 97, 36, 99, 104, 39, 102, 22, 48, 2, + /* 330 */ 110, 111, 106, 105, 47, 8, 9, 10, 11, 12, + /* 340 */ 116, 23, 21, 90, 120, 24, 41, 44, 22, 8, + /* 350 */ 9, 10, 64, 78, 22, 81, 15, 16, 17, 18, + /* 360 */ 19, 20, 40, 22, 23, 8, 9, 8, 9, 10, + /* 370 */ 11, 12, 15, 16, 17, 18, 19, 20, 93, 22, + /* 380 */ 23, 74, 75, 22, 82, 78, 79, 80, 69, 65, + /* 390 */ 83, 84, 124, 86, 87, 8, 9, 119, 63, 92, + /* 400 */ 118, 22, 15, 16, 17, 18, 19, 20, 24, 22, + /* 410 */ 23, 21, 8, 9, 10, 11, 12, 21, 49, 74, + /* 420 */ 75, 24, 23, 78, 79, 80, 67, 68, 83, 84, + /* 430 */ 85, 86, 74, 75, 21, 24, 78, 79, 80, 23, + /* 440 */ 23, 83, 84, 24, 86, 23, 15, 74, 75, 72, + /* 450 */ 73, 78, 79, 80, 24, 23, 83, 84, 85, 86, + /* 460 */ 21, 25, 74, 75, 21, 24, 78, 79, 80, 111, + /* 470 */ 2, 83, 84, 49, 86, 29, 24, 74, 75, 102, + /* 480 */ 25, 78, 79, 80, 42, 21, 83, 84, 85, 86, + /* 490 */ 25, 25, 21, 116, 24, 74, 75, 120, 24, 78, + /* 500 */ 79, 80, 49, 115, 83, 84, 4, 86, 74, 75, + /* 510 */ 15, 25, 78, 79, 80, 15, 15, 83, 84, 15, + /* 520 */ 86, 15, 74, 75, 0, 15, 78, 79, 80, 17, + /* 530 */ 23, 83, 84, 0, 86, 14, 125, 74, 75, 125, + /* 540 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 550 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 560 */ 125, 83, 84, 125, 86, 125, 125, 125, 125, 125, + /* 570 */ 74, 75, 125, 125, 78, 79, 80, 125, 125, 83, + /* 580 */ 84, 125, 86, 74, 75, 125, 125, 78, 79, 80, + /* 590 */ 125, 125, 83, 84, 125, 86, 125, 74, 75, 125, + /* 600 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 610 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 620 */ 125, 83, 84, 125, 86, 125, 125, 74, 75, 125, + /* 630 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, + /* 640 */ 125, 125, 125, 125, 125, 74, 75, 125, 125, 78, + /* 650 */ 79, 80, 125, 125, 83, 84, 125, 86, 74, 75, + /* 660 */ 125, 125, 78, 79, 80, 125, 125, 83, 84, 125, + /* 670 */ 86, 125, 74, 75, 125, 125, 78, 79, 80, 125, + /* 680 */ 125, 83, 84, 125, 86, 77, 78, 74, 75, 125, + /* 690 */ 125, 78, 79, 80, 86, 125, 83, 84, 125, 86, + /* 700 */ 77, 78, 94, 95, 96, 97, 125, 99, 125, 86, + /* 710 */ 125, 125, 125, 125, 77, 78, 125, 125, 95, 96, + /* 720 */ 97, 125, 99, 86, 125, 125, 125, 125, 125, 125, + /* 730 */ 125, 125, 95, 96, 97, 125, 99, }; -#define YY_SHIFT_COUNT (143) +#define YY_SHIFT_COUNT (146) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (541) +#define YY_SHIFT_MAX (533) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 16, 80, 188, 188, 188, 204, 188, 188, 269, 104, - /* 10 */ 348, 378, 332, 378, 378, 378, 378, 378, 378, 378, - /* 20 */ 378, 378, 378, 378, 378, 378, 378, 378, 378, 378, - /* 30 */ 378, 58, 58, 58, 139, 53, 53, 39, 0, 32, - /* 40 */ 139, 65, 65, 65, 249, 324, 75, 277, 99, 100, - /* 50 */ 234, 48, 122, 156, 205, 205, 122, 156, 205, 221, - /* 60 */ 233, 268, 290, 294, 322, 294, 347, 353, 294, 339, - /* 70 */ 361, 330, 338, 345, 385, 173, 303, 350, 512, 512, - /* 80 */ 512, 512, 512, 178, 207, 244, 244, 244, 244, 150, - /* 90 */ 208, 190, 241, 280, 180, 253, 165, 301, 191, 391, - /* 100 */ 395, 371, 397, 401, 403, 408, 409, 411, 415, 423, - /* 110 */ 416, 424, 429, 428, 425, 437, 395, 412, 454, 438, - /* 120 */ 442, 446, 451, 441, 450, 459, 464, 472, 473, 480, - /* 130 */ 453, 501, 494, 495, 499, 500, 503, 510, 508, 506, - /* 140 */ 520, 534, 541, 528, + /* 0 */ 39, 80, 184, 184, 184, 212, 184, 184, 265, 168, + /* 10 */ 357, 387, 341, 387, 387, 387, 387, 387, 387, 387, + /* 20 */ 387, 387, 387, 387, 387, 387, 387, 387, 387, 387, + /* 30 */ 387, 387, 53, 53, 53, 93, 7, 7, 11, 0, + /* 40 */ 32, 93, 98, 98, 98, 305, 359, 139, 54, 129, + /* 50 */ 37, 129, 130, 148, 176, 118, 126, 227, 227, 118, + /* 60 */ 126, 227, 246, 251, 280, 287, 318, 303, 326, 332, + /* 70 */ 322, 361, 319, 324, 335, 379, 206, 327, 281, 404, + /* 80 */ 404, 404, 404, 404, 404, 404, 237, 286, 214, 214, + /* 90 */ 214, 214, 135, 164, 242, 188, 223, 196, 288, 230, + /* 100 */ 321, 233, 390, 396, 369, 384, 397, 399, 413, 411, + /* 110 */ 416, 417, 419, 422, 430, 431, 439, 436, 432, 443, + /* 120 */ 424, 468, 446, 441, 452, 455, 442, 464, 465, 466, + /* 130 */ 470, 474, 471, 453, 502, 495, 500, 501, 504, 506, + /* 140 */ 510, 486, 507, 512, 524, 533, 521, }; -#define YY_REDUCE_COUNT (74) -#define YY_REDUCE_MIN (-102) -#define YY_REDUCE_MAX (592) +#define YY_REDUCE_COUNT (75) +#define YY_REDUCE_MIN (-103) +#define YY_REDUCE_MAX (637) static const short yy_reduce_ofst[] = { - /* 0 */ 373, -1, 37, 61, 86, 115, 161, 189, 220, 224, - /* 10 */ 298, 331, -29, 344, 357, 379, 394, 407, -61, 420, - /* 20 */ 433, 452, 465, 478, 491, 504, 517, 531, 544, 557, - /* 30 */ 570, 580, 585, 592, -74, -63, -31, -10, -85, -85, - /* 40 */ -69, -5, 69, 70, -75, -102, -6, 6, 19, 67, - /* 50 */ 21, 60, 87, 106, 153, 163, 129, 154, 195, 174, - /* 60 */ 200, 212, 239, 19, 256, 19, 275, 299, 19, 295, - /* 70 */ 307, 279, 286, 296, 67, + /* 0 */ 377, -1, 43, 63, 83, 103, 162, 185, 220, 224, + /* 10 */ 307, -29, 345, 358, 373, 388, 403, -61, 421, 434, + /* 20 */ 448, 463, 478, 496, 509, 523, 538, 553, 571, 584, + /* 30 */ 598, 613, 608, 623, 637, -74, -63, -31, 12, -85, + /* 40 */ -85, -69, 35, 50, 55, -75, -103, -27, -13, 24, + /* 50 */ 24, 24, 66, 48, 57, 120, 143, 187, 195, 165, + /* 60 */ 167, 211, 199, 210, 226, 228, 253, 24, 275, 274, + /* 70 */ 285, 302, 268, 278, 282, 66, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 10 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 20 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 30 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 40 */ 375, 446, 446, 446, 461, 507, 375, 469, 375, 375, - /* 50 */ 492, 454, 476, 474, 375, 375, 476, 474, 375, 486, - /* 60 */ 484, 467, 465, 438, 375, 375, 375, 375, 439, 375, - /* 70 */ 375, 510, 498, 494, 375, 375, 375, 375, 414, 413, - /* 80 */ 412, 408, 409, 375, 375, 403, 404, 402, 401, 375, - /* 90 */ 375, 503, 375, 375, 375, 495, 499, 375, 391, 458, - /* 100 */ 468, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 110 */ 375, 375, 375, 375, 391, 375, 485, 375, 433, 375, - /* 120 */ 445, 441, 375, 375, 437, 390, 375, 375, 375, 493, - /* 130 */ 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - /* 140 */ 375, 375, 375, 375, + /* 0 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 10 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 20 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 30 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, + /* 40 */ 381, 381, 452, 452, 452, 468, 516, 381, 476, 444, + /* 50 */ 458, 445, 381, 501, 461, 483, 481, 381, 381, 483, + /* 60 */ 481, 381, 495, 491, 474, 472, 381, 458, 381, 381, + /* 70 */ 381, 381, 519, 507, 503, 381, 381, 381, 381, 494, + /* 80 */ 493, 420, 419, 418, 414, 415, 381, 381, 409, 410, + /* 90 */ 408, 407, 381, 381, 512, 381, 381, 381, 504, 508, + /* 100 */ 381, 397, 465, 475, 381, 381, 381, 381, 381, 381, + /* 110 */ 381, 381, 381, 381, 381, 381, 381, 397, 381, 492, + /* 120 */ 381, 439, 381, 451, 447, 381, 381, 443, 396, 381, + /* 130 */ 381, 381, 502, 381, 381, 381, 381, 381, 381, 381, + /* 140 */ 381, 381, 381, 381, 381, 381, 381, }; /********** End of lemon-generated parsing tables *****************************/ @@ -630,15 +638,16 @@ static const char *const yyTokenName[] = { /* 112 */ "sliding_opt", /* 113 */ "fill_opt", /* 114 */ "fill_mode", - /* 115 */ "query_expression_body", - /* 116 */ "order_by_clause_opt", - /* 117 */ "slimit_clause_opt", - /* 118 */ "limit_clause_opt", - /* 119 */ "query_primary", - /* 120 */ "sort_specification_list", - /* 121 */ "sort_specification", - /* 122 */ "ordering_specification_opt", - /* 123 */ "null_ordering_opt", + /* 115 */ "group_by_list", + /* 116 */ "query_expression_body", + /* 117 */ "order_by_clause_opt", + /* 118 */ "slimit_clause_opt", + /* 119 */ "limit_clause_opt", + /* 120 */ "query_primary", + /* 121 */ "sort_specification_list", + /* 122 */ "sort_specification", + /* 123 */ "ordering_specification_opt", + /* 124 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -720,67 +729,70 @@ static const char *const yyRuleName[] = { /* 71 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", /* 72 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", /* 73 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 74 */ "join_type ::= INNER", - /* 75 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 76 */ "set_quantifier_opt ::=", - /* 77 */ "set_quantifier_opt ::= DISTINCT", - /* 78 */ "set_quantifier_opt ::= ALL", - /* 79 */ "select_list ::= NK_STAR", - /* 80 */ "select_list ::= select_sublist", - /* 81 */ "select_sublist ::= select_item", - /* 82 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 83 */ "select_item ::= expression", - /* 84 */ "select_item ::= expression column_alias", - /* 85 */ "select_item ::= expression AS column_alias", - /* 86 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 87 */ "where_clause_opt ::=", - /* 88 */ "where_clause_opt ::= WHERE search_condition", - /* 89 */ "partition_by_clause_opt ::=", - /* 90 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 91 */ "twindow_clause_opt ::=", - /* 92 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 93 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 94 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 95 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 96 */ "sliding_opt ::=", - /* 97 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 98 */ "fill_opt ::=", - /* 99 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 100 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 101 */ "fill_mode ::= NONE", - /* 102 */ "fill_mode ::= PREV", - /* 103 */ "fill_mode ::= NULL", - /* 104 */ "fill_mode ::= LINEAR", - /* 105 */ "fill_mode ::= NEXT", - /* 106 */ "group_by_clause_opt ::=", - /* 107 */ "group_by_clause_opt ::= GROUP BY expression_list", - /* 108 */ "having_clause_opt ::=", - /* 109 */ "having_clause_opt ::= HAVING search_condition", - /* 110 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 111 */ "query_expression_body ::= query_primary", - /* 112 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 113 */ "query_primary ::= query_specification", - /* 114 */ "order_by_clause_opt ::=", - /* 115 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 116 */ "slimit_clause_opt ::=", - /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 119 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 120 */ "limit_clause_opt ::=", - /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 123 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 124 */ "subquery ::= NK_LP query_expression NK_RP", - /* 125 */ "search_condition ::= boolean_value_expression", - /* 126 */ "sort_specification_list ::= sort_specification", - /* 127 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 128 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 129 */ "ordering_specification_opt ::=", - /* 130 */ "ordering_specification_opt ::= ASC", - /* 131 */ "ordering_specification_opt ::= DESC", - /* 132 */ "null_ordering_opt ::=", - /* 133 */ "null_ordering_opt ::= NULLS FIRST", - /* 134 */ "null_ordering_opt ::= NULLS LAST", + /* 74 */ "join_type ::=", + /* 75 */ "join_type ::= INNER", + /* 76 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 77 */ "set_quantifier_opt ::=", + /* 78 */ "set_quantifier_opt ::= DISTINCT", + /* 79 */ "set_quantifier_opt ::= ALL", + /* 80 */ "select_list ::= NK_STAR", + /* 81 */ "select_list ::= select_sublist", + /* 82 */ "select_sublist ::= select_item", + /* 83 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 84 */ "select_item ::= expression", + /* 85 */ "select_item ::= expression column_alias", + /* 86 */ "select_item ::= expression AS column_alias", + /* 87 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 88 */ "where_clause_opt ::=", + /* 89 */ "where_clause_opt ::= WHERE search_condition", + /* 90 */ "partition_by_clause_opt ::=", + /* 91 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 92 */ "twindow_clause_opt ::=", + /* 93 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 94 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 95 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 96 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 97 */ "sliding_opt ::=", + /* 98 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 99 */ "fill_opt ::=", + /* 100 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 101 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 102 */ "fill_mode ::= NONE", + /* 103 */ "fill_mode ::= PREV", + /* 104 */ "fill_mode ::= NULL", + /* 105 */ "fill_mode ::= LINEAR", + /* 106 */ "fill_mode ::= NEXT", + /* 107 */ "group_by_clause_opt ::=", + /* 108 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 109 */ "group_by_list ::= expression", + /* 110 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 111 */ "having_clause_opt ::=", + /* 112 */ "having_clause_opt ::= HAVING search_condition", + /* 113 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 114 */ "query_expression_body ::= query_primary", + /* 115 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 116 */ "query_primary ::= query_specification", + /* 117 */ "order_by_clause_opt ::=", + /* 118 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 119 */ "slimit_clause_opt ::=", + /* 120 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 121 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 122 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "limit_clause_opt ::=", + /* 124 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 125 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 126 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 127 */ "subquery ::= NK_LP query_expression NK_RP", + /* 128 */ "search_condition ::= boolean_value_expression", + /* 129 */ "sort_specification_list ::= sort_specification", + /* 130 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 131 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 132 */ "ordering_specification_opt ::=", + /* 133 */ "ordering_specification_opt ::= ASC", + /* 134 */ "ordering_specification_opt ::= DESC", + /* 135 */ "null_ordering_opt ::=", + /* 136 */ "null_ordering_opt ::= NULLS FIRST", + /* 137 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -932,13 +944,13 @@ static void yy_destructor( case 111: /* select_item */ case 112: /* sliding_opt */ case 113: /* fill_opt */ - case 115: /* query_expression_body */ - case 117: /* slimit_clause_opt */ - case 118: /* limit_clause_opt */ - case 119: /* query_primary */ - case 121: /* sort_specification */ + case 116: /* query_expression_body */ + case 118: /* slimit_clause_opt */ + case 119: /* limit_clause_opt */ + case 120: /* query_primary */ + case 122: /* sort_specification */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy212)); } break; case 76: /* literal_list */ @@ -947,10 +959,11 @@ static void yy_destructor( case 106: /* partition_by_clause_opt */ case 108: /* group_by_clause_opt */ case 110: /* select_sublist */ - case 116: /* order_by_clause_opt */ - case 120: /* sort_specification_list */ + case 115: /* group_by_list */ + case 117: /* order_by_clause_opt */ + case 121: /* sort_specification_list */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy192)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy174)); } break; case 77: /* db_name */ @@ -985,12 +998,12 @@ static void yy_destructor( PARSER_DESTRUCTOR_TRACE; } break; - case 122: /* ordering_specification_opt */ + case 123: /* ordering_specification_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 123: /* null_ordering_opt */ + case 124: /* null_ordering_opt */ { PARSER_DESTRUCTOR_TRACE; } @@ -1363,67 +1376,70 @@ static const struct { { 99, -3 }, /* (71) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ { 99, -3 }, /* (72) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ { 97, -6 }, /* (73) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 100, -1 }, /* (74) join_type ::= INNER */ - { 102, -9 }, /* (75) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 103, 0 }, /* (76) set_quantifier_opt ::= */ - { 103, -1 }, /* (77) set_quantifier_opt ::= DISTINCT */ - { 103, -1 }, /* (78) set_quantifier_opt ::= ALL */ - { 104, -1 }, /* (79) select_list ::= NK_STAR */ - { 104, -1 }, /* (80) select_list ::= select_sublist */ - { 110, -1 }, /* (81) select_sublist ::= select_item */ - { 110, -3 }, /* (82) select_sublist ::= select_sublist NK_COMMA select_item */ - { 111, -1 }, /* (83) select_item ::= expression */ - { 111, -2 }, /* (84) select_item ::= expression column_alias */ - { 111, -3 }, /* (85) select_item ::= expression AS column_alias */ - { 111, -3 }, /* (86) select_item ::= table_name NK_DOT NK_STAR */ - { 105, 0 }, /* (87) where_clause_opt ::= */ - { 105, -2 }, /* (88) where_clause_opt ::= WHERE search_condition */ - { 106, 0 }, /* (89) partition_by_clause_opt ::= */ - { 106, -3 }, /* (90) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 107, 0 }, /* (91) twindow_clause_opt ::= */ - { 107, -6 }, /* (92) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 107, -4 }, /* (93) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 107, -6 }, /* (94) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 107, -8 }, /* (95) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 112, 0 }, /* (96) sliding_opt ::= */ - { 112, -4 }, /* (97) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 113, 0 }, /* (98) fill_opt ::= */ - { 113, -4 }, /* (99) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 113, -6 }, /* (100) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 114, -1 }, /* (101) fill_mode ::= NONE */ - { 114, -1 }, /* (102) fill_mode ::= PREV */ - { 114, -1 }, /* (103) fill_mode ::= NULL */ - { 114, -1 }, /* (104) fill_mode ::= LINEAR */ - { 114, -1 }, /* (105) fill_mode ::= NEXT */ - { 108, 0 }, /* (106) group_by_clause_opt ::= */ - { 108, -3 }, /* (107) group_by_clause_opt ::= GROUP BY expression_list */ - { 109, 0 }, /* (108) having_clause_opt ::= */ - { 109, -2 }, /* (109) having_clause_opt ::= HAVING search_condition */ - { 73, -4 }, /* (110) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 115, -1 }, /* (111) query_expression_body ::= query_primary */ - { 115, -4 }, /* (112) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 119, -1 }, /* (113) query_primary ::= query_specification */ - { 116, 0 }, /* (114) order_by_clause_opt ::= */ - { 116, -3 }, /* (115) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 117, 0 }, /* (116) slimit_clause_opt ::= */ - { 117, -2 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 117, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 117, -4 }, /* (119) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 118, 0 }, /* (120) limit_clause_opt ::= */ - { 118, -2 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 118, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 118, -4 }, /* (123) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 86, -3 }, /* (124) subquery ::= NK_LP query_expression NK_RP */ - { 101, -1 }, /* (125) search_condition ::= boolean_value_expression */ - { 120, -1 }, /* (126) sort_specification_list ::= sort_specification */ - { 120, -3 }, /* (127) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 121, -3 }, /* (128) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 122, 0 }, /* (129) ordering_specification_opt ::= */ - { 122, -1 }, /* (130) ordering_specification_opt ::= ASC */ - { 122, -1 }, /* (131) ordering_specification_opt ::= DESC */ - { 123, 0 }, /* (132) null_ordering_opt ::= */ - { 123, -2 }, /* (133) null_ordering_opt ::= NULLS FIRST */ - { 123, -2 }, /* (134) null_ordering_opt ::= NULLS LAST */ + { 100, 0 }, /* (74) join_type ::= */ + { 100, -1 }, /* (75) join_type ::= INNER */ + { 102, -9 }, /* (76) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 103, 0 }, /* (77) set_quantifier_opt ::= */ + { 103, -1 }, /* (78) set_quantifier_opt ::= DISTINCT */ + { 103, -1 }, /* (79) set_quantifier_opt ::= ALL */ + { 104, -1 }, /* (80) select_list ::= NK_STAR */ + { 104, -1 }, /* (81) select_list ::= select_sublist */ + { 110, -1 }, /* (82) select_sublist ::= select_item */ + { 110, -3 }, /* (83) select_sublist ::= select_sublist NK_COMMA select_item */ + { 111, -1 }, /* (84) select_item ::= expression */ + { 111, -2 }, /* (85) select_item ::= expression column_alias */ + { 111, -3 }, /* (86) select_item ::= expression AS column_alias */ + { 111, -3 }, /* (87) select_item ::= table_name NK_DOT NK_STAR */ + { 105, 0 }, /* (88) where_clause_opt ::= */ + { 105, -2 }, /* (89) where_clause_opt ::= WHERE search_condition */ + { 106, 0 }, /* (90) partition_by_clause_opt ::= */ + { 106, -3 }, /* (91) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 107, 0 }, /* (92) twindow_clause_opt ::= */ + { 107, -6 }, /* (93) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 107, -4 }, /* (94) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 107, -6 }, /* (95) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 107, -8 }, /* (96) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 112, 0 }, /* (97) sliding_opt ::= */ + { 112, -4 }, /* (98) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 113, 0 }, /* (99) fill_opt ::= */ + { 113, -4 }, /* (100) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 113, -6 }, /* (101) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 114, -1 }, /* (102) fill_mode ::= NONE */ + { 114, -1 }, /* (103) fill_mode ::= PREV */ + { 114, -1 }, /* (104) fill_mode ::= NULL */ + { 114, -1 }, /* (105) fill_mode ::= LINEAR */ + { 114, -1 }, /* (106) fill_mode ::= NEXT */ + { 108, 0 }, /* (107) group_by_clause_opt ::= */ + { 108, -3 }, /* (108) group_by_clause_opt ::= GROUP BY group_by_list */ + { 115, -1 }, /* (109) group_by_list ::= expression */ + { 115, -3 }, /* (110) group_by_list ::= group_by_list NK_COMMA expression */ + { 109, 0 }, /* (111) having_clause_opt ::= */ + { 109, -2 }, /* (112) having_clause_opt ::= HAVING search_condition */ + { 73, -4 }, /* (113) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 116, -1 }, /* (114) query_expression_body ::= query_primary */ + { 116, -4 }, /* (115) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 120, -1 }, /* (116) query_primary ::= query_specification */ + { 117, 0 }, /* (117) order_by_clause_opt ::= */ + { 117, -3 }, /* (118) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 118, 0 }, /* (119) slimit_clause_opt ::= */ + { 118, -2 }, /* (120) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 118, -4 }, /* (121) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 118, -4 }, /* (122) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 119, 0 }, /* (123) limit_clause_opt ::= */ + { 119, -2 }, /* (124) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 119, -4 }, /* (125) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 119, -4 }, /* (126) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 86, -3 }, /* (127) subquery ::= NK_LP query_expression NK_RP */ + { 101, -1 }, /* (128) search_condition ::= boolean_value_expression */ + { 121, -1 }, /* (129) sort_specification_list ::= sort_specification */ + { 121, -3 }, /* (130) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 122, -3 }, /* (131) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 123, 0 }, /* (132) ordering_specification_opt ::= */ + { 123, -1 }, /* (133) ordering_specification_opt ::= ASC */ + { 123, -1 }, /* (134) ordering_specification_opt ::= DESC */ + { 124, 0 }, /* (135) null_ordering_opt ::= */ + { 124, -2 }, /* (136) null_ordering_opt ::= NULLS FIRST */ + { 124, -2 }, /* (137) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1514,27 +1530,27 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy212; } break; case 2: /* literal ::= NK_INTEGER */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 3: /* literal ::= NK_FLOAT */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 4: /* literal ::= NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 5: /* literal ::= NK_BOOL */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 6: /* literal ::= TIMESTAMP NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 7: /* literal ::= duration_literal */ case 17: /* expression ::= literal */ yytestcase(yyruleno==17); @@ -1546,25 +1562,25 @@ static YYACTIONTYPE yy_reduce( case 62: /* table_reference ::= table_primary */ yytestcase(yyruleno==62); case 63: /* table_reference ::= joined_table */ yytestcase(yyruleno==63); case 67: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==67); - case 111: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==111); - case 113: /* query_primary ::= query_specification */ yytestcase(yyruleno==113); - case 125: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==125); -{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 114: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==114); + case 116: /* query_primary ::= query_specification */ yytestcase(yyruleno==116); + case 128: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==128); +{ PARSER_TRACE; yylhsminor.yy212 = yymsp[0].minor.yy212; } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 8: /* duration_literal ::= NK_VARIABLE */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 9: /* literal_list ::= literal */ case 30: /* expression_list ::= expression */ yytestcase(yyruleno==30); -{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[0].minor.yy192 = yylhsminor.yy192; +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; case 10: /* literal_list ::= literal_list NK_COMMA literal */ case 31: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==31); -{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-2].minor.yy192 = yylhsminor.yy192; +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; case 11: /* db_name ::= NK_ID */ case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); @@ -1572,369 +1588,380 @@ static YYACTIONTYPE yy_reduce( case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); -{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy241 = yylhsminor.yy241; +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy192)); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, yymsp[-1].minor.yy174)); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 20: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 22: /* expression ::= NK_LP expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 23: /* expression ::= NK_PLUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 24: /* expression ::= NK_MINUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), NULL)); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 25: /* expression ::= expression NK_PLUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 26: /* expression ::= expression NK_MINUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 27: /* expression ::= expression NK_STAR expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 28: /* expression ::= expression NK_SLASH expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 29: /* expression ::= expression NK_REM expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy168); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 32: /* column_reference ::= column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNode(pCxt, &yymsp[0].minor.yy241, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241)); } - yymsp[0].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy79, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy79)); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; case 33: /* column_reference ::= table_name NK_DOT column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241, createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79, createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 34: /* predicate ::= expression compare_op expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 35: /* predicate ::= expression BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy168), releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-4].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy212), releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-4].minor.yy212 = yylhsminor.yy212; break; case 36: /* predicate ::= expression NOT BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), releaseRawExprNode(pCxt, yymsp[-5].minor.yy168), releaseRawExprNode(pCxt, yymsp[0].minor.yy168)); } - yymsp[-5].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[-5].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; break; case 37: /* predicate ::= expression IS NULL */ -{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), true); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), true); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 38: /* predicate ::= expression IS NOT NULL */ -{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), false); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), false); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 39: /* predicate ::= expression in_op in_predicate_value */ -{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 40: /* compare_op ::= NK_LT */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_THAN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_THAN; } break; case 41: /* compare_op ::= NK_GT */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_THAN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_THAN; } break; case 42: /* compare_op ::= NK_LE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_EQUAL; } break; case 43: /* compare_op ::= NK_GE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_EQUAL; } break; case 44: /* compare_op ::= NK_NE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NOT_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NOT_EQUAL; } break; case 45: /* compare_op ::= NK_EQ */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_EQUAL; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_EQUAL; } break; case 46: /* compare_op ::= LIKE */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LIKE; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LIKE; } break; case 47: /* compare_op ::= NOT LIKE */ -{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_LIKE; } +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_LIKE; } break; case 48: /* compare_op ::= MATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_MATCH; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_MATCH; } break; case 49: /* compare_op ::= NMATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NMATCH; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NMATCH; } break; case 50: /* in_op ::= IN */ -{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_IN; } +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_IN; } break; case 51: /* in_op ::= NOT IN */ -{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_IN; } +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_IN; } break; case 52: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yymsp[-2].minor.yy168 = createNodeListNode(pCxt, yymsp[-1].minor.yy192); } +{ PARSER_TRACE; yymsp[-2].minor.yy212 = createNodeListNode(pCxt, yymsp[-1].minor.yy174); } break; case 54: /* boolean_value_expression ::= NOT boolean_primary */ -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy212, NULL); } break; case 55: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 56: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 58: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ case 71: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==71); case 72: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==72); -{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } +{ PARSER_TRACE; yymsp[-2].minor.yy212 = yymsp[-1].minor.yy212; } break; case 59: /* from_clause ::= FROM table_reference_list */ - case 88: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==88); - case 109: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==109); -{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + case 89: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==89); + case 112: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==112); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = yymsp[0].minor.yy212; } break; case 61: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy212, yymsp[0].minor.yy212, NULL); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; case 64: /* table_primary ::= table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 65: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, &yymsp[-3].minor.yy79, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; case 66: /* table_primary ::= subquery alias_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; case 68: /* alias_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy241 = nil_token; } +{ PARSER_TRACE; yymsp[1].minor.yy79 = nil_token; } break; case 69: /* alias_opt ::= table_alias */ -{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy241; } - yymsp[0].minor.yy241 = yylhsminor.yy241; +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy79; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; case 70: /* alias_opt ::= AS table_alias */ -{ PARSER_TRACE; yymsp[-1].minor.yy241 = yymsp[0].minor.yy241; } +{ PARSER_TRACE; yymsp[-1].minor.yy79 = yymsp[0].minor.yy79; } break; case 73: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy229, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-5].minor.yy168 = yylhsminor.yy168; +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, yymsp[-4].minor.yy162, yymsp[-5].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; break; - case 74: /* join_type ::= INNER */ -{ PARSER_TRACE; yymsp[0].minor.yy229 = JOIN_TYPE_INNER; } + case 74: /* join_type ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy162 = JOIN_TYPE_INNER; } break; - case 75: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 75: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy162 = JOIN_TYPE_INNER; } + break; + case 76: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { PARSER_TRACE; - yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy209, yymsp[-6].minor.yy192, yymsp[-5].minor.yy168); - yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); - yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy192); - yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); - yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy192); - yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + yymsp[-8].minor.yy212 = createSelectStmt(pCxt, yymsp[-7].minor.yy237, yymsp[-6].minor.yy174, yymsp[-5].minor.yy212); + yymsp[-8].minor.yy212 = addWhereClause(pCxt, yymsp[-8].minor.yy212, yymsp[-4].minor.yy212); + yymsp[-8].minor.yy212 = addPartitionByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-3].minor.yy174); + yymsp[-8].minor.yy212 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy212, yymsp[-2].minor.yy212); + yymsp[-8].minor.yy212 = addGroupByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-1].minor.yy174); + yymsp[-8].minor.yy212 = addHavingClause(pCxt, yymsp[-8].minor.yy212, yymsp[0].minor.yy212); } break; - case 76: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy209 = false; } + case 77: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy237 = false; } break; - case 77: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy209 = true; } + case 78: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = true; } break; - case 78: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy209 = false; } + case 79: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = false; } break; - case 79: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy192 = NULL; } + case 80: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy174 = NULL; } break; - case 80: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy192 = yymsp[0].minor.yy192; } - yymsp[0].minor.yy192 = yylhsminor.yy192; + case 81: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy174 = yymsp[0].minor.yy174; } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 81: /* select_sublist ::= select_item */ - case 126: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==126); -{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, yymsp[0].minor.yy168); } - yymsp[0].minor.yy192 = yylhsminor.yy192; + case 82: /* select_sublist ::= select_item */ + case 129: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==129); +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy212); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 82: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 127: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==127); -{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy192 = yylhsminor.yy192; + case 83: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 130: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==130); +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; - case 83: /* select_item ::= expression */ + case 84: /* select_item ::= expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy168); - yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy168), &t); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); + yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), &t); } - yymsp[0].minor.yy168 = yylhsminor.yy168; + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 84: /* select_item ::= expression column_alias */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + case 85: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; - case 85: /* select_item ::= expression AS column_alias */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), &yymsp[0].minor.yy241); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 86: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 86: /* select_item ::= table_name NK_DOT NK_STAR */ -{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 87: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 87: /* where_clause_opt ::= */ - case 91: /* twindow_clause_opt ::= */ yytestcase(yyruleno==91); - case 96: /* sliding_opt ::= */ yytestcase(yyruleno==96); - case 98: /* fill_opt ::= */ yytestcase(yyruleno==98); - case 108: /* having_clause_opt ::= */ yytestcase(yyruleno==108); - case 116: /* slimit_clause_opt ::= */ yytestcase(yyruleno==116); - case 120: /* limit_clause_opt ::= */ yytestcase(yyruleno==120); -{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } + case 88: /* where_clause_opt ::= */ + case 92: /* twindow_clause_opt ::= */ yytestcase(yyruleno==92); + case 97: /* sliding_opt ::= */ yytestcase(yyruleno==97); + case 99: /* fill_opt ::= */ yytestcase(yyruleno==99); + case 111: /* having_clause_opt ::= */ yytestcase(yyruleno==111); + case 119: /* slimit_clause_opt ::= */ yytestcase(yyruleno==119); + case 123: /* limit_clause_opt ::= */ yytestcase(yyruleno==123); +{ PARSER_TRACE; yymsp[1].minor.yy212 = NULL; } break; - case 89: /* partition_by_clause_opt ::= */ - case 106: /* group_by_clause_opt ::= */ yytestcase(yyruleno==106); - case 114: /* order_by_clause_opt ::= */ yytestcase(yyruleno==114); -{ PARSER_TRACE; yymsp[1].minor.yy192 = NULL; } + case 90: /* partition_by_clause_opt ::= */ + case 107: /* group_by_clause_opt ::= */ yytestcase(yyruleno==107); + case 117: /* order_by_clause_opt ::= */ yytestcase(yyruleno==117); +{ PARSER_TRACE; yymsp[1].minor.yy174 = NULL; } break; - case 90: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 107: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==107); - case 115: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==115); -{ PARSER_TRACE; yymsp[-2].minor.yy192 = yymsp[0].minor.yy192; } + case 91: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 108: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==108); + case 118: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==118); +{ PARSER_TRACE; yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } break; - case 92: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy168), &yymsp[-1].minor.yy0); } + case 93: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), &yymsp[-1].minor.yy0); } break; - case 93: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy168)); } + case 94: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } break; - case 94: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy168, NULL, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + case 95: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy212, NULL, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 95: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-7].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy168, yymsp[-3].minor.yy168, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + case 96: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy212, yymsp[-3].minor.yy212, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 97: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = yymsp[-1].minor.yy168; } + case 98: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = yymsp[-1].minor.yy212; } break; - case 99: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createFillNode(pCxt, yymsp[-1].minor.yy14, NULL); } + case 100: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createFillNode(pCxt, yymsp[-1].minor.yy44, NULL); } break; - case 100: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy192)); } + case 101: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } break; - case 101: /* fill_mode ::= NONE */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NONE; } + case 102: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NONE; } break; - case 102: /* fill_mode ::= PREV */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_PREV; } + case 103: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_PREV; } break; - case 103: /* fill_mode ::= NULL */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NULL; } + case 104: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NULL; } break; - case 104: /* fill_mode ::= LINEAR */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_LINEAR; } + case 105: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_LINEAR; } break; - case 105: /* fill_mode ::= NEXT */ -{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NEXT; } + case 106: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NEXT; } break; - case 110: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 109: /* group_by_list ::= expression */ +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } + yymsp[0].minor.yy174 = yylhsminor.yy174; + break; + case 110: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; + break; + case 113: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { PARSER_TRACE; - yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy192); - yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); - yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + yylhsminor.yy212 = addOrderByClause(pCxt, yymsp[-3].minor.yy212, yymsp[-2].minor.yy174); + yylhsminor.yy212 = addSlimitClause(pCxt, yylhsminor.yy212, yymsp[-1].minor.yy212); + yylhsminor.yy212 = addLimitClause(pCxt, yylhsminor.yy212, yymsp[0].minor.yy212); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 112: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + case 115: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy212 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==121); -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 120: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 124: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==122); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 121: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 125: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 119: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 123: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==123); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 122: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 126: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 124: /* subquery ::= NK_LP query_expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy168 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 127: /* subquery ::= NK_LP query_expression NK_RP */ +{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 128: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy168), yymsp[-1].minor.yy10, yymsp[0].minor.yy177); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 131: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[-1].minor.yy188, yymsp[0].minor.yy107); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 129: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy10 = ORDER_ASC; } + case 132: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy188 = ORDER_ASC; } break; - case 130: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_ASC; } + case 133: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_ASC; } break; - case 131: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_DESC; } + case 134: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_DESC; } break; - case 132: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy177 = NULL_ORDER_DEFAULT; } + case 135: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy107 = NULL_ORDER_DEFAULT; } break; - case 133: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_FIRST; } + case 136: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_FIRST; } break; - case 134: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_LAST; } + case 137: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index dede151004..cd42b9ad64 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -116,10 +116,6 @@ static uint32_t toNewTokenId(uint32_t tokenId) { return NEW_TK_FROM; case TK_JOIN: return NEW_TK_JOIN; - // case TK_ON: - // return NEW_TK_ON; - // case TK_INNER: - // return NEW_TK_INNER; // case TK_PARTITION: // return NEW_TK_PARTITION; case TK_SESSION: @@ -163,6 +159,8 @@ static uint32_t toNewTokenId(uint32_t tokenId) { case TK_OFFSET: return NEW_TK_OFFSET; case TK_SPACE: + case NEW_TK_ON: + case NEW_TK_INNER: break; default: printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!tokenId = %d\n", tokenId); @@ -248,6 +246,10 @@ typedef enum ESqlClause { } ESqlClause; static bool afterGroupBy(ESqlClause clause) { + return clause > SQL_CLAUSE_GROUP_BY; +} + +static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } @@ -259,7 +261,7 @@ typedef struct STranslateContext { SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* int32_t currLevel; ESqlClause currClause; - void* pExt; + SSelectStmt* pCurrStmt; } STranslateContext; static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); @@ -284,6 +286,12 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "There mustn't be aggregation"; case TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT: return "ORDER BY item must be the number of a SELECT-list expression"; + case TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION: + return "Not a GROUP BY expression"; + case TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION: + return "Not SELECTed expression"; + case TSDB_CODE_PAR_NOT_SINGLE_GROUP: + return "Not a single-group group function"; default: return "Unknown error"; } @@ -329,7 +337,7 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST if ('\0' != pCol->dbName[0]) { cmp = strcmp(pCol->dbName, pTable->dbName); } else { - cmp = strcmp(currentDb, pTable->dbName); + cmp = (QUERY_NODE_REAL_TABLE == nodeType(pTable) ? strcmp(currentDb, pTable->dbName) : 0); } if (0 == cmp) { cmp = strcmp(pCol->tableAlias, pTable->tableAlias); @@ -415,23 +423,29 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { return found; } -static bool translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); + bool foundTable = false; for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + foundTable = true; if (findAndSetColumn(pCol, pTable)) { break; } generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } } - return true; + if (!foundTable) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pCol->tableAlias); + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; } -static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); bool found = false; @@ -440,20 +454,20 @@ static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* p if (findAndSetColumn(pCol, pTable)) { if (found) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } found = true; } } if (!found) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); - return false; + return DEAL_RES_ERROR; } - return true; + return DEAL_RES_CONTINUE; } static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol) { - SNodeList* pProjectionList = pCxt->pExt; + SNodeList* pProjectionList = pCxt->pCurrStmt->pProjectionList; SNode* pNode; FOREACH(pNode, pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; @@ -465,10 +479,10 @@ static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol) return false; } -static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { +static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { // count(*)/first(*)/last(*) if (0 == strcmp(pCol->colName, "*")) { - return true; + return DEAL_RES_CONTINUE; } if ('\0' != pCol->tableAlias[0]) { return translateColumnWithPrefix(pCxt, pCol); @@ -477,7 +491,7 @@ static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { if (SQL_CLAUSE_ORDER_BY == pCxt->currClause) { found = translateColumnUseAlias(pCxt, pCol); } - return found ? true : translateColumnWithoutPrefix(pCxt, pCol); + return found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol); } static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { @@ -500,12 +514,12 @@ static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { return j; } -static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { +static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { if (pVal->isDuration) { char unit = 0; if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - return false; + return DEAL_RES_ERROR; } } else { switch (pVal->node.resType.type) { @@ -552,7 +566,7 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { if (taosParseTime(tmp, &pVal->datum.u, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { tfree(tmp); generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - return false; + return DEAL_RES_ERROR; } tfree(tmp); break; @@ -565,55 +579,56 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { break; } } - return true; + return DEAL_RES_CONTINUE; } -static bool translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { +static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; if (nodesIsArithmeticOp(pOp)) { if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - return false; + return DEAL_RES_ERROR; } pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - return true; } else if (nodesIsComparisonOp(pOp)) { if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - return false; + return DEAL_RES_ERROR; } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - return true; } else { // todo json operator - return true; } - return true; + return DEAL_RES_CONTINUE; } -static bool translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { +static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pCxt->fmgt, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); - return false; + return DEAL_RES_ERROR; } int32_t code = fmGetFuncResultType(pFunc); if (TSDB_CODE_SUCCESS != code) { generateSyntaxErrMsg(pCxt, code, pFunc->functionName); - return false; + return DEAL_RES_ERROR; } - if (fmIsAggFunc(pFunc->funcId) && afterGroupBy(pCxt->currClause)) { + if (fmIsAggFunc(pFunc->funcId) && beforeHaving(pCxt->currClause)) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); - return false; + return DEAL_RES_ERROR; } - return true; + return DEAL_RES_CONTINUE; } -static bool doTranslateExpr(SNode* pNode, void* pContext) { +static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { + return (TSDB_CODE_SUCCESS == translateSubquery(pCxt, pNode) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: @@ -625,11 +640,11 @@ static bool doTranslateExpr(SNode* pNode, void* pContext) { case QUERY_NODE_FUNCTION: return translateFunction(pCxt, (SFunctionNode*)pNode); case QUERY_NODE_TEMP_TABLE: - return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + return translateExprSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); default: break; } - return true; + return DEAL_RES_CONTINUE; } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { @@ -642,6 +657,103 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } +static bool isAliasColumn(SColumnNode* pCol) { + return ('\0' == pCol->tableAlias[0]); +} + +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + +static SNodeList* getGroupByList(STranslateContext* pCxt) { + if (isDistinctOrderBy(pCxt)) { + return pCxt->pCurrStmt->pProjectionList; + } + return pCxt->pCurrStmt->pGroupByList; +} + +static SNode* getGroupByNode(SNode* pNode) { + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + return nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0); + } + return pNode; +} + +static int32_t getGroupByErrorCode(STranslateContext* pCxt) { + if (isDistinctOrderBy(pCxt)) { + return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION; + } + return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; +} + +static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { + STranslateContext* pCxt = (STranslateContext*)pContext; + if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && !isDistinctOrderBy(pCxt)) { + return DEAL_RES_IGNORE_CHILD; + } + SNode* pGroupNode; + FOREACH(pGroupNode, getGroupByList(pCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + } + if (QUERY_NODE_COLUMN == nodeType(pNode) || + (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && isDistinctOrderBy(pCxt))) { + generateSyntaxErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { + nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + return pCxt->errCode; +} + +static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList) { + if (NULL == getGroupByList(pCxt)) { + return TSDB_CODE_SUCCESS; + } + nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + return pCxt->errCode; +} + +typedef struct CheckAggColCoexistCxt { + STranslateContext* pTranslateCxt; + bool existAggFunc; + bool existCol; +} CheckAggColCoexistCxt; + +static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { + CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->existAggFunc = true; + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + pCxt->existCol = true; + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL != pSelect->pGroupByList) { + return TSDB_CODE_SUCCESS; + } + CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; + nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + if (!pSelect->isDistinct) { + nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + } + if (cxt.existAggFunc && cxt.existCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_NOT_SINGLE_GROUP); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -732,12 +844,13 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec *pOther = false; SNode* pNode; FOREACH(pNode, pOrderByList) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { - SValueNode* pVal = (SValueNode*)pNode; - if (translateValue(pCxt, pVal)) { + SNode* pExpr = ((SOrderByExprNode*)pNode)->pExpr; + if (QUERY_NODE_VALUE == nodeType(pExpr)) { + SValueNode* pVal = (SValueNode*)pExpr; + if (!translateValue(pCxt, pVal)) { return false; } - int32_t pos = getPositionValue((SValueNode*)pNode); + int32_t pos = getPositionValue((SValueNode*)pExpr); if (pos < 0) { ERASE_NODE(pOrderByList); nodesDestroyNode(pNode); @@ -747,9 +860,9 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec return false; } else { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos), pCol); - REPLACE_NODE(pCol); - nodesDestroyNode(pNode); + setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), pCol); + ((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol; + nodesDestroyNode(pExpr); } } else { *pOther = true; @@ -758,17 +871,20 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec return true; } -static int32_t translateOrderBy(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList) { +static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - if (!translateOrderByPosition(pCxt, pProjectionList, pOrderByList, &other)) { + if (!translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other)) { return pCxt->errCode; } if (!other) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_ORDER_BY; - pCxt->pExt = pProjectionList; - return translateExprList(pCxt, pOrderByList); + int32_t code = translateExprList(pCxt, pSelect->pOrderByList); + if (TSDB_CODE_SUCCESS == code) { + code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); + } + return code; } static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -778,12 +894,22 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExprList(pCxt, pSelect->pProjectionList); } + if (TSDB_CODE_SUCCESS == code) { + code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList); + } return code; } -static int32_t translateHaving(STranslateContext* pCxt, SNode* pHaving) { +static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + } pCxt->currClause = SQL_CLAUSE_HAVING; - return translateExpr(pCxt, pHaving); + int32_t code = translateExpr(pCxt, pSelect->pHaving); + if (TSDB_CODE_SUCCESS == code) { + code = checkExprForGroupBy(pCxt, pSelect->pHaving); + } + return code; } static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList) { @@ -818,8 +944,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { // } SSelectStmt; static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { - int32_t code = TSDB_CODE_SUCCESS; - code = translateFrom(pCxt, pSelect->pFromTable); + pCxt->pCurrStmt = pSelect; + int32_t code = translateFrom(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { code = translateWhere(pCxt, pSelect->pWhere); } @@ -833,15 +959,17 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translateGroupBy(pCxt, pSelect->pGroupByList); } if (TSDB_CODE_SUCCESS == code) { - code = translateHaving(pCxt, pSelect->pHaving); + code = translateHaving(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateSelectList(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - code = translateOrderBy(pCxt, pSelect->pProjectionList, pSelect->pOrderByList); + code = translateOrderBy(pCxt, pSelect); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkAggColCoexist(pCxt, pSelect); } - // printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, code); return code; } @@ -860,11 +988,11 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { ++(pCxt->currLevel); ESqlClause currClause = pCxt->currClause; - void* pExt = pCxt->pExt; + SSelectStmt* pCurrStmt = pCxt->pCurrStmt; int32_t code = translateQuery(pCxt, pNode); --(pCxt->currLevel); pCxt->currClause = currClause; - pCxt->pExt = pExt; + pCxt->pCurrStmt = pCurrStmt; return code; } diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/ttokenizer.c index 8e5b22aaa3..d5d6b11833 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/ttokenizer.c @@ -227,7 +227,9 @@ static SKeyword keywordTable[] = { {"OUTPUTTYPE", TK_OUTPUTTYPE}, {"AGGREGATE", TK_AGGREGATE}, {"BUFSIZE", TK_BUFSIZE}, - {"PORT", TK_PORT}, + {"PORT", TK_PORT}, + {"INNER", NEW_TK_INNER}, + {"ON", NEW_TK_ON}, }; static const char isIdChar[] = { diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index be239bccfb..2616c6c251 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -53,8 +53,8 @@ protected: code = doTranslate(&cxt_, &query_); // cout << "doTranslate return " << code << endl; if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return (TSDB_CODE_SUCCESS != translateCode); + cout << "sql:[" << cxt_.pSql << "] code:" << code << ", " << translateCode << ", msg:" << errMagBuf_ << endl; + return (code == translateCode); } if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { cout << "input sql : [" << cxt_.pSql << "]" << endl; @@ -71,6 +71,24 @@ protected: private: static const int max_err_len = 1024; + void exprNodeToStr(const SNode* node, string& str, bool isProject) { + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_OPERATOR: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)node; + str.append(" [" + dataTypeToStr(pExpr->resType) + "]"); + if (isProject) { + str.append(" AS " + string(pExpr->aliasName)); + } + break; + } + default: + break; + } + } + string dataTypeToStr(const SDataType& dt) { switch (dt.type) { case TSDB_DATA_TYPE_NULL: @@ -119,7 +137,7 @@ private: return "Unknown Data Type " + to_string(dt.type); } - void valueNodeToStr(const SValueNode* pVal, string& str, bool isProject) { + void valueNodeToStr(const SValueNode* pVal, string& str) { switch (pVal->node.resType.type) { case TSDB_DATA_TYPE_NULL: str.append("null"); @@ -160,10 +178,41 @@ private: default: break; } - str.append(" [" + dataTypeToStr(pVal->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pVal->node.aliasName)); + } + + void columnNodeToStr(const SColumnNode* pCol, string& str) { + if ('\0' != pCol->dbName[0]) { + str.append(pCol->dbName); + str.append("."); } + if ('\0' != pCol->tableAlias[0]) { + str.append(pCol->tableAlias); + str.append("."); + } + str.append(pCol->colName); + } + + void operatorToStr(const SOperatorNode* pOp, string& str) { + nodeToStr(pOp->pLeft, str, false); + str.append(opTypeToStr(pOp->opType)); + nodeToStr(pOp->pRight, str, false); + } + + void functionToStr(const SFunctionNode* pFunc, string& str) { + str.append(pFunc->functionName); + str.append("("); + nodeListToStr(pFunc->pParameterList, "", str, false, ", "); + str.append(")"); + } + + void groupingSetToStr(SGroupingSetNode* pGroup, string& str) { + nodeToStr(nodesListGetNode(pGroup->pParameterList, 0), str, false); + } + + void orderByExprToStr(SOrderByExprNode* pOrderBy, string& str) { + nodeToStr(pOrderBy->pExpr, str, false); + str.append((ORDER_ASC == pOrderBy->order ? " ASC" : " DESC")); + str.append((NULL_ORDER_FIRST == pOrderBy->nullOrder ? " NULLS FIRST" : " NULLS LAST")); } void nodeToStr(const SNode* node, string& str, bool isProject) { @@ -173,48 +222,41 @@ private: switch (nodeType(node)) { case QUERY_NODE_COLUMN: { - SColumnNode* pCol = (SColumnNode*)node; - if ('\0' != pCol->dbName[0]) { - str.append(pCol->dbName); - str.append("."); - } - if ('\0' != pCol->tableAlias[0]) { - str.append(pCol->tableAlias); - str.append("."); - } - str.append(pCol->colName); - str.append(" [" + dataTypeToStr(pCol->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pCol->node.aliasName)); - } + columnNodeToStr((SColumnNode*)node, str); break; } case QUERY_NODE_VALUE: { - valueNodeToStr((SValueNode*)node, str, isProject); + valueNodeToStr((SValueNode*)node, str); break; } case QUERY_NODE_OPERATOR: { - SOperatorNode* pOp = (SOperatorNode*)node; - nodeToStr(pOp->pLeft, str, false); - str.append(opTypeToStr(pOp->opType)); - nodeToStr(pOp->pRight, str, false); - str.append(" [" + dataTypeToStr(pOp->node.resType) + "]"); - if (isProject) { - str.append(" AS " + string(pOp->node.aliasName)); - } + operatorToStr((SOperatorNode*)node, str); + break; + } + case QUERY_NODE_FUNCTION: { + functionToStr((SFunctionNode*)node, str); + break; + } + case QUERY_NODE_GROUPING_SET: { + groupingSetToStr((SGroupingSetNode*)node, str); + break; + } + case QUERY_NODE_ORDER_BY_EXPR: { + orderByExprToStr((SOrderByExprNode*)node, str); break; } default: break; } + exprNodeToStr(node, str, isProject); } - void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false) { + void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false, const string& sep = string("\n")) { SNode* node = nullptr; FOREACH(node, nodelist) { str.append(prefix); nodeToStr(node, str, isProject); - str.append("\n"); + str.append(sep); } } @@ -265,8 +307,20 @@ private: } str.append("\n"); nodeListToStr(select->pProjectionList, prefix + "\t", str, true); - str.append("\n" + prefix + "FROM\n"); + str.append(prefix + "FROM\n"); tableToStr(select->pFromTable, prefix + "\t", str); + if (nullptr != select->pWhere) { + str.append("\n" + prefix + "WHERE\n\t"); + nodeToStr(select->pWhere, str, false); + } + if (nullptr != select->pGroupByList) { + str.append("\n" + prefix + "GROUP BY\n"); + nodeListToStr(select->pGroupByList, prefix + "\t", str, true); + } + if (nullptr != select->pOrderByList) { + str.append(prefix + "ORDER BY\n"); + nodeListToStr(select->pOrderByList, prefix + "\t", str, true); + } } void selectToSql(const SNode* node, string& sql) { @@ -332,15 +386,23 @@ private: case OP_TYPE_SUB: return " - "; case OP_TYPE_MULTI: + return " * "; case OP_TYPE_DIV: + return " / "; case OP_TYPE_MOD: + return " % "; case OP_TYPE_GREATER_THAN: + return " > "; case OP_TYPE_GREATER_EQUAL: + return " >= "; case OP_TYPE_LOWER_THAN: + return " < "; case OP_TYPE_LOWER_EQUAL: + return " <= "; case OP_TYPE_EQUAL: return " = "; case OP_TYPE_NOT_EQUAL: + return " != "; case OP_TYPE_IN: case OP_TYPE_NOT_IN: case OP_TYPE_LIKE: @@ -454,11 +516,41 @@ TEST_F(NewParserTest, selectExpression) { TEST_F(NewParserTest, selectClause) { setDatabase("root", "test"); + // GROUP BY clause + bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); + ASSERT_TRUE(run()); + + bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 HAVING count(c1) > 10"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); + ASSERT_TRUE(run()); + + bind("SELECT count(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); + ASSERT_TRUE(run()); + + // ORDER BY clause bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY cnt"); ASSERT_TRUE(run()); bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1"); ASSERT_TRUE(run()); + + // DISTINCT clause + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, selectSyntaxError) { @@ -480,15 +572,88 @@ TEST_F(NewParserTest, selectSyntaxError) { TEST_F(NewParserTest, selectSemanticError) { setDatabase("root", "test"); - bind("SELECT * FROM t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); - + // TSDB_CODE_PAR_INVALID_COLUMN bind("SELECT c1, c3 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); + bind("SELECT t1.c1, t1.c3 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); + + // TSDB_CODE_PAR_TABLE_NOT_EXIST + bind("SELECT * FROM t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + bind("SELECT * FROM test.t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + bind("SELECT t2.c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + + // TSDB_CODE_PAR_AMBIGUOUS_COLUMN bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN)); + + // TSDB_CODE_PAR_WRONG_VALUE_TYPE + bind("SELECT 10n FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); + + bind("SELECT TIMESTAMP '2010' FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); + + // TSDB_CODE_PAR_INVALID_FUNTION + bind("SELECT cnt(*) FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_FUNTION)); + + // TSDB_CODE_PAR_FUNTION_PARA_NUM + // TSDB_CODE_PAR_FUNTION_PARA_TYPE + + // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION + bind("SELECT c2 FROM t1 tt1 JOIN t1 tt2 ON count(*) > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); bind("SELECT c2 FROM t1 where count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); + + bind("SELECT c2 FROM t1 GROUP BY count(*)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); + + // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT + bind("SELECT c2 FROM t1 ORDER BY 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); + + bind("SELECT c2 FROM t1 ORDER BY 2"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); + + // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION + bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c1 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*), c1 cnt FROM t1 GROUP BY c2 HAVING c2 > 0"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + // TSDB_CODE_PAR_NOT_SINGLE_GROUP + bind("SELECT count(*), c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT count(*) FROM t1 ORDER BY c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT c1 FROM t1 ORDER BY count(*)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 53ee288930..6564817301 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -96,9 +96,9 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_VALUE; } - pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo)); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 846f2d5099..6f8da57ee7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -151,7 +151,8 @@ typedef struct { char version : 4; // RPC version char comp : 4; // compression algorithm, 0:no compression 1:lz4 char resflag : 2; // reserved bits - char spi : 3; // security parameter index + char spi : 1; // security parameter index + char secured : 2; char encrypt : 3; // encrypt algorithm, 0: no encryption uint32_t code; // del later @@ -170,6 +171,11 @@ typedef struct { uint8_t auth[TSDB_AUTH_LEN]; } STransDigestMsg; +typedef struct { + uint8_t user[TSDB_UNI_LEN]; + uint8_t secret[TSDB_PASSWORD_LEN]; +} STransUserMsg; + #pragma pack(pop) #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) @@ -227,7 +233,7 @@ typedef struct { uv_async_t* asyncs; } SAsyncPool; -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); @@ -236,4 +242,8 @@ int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +// int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); + +// int transUnpackMsg(char *msg, SRpcMsg *pMsg, bool ); + #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5fc937bccd..c3d3cfa2ab 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -30,12 +30,21 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->cfp = pInit->cfp; - // pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->numOfThreads = pInit->numOfThreads; + if (pInit->connType == TAOS_CONN_SERVER) { + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + } else { + pRpc->numOfThreads = pInit->numOfThreads; + } pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); pRpc->parent = pInit->parent; + if (pInit->user) { + memcpy(pRpc->user, pInit->user, strlen(pInit->user)); + } + if (pInit->secret) { + memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret)); + } return pRpc; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 29770831fa..00d9174e76 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -38,6 +38,7 @@ typedef struct SCliConn { int32_t ref; // debug and log info struct sockaddr_in addr; + struct sockaddr_in locaddr; } SCliConn; typedef struct SCliMsg { @@ -130,14 +131,16 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - tDebug("client conn %p %s received from %s:%d", conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), - ntohs(conn->addr.sin_port)); + tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), + inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), + ntohs(conn->locaddr.sin_port)); if (conn->push != NULL && conn->notifyCount != 0) { (*conn->push->callback)(conn->push->arg, &rpcMsg); + conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn(sync) %p handle resp", conn); + tTrace("client conn%p handle resp", conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { tTrace("client conn(sync) %p handle resp", conn); @@ -146,6 +149,7 @@ static void clientHandleResp(SCliConn* conn) { } } conn->notifyCount += 1; + conn->secured = pHead->secured; // buf's mem alread translated to rpcMsg.pCont transClearBuffer(&conn->readBuf); @@ -154,12 +158,12 @@ static void clientHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; // user owns conn->persist = 1 - if (conn->push != NULL) { + if (conn->push == NULL) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); - } - destroyCmsg(pMsg); - conn->data = NULL; + destroyCmsg(conn->data); + conn->data = NULL; + } // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); @@ -182,21 +186,23 @@ static void clientHandleExcept(SCliConn* pConn) { if (pConn->push != NULL && pConn->notifyCount != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); + pConn->push = NULL; } else { if (pCtx->pSem == NULL) { (pCtx->pTransInst->cfp)(pCtx->pTransInst->parent, &rpcMsg, NULL); } else { memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg)); - // SRpcMsg rpcMsg tsem_post(pCtx->pSem); } if (pConn->push != NULL) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); } + pConn->push = NULL; + } + if (pConn->push == NULL) { + destroyCmsg(pConn->data); + pConn->data = NULL; } - - destroyCmsg(pMsg); - pConn->data = NULL; // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); pConn->notifyCount += 1; @@ -382,17 +388,40 @@ static void clientWriteCb(uv_write_t* req, int status) { static void clientWrite(SCliConn* pConn) { SCliMsg* pCliMsg = pConn->data; - SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); - STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + STransConnCtx* pCtx = pCliMsg->ctx; + SRpcInfo* pTransInst = pCtx->pTransInst; - int msgLen = transMsgLenFromCont(pMsg->contLen); + SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); + + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + int msgLen = transMsgLenFromCont(pMsg->contLen); + + if (!pConn->secured) { + char* buf = calloc(1, msgLen + sizeof(STransUserMsg)); + memcpy(buf, (char*)pHead, msgLen); + + STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); + memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user)); + memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret)); + + // to avoid mem leak + destroyUserdata(pMsg); + + pMsg->pCont = (char*)buf + sizeof(STransMsgHead); + pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead); + + pHead = (STransMsgHead*)buf; + pHead->secured = 1; + msgLen += sizeof(STransUserMsg); + } pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("client conn %p %s is send to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { @@ -407,6 +436,9 @@ static void clientConnCb(uv_connect_t* req, int status) { int addrlen = sizeof(pConn->addr); uv_tcp_getpeername((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->addr, &addrlen); + addrlen = sizeof(pConn->locaddr); + uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); + tTrace("client conn %p create", pConn); assert(pConn->stream == req->handle); @@ -428,10 +460,6 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("client msg tran time cost: %" PRIu64 "", el); et = taosGetTimestampUs(); - // if (pMsg->msg.handle != NULL) { - // // handle - //} - STransConnCtx* pCtx = pMsg->ctx; SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { @@ -557,7 +585,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index d0e504a0a1..7aa5aa16f1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -211,7 +211,8 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { /* * formate of data buffer: * |<--------------------------data from socket------------------------------->| - * |<------STransMsgHead------->|<-------------------other data--------------->| + * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user + * info--->| */ static const int CAPACITY = 1024; @@ -239,6 +240,9 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } return 0; } +int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} + +int transUnpackMsg(STransMsgHead* msgHead) {} int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { tfree(buf->buf); @@ -246,9 +250,7 @@ int transDestroyBuffer(SConnBuffer* buf) { transClearBuffer(buf); } -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { - static int sz = 10; - +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b8bbea92ce..7ddeb99c9d 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -31,9 +31,11 @@ typedef struct SSrvConn { void* pTransInst; // rpc init void* ahandle; // void* hostThrd; - void* pSrvMsg; + SArray* srvMsgs; + // void* pSrvMsg; struct sockaddr_in addr; + struct sockaddr_in locaddr; // SRpcMsg sendMsg; // del later @@ -94,6 +96,7 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); +static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSrvMsg* msg); @@ -231,6 +234,11 @@ static void uvHandleReq(SSrvConn* pConn) { p->chandle = NULL; STransMsgHead* pHead = (STransMsgHead*)p->msg; + if (pHead->secured == 1) { + STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); + memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); + memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); + } pConn->inType = pHead->msgType; assert(transIsReq(pHead->msgType)); @@ -258,8 +266,9 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; - tDebug("server conn %p %s received from %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -305,14 +314,19 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnWriteCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - - SSrvMsg* smsg = conn->pSrvMsg; - destroySmsg(smsg); - conn->pSrvMsg = NULL; - transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); + assert(taosArrayGetSize(conn->srvMsgs) >= 1); + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + taosArrayRemove(conn->srvMsgs, 0); + destroySmsg(msg); + + // send second data, just use for push + if (taosArrayGetSize(conn->srvMsgs) > 0) { + msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); + uvStartSendRespInternal(msg); + } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); // @@ -331,41 +345,55 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); - SRpcMsg* pMsg = &smsg->msg; + SSrvConn* pConn = smsg->pConn; + SRpcMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + + pHead->secured = pMsg->code == 0 ? 1 : 0; // pHead->msgType = smsg->pConn->inType + 1; + pHead->code = htonl(pMsg->code); // add more info char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); if (transCompressMsg(msg, len, NULL)) { // impl later } - tDebug("server conn %p %s is sent to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); wb->base = msg; wb->len = len; } -static void uvStartSendResp(SSrvMsg* smsg) { - // impl + +static void uvStartSendRespInternal(SSrvMsg* smsg) { uv_buf_t wb; uvPrepareSendData(smsg, &wb); SSrvConn* pConn = smsg->pConn; uv_timer_stop(pConn->pTimer); - pConn->pSrvMsg = smsg; + // pConn->pSrvMsg = smsg; // conn->pWriter->data = smsg; uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb); - - // SRpcMsg* rpcMsg = smsg->msg; - +} +static void uvStartSendResp(SSrvMsg* smsg) { + // impl + SSrvConn* pConn = smsg->pConn; + if (taosArrayGetSize(pConn->srvMsgs) > 0) { + tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + taosArrayPush(pConn->srvMsgs, &smsg); + return; + } + taosArrayPush(pConn->srvMsgs, &smsg); + uvStartSendRespInternal(smsg); return; } static void destroySmsg(SSrvMsg* smsg) { @@ -487,13 +515,23 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tTrace("server conn %p created, fd: %d", pConn, fd); + int addrlen = sizeof(pConn->addr); if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) { tError("server conn %p failed to get peer info", pConn); destroyConn(pConn, true); - } else { - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + return; } + + addrlen = sizeof(pConn->locaddr); + if (0 != uv_tcp_getsockname(pConn->pTcp, (struct sockaddr*)&pConn->locaddr, &addrlen)) { + tError("server conn %p failed to get local info", pConn); + destroyConn(pConn, true); + return; + } + + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { tDebug("failed to create new connection"); destroyConn(pConn, true); @@ -522,7 +560,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -562,6 +600,7 @@ void* workerThread(void* arg) { static SSrvConn* createConn() { SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // tTrace("conn %p created", pConn); ++pConn->ref; return pConn; @@ -576,8 +615,15 @@ static void destroyConn(SSrvConn* conn, bool clear) { return; } transDestroyBuffer(&conn->readBuf); - destroySmsg(conn->pSrvMsg); - conn->pSrvMsg = NULL; + + for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) { + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); + destroySmsg(msg); + } + taosArrayDestroy(conn->srvMsgs); + + // destroySmsg(conn->pSrvMsg); + // conn->pSrvMsg = NULL; if (clear) { uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 3c9c40f46a..b4f50219ff 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -3,12 +3,13 @@ add_executable(client "") add_executable(server "") add_executable(transUT "") add_executable(syncClient "") +add_executable(pushClient "") +add_executable(pushServer "") target_sources(transUT PRIVATE "transUT.cc" ) - target_sources(transportTest PRIVATE "transportTests.cc" @@ -26,6 +27,15 @@ target_sources (syncClient "syncClient.c" ) +target_sources(pushClient + PRIVATE + "pushClient.c" +) +target_sources(pushServer + PRIVATE + "pushServer.c" +) + target_include_directories(transportTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" @@ -92,4 +102,29 @@ target_link_libraries (syncClient transport ) +target_include_directories(pushClient + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (pushClient + os + util + common + gtest_main + transport +) + +target_include_directories(pushServer + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (pushServer + os + util + common + gtest_main + transport +) diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index 2756eb4666..f1aadafacc 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -49,9 +49,8 @@ static int tcount = 0; typedef struct SPushArg { tsem_t sem; - } SPushArg; - +// ping int pushCallback(void *arg, SRpcMsg *msg) { SPushArg *push = arg; tsem_post(&push->sem); @@ -59,7 +58,8 @@ int pushCallback(void *arg, SRpcMsg *msg) { SRpcPush *createPushArg() { SRpcPush *push = calloc(1, sizeof(SRpcPush)); push->arg = calloc(1, sizeof(SPushArg)); - tsem_init(&push->arg->sem, 0, 0); + + tsem_init(&(((SPushArg *)push->arg)->sem), 0, 0); push->callback = pushCallback; return push; } @@ -83,14 +83,17 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; rpcMsg.push = push; - ; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - tsem_wait(&pInfo->rspSem); + tsem_wait(&pInfo->rspSem); // ping->pong // tsem_wait(&pInfo->rspSem); - tsem_wait(&push->sem); + SPushArg *arg = push->arg; + /// e + tsem_wait(&arg->sem); // push callback + + // query_fetch(client->h) int64_t end = taosGetTimestampUs() - start; if (end <= 100) { u100++; diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c new file mode 100644 index 0000000000..0bcc47383b --- /dev/null +++ b/source/libs/transport/test/pushServer.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +//#define _DEFAULT_SOURCE +#include "os.h" +#include "rpcLog.h" +#include "tglobal.h" +#include "tqueue.h" +#include "trpc.h" + +int msgSize = 128; +int commit = 0; +int dataFd = -1; +STaosQueue *qhandle = NULL; +STaosQset * qset = NULL; + +void processShellMsg() { + static int num = 0; + STaosQall *qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + void * pvnode; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode, NULL); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + + if (dataFd >= 0) { + if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + tInfo("failed to write data file, reason:%s", strerror(errno)); + } + } + } + + if (commit >= 2) { + num += numOfMsgs; + // if (taosFsync(dataFd) < 0) { + // tInfo("failed to flush data to file, reason:%s", strerror(errno)); + //} + + if (num % 10000 == 0) { + tInfo("%d request have been written into disk", num); + } + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + void *handle = pRpcMsg->handle; + taosFreeQitem(pRpcMsg); + + { + // sleep(1); + SRpcMsg nRpcMsg = {0}; + nRpcMsg.pCont = rpcMallocCont(msgSize); + nRpcMsg.contLen = msgSize; + nRpcMsg.handle = handle; + nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; + rpcSendResponse(&nRpcMsg); + } + } + } + + taosFreeQall(qall); +} + +int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + + if (strcmp(meterId, "michael") == 0) { + *spi = 1; + *encrypt = 0; + strcpy(secret, "mypassword"); + strcpy(ckey, "key"); + } else if (strcmp(meterId, "jeff") == 0) { + *spi = 0; + *encrypt = 0; + } else { + ret = -1; // user not there + } + + return ret; +} + +void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(qhandle, pTemp); +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + char dataName[20] = "server.data"; + + taosBlockSIGPIPE(); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 7000; + rpcInit.label = "SER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + rpcInit.localPort = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { + commit = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + dDebugFlag = rpcDebugFlag; + uDebugFlag = rpcDebugFlag; + } else { + printf("\nusage: %s [options] \n", argv[0]); + printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); + printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + tsAsyncLog = 0; + rpcInit.connType = TAOS_CONN_SERVER; + taosInitLog("server.log", 100000, 10); + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to start RPC server"); + return -1; + } + // sleep(5); + + tInfo("RPC server is running, ctrl-c to exit"); + + if (commit) { + dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + } + qhandle = taosOpenQueue(); + qset = taosOpenQset(); + taosAddIntoQset(qset, qhandle, NULL); + + processShellMsg(); + + if (dataFd >= 0) { + close(dataFd); + remove(dataName); + } + + return 0; +} diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c index 2280d0b922..796aab611b 100644 --- a/source/nodes/src/nodesTraverseFuncs.c +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -20,18 +20,22 @@ typedef enum ETraversalOrder { TRAVERSAL_POSTORDER } ETraversalOrder; -static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); -static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { +static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { if (NULL == pNode) { - return true; + return DEAL_RES_CONTINUE; } - if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) { - return false; + EDealRes res = DEAL_RES_CONTINUE; + + if (TRAVERSAL_PREORDER == order) { + res = walker(pNode, pContext); + if (DEAL_RES_CONTINUE != res) { + return res; + } } - bool res = true; switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: case QUERY_NODE_VALUE: @@ -41,7 +45,7 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; res = walkNode(pOpNode->pLeft, order, walker, pContext); - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pOpNode->pRight, order, walker, pContext); } break; @@ -61,10 +65,10 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke case QUERY_NODE_JOIN_TABLE: { SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pJoinTableNode->pRight, order, walker, pContext); } - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); } break; @@ -84,13 +88,13 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke case QUERY_NODE_INTERVAL_WINDOW: { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; res = walkNode(pInterval->pInterval, order, walker, pContext); - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pInterval->pOffset, order, walker, pContext); } - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pInterval->pSliding, order, walker, pContext); } - if (res) { + if (DEAL_RES_ERROR != res) { res = walkNode(pInterval->pFill, order, walker, pContext); } break; @@ -108,21 +112,21 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke break; } - if (res && TRAVERSAL_POSTORDER == order) { + if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { res = walker(pNode, pContext); } return res; } -static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { SNode* node; FOREACH(node, pNodeList) { - if (!walkNode(node, order, walker, pContext)) { - return false; + if (DEAL_RES_ERROR == walkNode(node, order, walker, pContext)) { + return DEAL_RES_ERROR; } } - return true; + return DEAL_RES_CONTINUE; } void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { @@ -140,7 +144,3 @@ void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContex void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } - -bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - -} diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index 155eb6effe..e0e589157c 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -76,7 +76,7 @@ SNode* nodesMakeNode(ENodeType type) { return NULL; } -static bool destroyNode(SNode* pNode, void* pContext) { +static EDealRes destroyNode(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_VALUE: tfree(((SValueNode*)pNode)->literal); @@ -85,6 +85,7 @@ static bool destroyNode(SNode* pNode, void* pContext) { break; } tfree(pNode); + return DEAL_RES_CONTINUE; } void nodesDestroyNode(SNode* pNode) { @@ -116,9 +117,23 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { pList->pTail->pNext = p; } pList->pTail = p; + ++(pList->length); return pList; } +SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { + if (NULL == pCell->pPrev) { + pList->pHead = pCell->pNext; + } else { + pCell->pPrev->pNext = pCell->pNext; + pCell->pNext->pPrev = pCell->pPrev; + } + SListCell* pNext = pCell->pNext; + tfree(pCell); + --(pList->length); + return pNext; +} + SNode* nodesListGetNode(SNodeList* pList, int32_t index) { SNode* node; FOREACH(node, pList) { @@ -137,6 +152,11 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } +bool nodesIsExprNode(const SNode* pNode) { + ENodeType type = nodeType(pNode); + return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); +} + bool nodesIsArithmeticOp(const SOperatorNode* pOp) { switch (pOp->opType) { case OP_TYPE_ADD: