xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/privilege

This commit is contained in:
Shengliang Guan 2022-02-14 18:49:56 +08:00
parent a72fe7750b 11e25c9c66
commit baebd82366
49 changed files with 3610 additions and 2055 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
example/src/tmq.c
View File

@ -44,35 +44,35 @@ int32_t init_env() {
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
/*pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");*/
/*if (taos_errno(pRes) != 0) {*/
/*printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));*/
/*return -1;*/
/*}*/
taos_free_result(pRes);
/*taos_free_result(pRes);*/
pRes = taos_query(pConn, "create table tu using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
/*pRes = taos_query(pConn, "create table if not exists tu using st1 tags(1)");*/
/*if (taos_errno(pRes) != 0) {*/
/*printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes));*/
/*return -1;*/
/*}*/
/*taos_free_result(pRes);*/
pRes = taos_query(pConn, "create table tu2 using st1 tags(2)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
/*pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");*/
/*if (taos_errno(pRes) != 0) {*/
/*printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));*/
/*return -1;*/
/*}*/
/*taos_free_result(pRes);*/
const char* sql = "select * from st1";
pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
/*if (taos_errno(pRes) != 0) {*/
/*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/
/*return -1;*/
/*}*/
/*taos_free_result(pRes);*/
if (taos_errno(pRes) != 0) {
printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
taos_close(pConn);
return 0;
}
@ -116,7 +116,7 @@ void basic_consume_loop(tmq_t *tmq,
int32_t cnt = 0;
/*clock_t startTime = clock();*/
while (running) {
tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0);
tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500);
if (tmqmessage) {
cnt++;
msg_process(tmqmessage);

94
include/common/tmsg.h
View File

@ -1739,11 +1739,10 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
return buf;
}
typedef struct SMqSetCVgReq {
typedef struct {
int64_t leftForVer;
int32_t vgId;
int64_t oldConsumerId;
int64_t newConsumerId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN];
char* sql;
@ -1752,59 +1751,78 @@ typedef struct SMqSetCVgReq {
char* qmsg;
} SMqSetCVgReq;
static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) {
int32_t tlen = 0;
tlen += taosEncodeFixedU64(buf, pMsg->sId);
tlen += taosEncodeFixedU64(buf, pMsg->queryId);
tlen += taosEncodeFixedU64(buf, pMsg->taskId);
tlen += taosEncodeFixedU32(buf, pMsg->sqlLen);
tlen += taosEncodeFixedU32(buf, pMsg->phyLen);
//tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen);
return tlen;
}
static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) {
buf = taosDecodeFixedU64(buf, &pMsg->sId);
buf = taosDecodeFixedU64(buf, &pMsg->queryId);
buf = taosDecodeFixedU64(buf, &pMsg->taskId);
buf = taosDecodeFixedU32(buf, &pMsg->sqlLen);
buf = taosDecodeFixedU32(buf, &pMsg->phyLen);
//buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen);
return buf;
}
static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->topicName);
tlen += taosEncodeString(buf, pReq->cgroup);
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->logicalPlan);
tlen += taosEncodeString(buf, pReq->physicalPlan);
tlen += taosEncodeString(buf, pReq->qmsg);
//tlen += tEncodeSSubQueryMsg(buf, &pReq->msg);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->topicName);
buf = taosDecodeStringTo(buf, pReq->cgroup);
buf = taosDecodeString(buf, &pReq->sql);
buf = taosDecodeString(buf, &pReq->logicalPlan);
buf = taosDecodeString(buf, &pReq->physicalPlan);
buf = taosDecodeString(buf, &pReq->qmsg);
return buf;
}
typedef struct {
int64_t leftForVer;
int32_t vgId;
int64_t oldConsumerId;
int64_t newConsumerId;
//char topicName[TSDB_TOPIC_FNAME_LEN];
//char cgroup[TSDB_CONSUMER_GROUP_LEN];
//char* sql;
//char* logicalPlan;
//char* physicalPlan;
//char* qmsg;
} SMqMVRebReq;
static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
//tlen += taosEncodeString(buf, pReq->topicName);
//tlen += taosEncodeString(buf, pReq->cgroup);
//tlen += taosEncodeString(buf, pReq->sql);
//tlen += taosEncodeString(buf, pReq->logicalPlan);
//tlen += taosEncodeString(buf, pReq->physicalPlan);
//tlen += taosEncodeString(buf, pReq->qmsg);
//tlen += tEncodeSSubQueryMsg(buf, &pReq->msg);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
//buf = taosDecodeStringTo(buf, pReq->topicName);
//buf = taosDecodeStringTo(buf, pReq->cgroup);
//buf = taosDecodeString(buf, &pReq->sql);
//buf = taosDecodeString(buf, &pReq->logicalPlan);
//buf = taosDecodeString(buf, &pReq->physicalPlan);
//buf = taosDecodeString(buf, &pReq->qmsg);
//buf = tDecodeSSubQueryMsg(buf, &pReq->msg);
return buf;
}
typedef struct SMqSetCVgRsp {
typedef struct {
SMsgHead header;
int32_t vgId;
int64_t consumerId;
@ -1812,6 +1830,14 @@ typedef struct SMqSetCVgRsp {
char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqSetCVgRsp;
typedef struct {
SMsgHead header;
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqMVRebRsp;
typedef struct {
uint32_t nCols;
SSchema *pSchema;
@ -1912,7 +1938,7 @@ typedef struct {
typedef struct {
int64_t consumerId;
int64_t epoch;
int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp;
@ -1969,7 +1995,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeFixedI64(buf, pRsp->epoch);
tlen += taosEncodeFixedI32(buf, pRsp->epoch);
tlen += taosEncodeString(buf, pRsp->cgroup);
int32_t sz = taosArrayGetSize(pRsp->topics);
tlen += taosEncodeFixedI32(buf, sz);
@ -1982,7 +2008,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu
static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI64(buf, &pRsp->epoch);
buf = taosDecodeFixedI32(buf, &pRsp->epoch);
buf = taosDecodeStringTo(buf, pRsp->cgroup);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
@ -2002,4 +2028,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
}
#endif
#endif /*_TD_COMMON_TAOS_MSG_H_*/
#endif /*_TD_COMMON_TAOS_MSG_H_*/

1
include/common/tmsgdef.h
View File

@ -164,6 +164,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)

43
include/libs/catalog/catalog.h
View File

@ -30,7 +30,7 @@ extern "C" {
#include "tmsg.h"
#include "transport.h"
struct SCatalog;
typedef struct SCatalog SCatalog;
enum {
CTG_DBG_DB_NUM = 1,
@ -64,6 +64,7 @@ typedef struct SCatalogCfg {
typedef struct SSTableMetaVersion {
char dbFName[TSDB_DB_FNAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t suid;
int16_t sversion;
int16_t tversion;
@ -84,7 +85,7 @@ int32_t catalogInit(SCatalogCfg *cfg);
* @param catalogHandle (output, NO need to free it)
* @return error code
*/
int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle);
int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
/**
* Free a cluster's all catalog info, usually it's not necessary, until the application is closing.
@ -92,9 +93,9 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle);
* @param pCatalog (input, NO more usage)
* @return error code
*/
void catalogFreeHandle(struct SCatalog* pCatalog);
void catalogFreeHandle(SCatalog* pCatalog);
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version);
/**
* Get a DB's all vgroup info.
@ -106,13 +107,13 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName,
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code
*/
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo);
int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo);
int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId);
int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid);
int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
/**
* Get a table's meta data.
@ -123,7 +124,7 @@ int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, c
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
/**
* Get a super table's meta data.
@ -134,13 +135,13 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg);
int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
/**
* Force renew a table's local cached meta data.
* Force refresh a table's local cached meta data.
* @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
@ -148,10 +149,10 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code
*/
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable);
int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable);
/**
* Force renew a table's local cached meta data and get the new one.
* Force refresh a table's local cached meta data and get the new one.
* @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
@ -160,7 +161,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code
*/
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable);
int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable);
@ -173,7 +174,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code
*/
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList);
int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList);
/**
* Get a table's vgroup from its name's hash value.
@ -184,7 +185,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter,
* @param vgInfo (output, vgroup info)
* @return error code
*/
int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo);
int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo);
/**
@ -196,14 +197,14 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter
* @param pRsp (output, response data)
* @return error code
*/
int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);
int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);
int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
/**

6
include/libs/qcom/query.h
View File

@ -80,16 +80,16 @@ typedef struct STableMeta {
SSchema schema[];
} STableMeta;
typedef struct SDBVgroupInfo {
typedef struct SDBVgInfo {
int32_t vgVersion;
int8_t hashMethod;
SHashObj *vgHash; //key:vgId, value:SVgroupInfo
} SDBVgroupInfo;
} SDBVgInfo;
typedef struct SUseDbOutput {
char db[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SDBVgroupInfo *dbVgroup;
SDBVgInfo *dbVgroup;
} SUseDbOutput;
enum {

134
include/libs/sync/sync.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
@ -23,7 +23,7 @@ extern "C" {
#include <stdint.h>
#include "taosdef.h"
typedef int32_t SyncNodeId;
typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId;
typedef int64_t SyncIndex;
typedef uint64_t SyncTerm;
@ -46,109 +46,113 @@ typedef struct {
} SNodeInfo;
typedef struct {
int32_t selfIndex;
int32_t replica;
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
} SSyncCluster;
} SSyncCfg;
typedef struct {
int32_t selfIndex;
int32_t replica;
SNodeInfo node[TSDB_MAX_REPLICA];
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
ESyncState role[TSDB_MAX_REPLICA];
} SNodesRole;
// abstract definition of snapshot
typedef struct SSnapshot {
void* data;
SyncIndex lastApplyIndex;
} SSnapshot;
typedef struct SSyncFSM {
void* pData;
void* data;
// apply committed log, bufs will be free by sync module
int32_t (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData);
// when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
// cluster commit callback
int32_t (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData);
// when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
// fsm return snapshot in ppBuf, bufs will be free by sync module
// TODO: getSnapshot SHOULD be async?
int32_t (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int32_t* objId, bool* isLast);
// when log entry is updated by a new one, FpRollBackCb is called
// user can do something to roll back. for example, delete data from tsdb, or just ignore it
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
// fsm apply snapshot with pBuf data
int32_t (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int32_t objId, bool isLast);
// user should implement this function, use "data" to take snapshot into "snapshot"
int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
// call when restore snapshot and log done
int32_t (*onRestoreDone)(struct SSyncFSM* fsm);
void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf);
void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole);
// user should implement this function, restore "data" from "snapshot"
int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
} SSyncFSM;
// abstract definition of log store in raft
// SWal implements it
typedef struct SSyncLogStore {
void* pData;
void* data;
// write log with given index
int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf);
// append one log entry
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf);
/**
* read log from given index(included) with limit, return the actual num in nBuf,
* pBuf will be free in sync module
**/
int32_t (*logRead)(struct SSyncLogStore* logStore, SyncIndex index, int limit,
SSyncBuffer* pBuf, int* nBuf);
// get one log entry, user need to free pBuf->data
int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf);
// mark log with given index has been commtted
int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index);
// update log store commit index with "index"
int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
// prune log before given index(not included)
int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index);
// truncate log with index, entries after the given index (>index) will be deleted
int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index);
// rollback log after given index(included)
int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index);
// return commit index of log
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
// return index of last entry
SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore);
// return term of last entry
SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore);
// return last index of log
SyncIndex (*logLastIndex)(struct SSyncLogStore* logStore);
} SSyncLogStore;
typedef struct SStateManager {
void* pData;
// raft need to persist two variables in storage: currentTerm, voteFor
typedef struct SStateMgr {
void* data;
// save serialized server state data, buffer will be free by Sync
int32_t (*saveServerState)(struct SStateManager* stateMng, const char* buffer, int n);
int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
// read serialized server state data, buffer will be free by Sync
int32_t (*readServerState)(struct SStateManager* stateMng, char** ppBuffer, int* n);
int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
// save serialized cluster state data, buffer will be free by Sync
void (*saveClusterState)(struct SStateManager* stateMng, const char* buffer, int n);
int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
// read serialized cluster state data, buffer will be free by Sync
int32_t (*readClusterState)(struct SStateManager* stateMng, char** ppBuffer, int* n);
} SStateManager;
} SStateMgr;
typedef struct {
SyncGroupId vgId;
SyncIndex appliedIndex;
SSyncCluster syncCfg;
SSyncFSM fsm;
SSyncCfg syncCfg;
SSyncLogStore logStore;
SStateManager stateManager;
SStateMgr stateManager;
SSyncFSM syncFsm;
} SSyncInfo;
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
// will be defined in syncInt.h, here just for complie
typedef struct SSyncNode {
} SSyncNode;
int32_t syncInit();
void syncCleanUp();
SSyncNode* syncStart(const SSyncInfo*);
void syncReconfig(const SSyncNode*, const SSyncCluster*);
void syncStop(const SSyncNode*);
int64_t syncStart(const SSyncInfo*);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg*);
int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak);
// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode);
int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode);
ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole*);
extern int32_t sDebugFlag;

23
include/nodes/nodes.h
View File

@ -31,16 +31,7 @@ extern "C" {
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
// only be use in FOREACH
#define ERASE_NODE(list) \
if (NULL == cell->pPrev) { \
(list)->pHead = cell->pNext; \
} else { \
cell->pPrev->pNext = cell->pNext; \
cell->pNext->pPrev = cell->pPrev; \
} \
SListCell* tmp = cell; \
cell = cell->pNext; \
tfree(tmp);
#define ERASE_NODE(list) cell = nodesListErase(list, cell);
#define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
@ -343,18 +334,22 @@ void nodesDestroyNode(SNode* pNode);
SNodeList* nodesMakeList();
SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
SNode* nodesListGetNode(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList);
typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext);
typedef enum EDealRes {
DEAL_RES_CONTINUE = 1,
DEAL_RES_IGNORE_CHILD,
DEAL_RES_ERROR,
} EDealRes;
typedef EDealRes (*FQueryNodeWalker)(SNode* pNode, void* pContext);
void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext);
void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext);
void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext);
void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext);
bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext);
bool nodesEqualNode(const SNode* a, const SNode* b);
void nodesCloneNode(const SNode* pNode);
@ -362,6 +357,8 @@ void nodesCloneNode(const SNode* pNode);
int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen);
int32_t nodesStringToNode(const char* pStr, SNode** pNode);
bool nodesIsExprNode(const SNode* pNode);
bool nodesIsArithmeticOp(const SOperatorNode* pOp);
bool nodesIsComparisonOp(const SOperatorNode* pOp);
bool nodesIsJsonOp(const SOperatorNode* pOp);

9
include/util/taoserror.h
View File

@ -247,7 +247,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0)
#define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1)
// mnode-topic
// mnode-mq
#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0)
#define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1)
#define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2)
@ -256,7 +256,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5)
#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6)
#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7)
#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7)
#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8)
#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9)
#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0)
// dnode
#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)
@ -454,6 +456,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) //Inconsistent datatypes
#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation
#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression
#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression
#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression
#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function
#ifdef __cplusplus
}

14
source/client/src/clientHb.c
View File

@ -44,7 +44,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
if (rsp->vgVersion < 0) {
code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
} else {
SDBVgroupInfo vgInfo = {0};
SDBVgInfo vgInfo = {0};
vgInfo.vgVersion = rsp->vgVersion;
vgInfo.hashMethod = rsp->hashMethod;
vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
@ -60,12 +60,9 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
taosHashCleanup(vgInfo.vgHash);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
}
code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo);
if (code) {
taosHashCleanup(vgInfo.vgHash);
}
}
catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo);
}
if (code) {
@ -86,13 +83,14 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
rsp->numOfColumns = ntohl(rsp->numOfColumns);
rsp->suid = be64toh(rsp->suid);
rsp->dbId = be64toh(rsp->dbId);
if (rsp->numOfColumns < 0) {
schemaNum = 0;
tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid);
catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid);
} else {
tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);

144
source/client/src/tmq.c
View File

@ -58,7 +58,7 @@ struct tmq_t {
char clientId[256];
SRWLatch lock;
int64_t consumerId;
int64_t epoch;
int32_t epoch;
int64_t status;
tsem_t rspSem;
STscObj* pTscObj;
@ -113,10 +113,10 @@ typedef struct SMqConsumeCbParam {
} SMqConsumeCbParam;
typedef struct SMqCommitCbParam {
tmq_t* tmq;
SMqClientVg* pVg;
int32_t async;
tsem_t rspSem;
tmq_t* tmq;
SMqClientVg* pVg;
int32_t async;
tsem_t rspSem;
} SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() {
@ -163,8 +163,8 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
}
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*) param;
tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
if (pParam->tmq->commit_cb) {
pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
}
@ -221,13 +221,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tNameExtractFullName(&name, topicFname);
tscDebug("subscribe topic: %s", topicFname);
SMqClientTopic topic = {
.nextVgIdx = 0,
.sql = NULL,
.sqlLen = 0,
.topicId = 0,
.topicName = topicFname,
.vgs = NULL
};
.nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
/*SMqClientTopic topic = {*/
@ -461,10 +455,10 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
void tmqShowMsg(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
static bool noPrintSchema;
char pBuf[128];
static bool noPrintSchema;
char pBuf[128];
SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message;
int32_t colNum = pRsp->schemas->nCols;
int32_t colNum = pRsp->schemas->nCols;
if (!noPrintSchema) {
printf("|");
for (int32_t i = 0; i < colNum; i++) {
@ -506,7 +500,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param;
SMqClientVg* pVg = pParam->pVg;
if (code != 0) {
/*printf("msg discard\n");*/
printf("msg discard\n");
tsem_post(&pParam->rspSem);
return 0;
}
@ -517,7 +511,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
return -1;
}
tDecodeSMqConsumeRsp(pMsg->pData, pRsp);
/*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->numOfTopics == 0) {
/*printf("no data\n");*/
free(pRsp);
@ -584,7 +578,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
return 0;
}
int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) {
int32_t tmqAskEp(tmq_t* tmq, bool wait) {
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen);
if (buf == NULL) {
@ -592,6 +586,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) {
goto END;
}
buf->consumerId = htobe64(tmq->consumerId);
buf->epoch = htonl(tmq->epoch);
strcpy(buf->cgroup, tmq->groupId);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
@ -602,7 +597,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) {
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam));
SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam));
if (pParam == NULL) {
tscError("failed to malloc subscribe param");
goto END;
@ -652,7 +647,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* tmq_message = NULL;
int64_t status = atomic_load_64(&tmq->status);
tmqAsyncAskEp(tmq, status == 0);
tmqAskEp(tmq, status == 0);
if (blocking_time < 0) blocking_time = 1;
if (blocking_time > 1000) blocking_time = 1000;
@ -670,48 +665,58 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
}
tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs));
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg);
if (pReq == NULL) {
usleep(blocking_time * 1000);
return NULL;
int32_t beginVgIdx = pTopic->nextVgIdx;
while (1) {
pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
/*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg);
if (pReq == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
return NULL;
}
SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam));
if (param == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
return NULL;
}
param->tmq = tmq;
param->retMsg = &tmq_message;
param->pVg = pVg;
tsem_init(&param->rspSem, 0, 0);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = param;
sendInfo->fp = tmqPollCb;
/*printf("req offset: %ld\n", pReq->offset);*/
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tmq->pollCnt++;
tsem_wait(&param->rspSem);
tsem_destroy(&param->rspSem);
free(param);
if (tmq_message == NULL) {
if (beginVgIdx == pTopic->nextVgIdx) {
usleep(blocking_time * 1000);
} else {
continue;
}
}
return tmq_message;
}
SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam));
if (param == NULL) {
usleep(blocking_time * 1000);
return NULL;
}
param->tmq = tmq;
param->retMsg = &tmq_message;
param->pVg = pVg;
tsem_init(&param->rspSem, 0, 0);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = param;
sendInfo->fp = tmqPollCb;
/*printf("req offset: %ld\n", pReq->offset);*/
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tmq->pollCnt++;
tsem_wait(&param->rspSem);
tsem_destroy(&param->rspSem);
free(param);
if (tmq_message == NULL) {
usleep(blocking_time * 1000);
}
return tmq_message;
/*tsem_wait(&pRequest->body.rspSem);*/
/*if (body != NULL) {*/
@ -726,21 +731,20 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
}
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
if (tmq_topic_vgroup_list != NULL) {
//TODO
// TODO
}
//TODO: change semaphore to gate
// TODO: change semaphore to gate
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam));
SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam));
if (pParam == NULL) {
continue;
}
@ -771,9 +775,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) {
free(tmq_message);
}
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
return TMQ_RESP_ERR__SUCCESS;
}
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
const char* tmq_err2str(tmq_resp_err_t err) {
if (err == TMQ_RESP_ERR__SUCCESS) {

2
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -116,6 +116,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
/*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg;
// Requests handled by VNODE
@ -149,6 +150,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
}

3
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
pMgmt->openVnodes, pMgmt->totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId};
SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
SVnode * pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
@ -594,6 +594,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
vnodeCfg.pDnode = pDnode;
vnodeCfg.pTfs = pDnode->pTfs;
vnodeCfg.dbId = wrapperCfg.dbUid;
SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
if (pImpl == NULL) {
dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr());

37
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -424,6 +424,7 @@ typedef struct {
int32_t status;
int32_t vgNum;
SArray* consumers; // SArray<SMqSubConsumer>
SArray* lostConsumers; // SArray<SMqSubConsumer>
SArray* unassignedVg; // SArray<SMqConsumerEp>
} SMqSubscribeObj;
@ -432,10 +433,17 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
if (pSub == NULL) {
return NULL;
}
pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer));
if (pSub->consumers == NULL) {
goto _err;
}
pSub->lostConsumers = taosArrayInit(0, sizeof(SMqSubConsumer));
if (pSub->lostConsumers == NULL) {
goto _err;
}
pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->unassignedVg == NULL) {
goto _err;
@ -448,8 +456,9 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
return pSub;
_err:
tfree(pSub->unassignedVg);
tfree(pSub->consumers);
tfree(pSub->lostConsumers);
tfree(pSub->unassignedVg);
tfree(pSub);
return NULL;
}
@ -468,6 +477,13 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb
tlen += tEncodeSMqSubConsumer(buf, pSubConsumer);
}
sz = taosArrayGetSize(pSub->lostConsumers);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->lostConsumers, i);
tlen += tEncodeSMqSubConsumer(buf, pSubConsumer);
}
sz = taosArrayGetSize(pSub->unassignedVg);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
@ -496,6 +512,17 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
taosArrayPush(pSub->consumers, &subConsumer);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->lostConsumers = taosArrayInit(sz, sizeof(SMqSubConsumer));
if (pSub->lostConsumers == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqSubConsumer subConsumer = {0};
buf = tDecodeSMqSubConsumer(buf, &subConsumer);
taosArrayPush(pSub->lostConsumers, &subConsumer);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedVg == NULL) {
@ -545,7 +572,7 @@ typedef struct {
char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray* currentTopics; // SArray<char*>
SArray* recentRemovedTopics; // SArray<char*>
int64_t epoch;
int32_t epoch;
// stat
int64_t pollCnt;
// status
@ -561,8 +588,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
tlen += taosEncodeFixedI64(buf, pConsumer->connId);
tlen += taosEncodeFixedI64(buf, pConsumer->epoch);
tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt);
tlen += taosEncodeFixedI32(buf, pConsumer->status);
tlen += taosEncodeString(buf, pConsumer->cgroup);
sz = taosArrayGetSize(pConsumer->currentTopics);
@ -585,8 +613,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
int32_t sz;
buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
buf = taosDecodeFixedI64(buf, &pConsumer->connId);
buf = taosDecodeFixedI64(buf, &pConsumer->epoch);
buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt);
buf = taosDecodeFixedI32(buf, &pConsumer->status);
buf = taosDecodeStringTo(buf, pConsumer->cgroup);
buf = taosDecodeFixedI32(buf, &sz);

5
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -59,9 +59,10 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pConsumer->recentRemovedTopics = taosArrayInit(0, sizeof(char*));
pConsumer->epoch = 1;
pConsumer->consumerId = consumerId;
pConsumer->status = MQ_CONSUMER_STATUS__INIT;
atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT);
strcpy(pConsumer->cgroup, cgroup);
taosInitRWLatch(&pConsumer->lock);
return pConsumer;
@ -169,7 +170,7 @@ SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) {
SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (pConsumer == NULL) {
/*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/
terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
}
return pConsumer;
}

2
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1523,4 +1523,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3
static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}
}

170
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -55,7 +55,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg);
static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg);
static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
const SMqConsumerEp *pSub);
const SMqConsumerEp *pConsumerEp);
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
@ -72,6 +72,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
@ -105,13 +106,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
}
static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) {
SMqSetCVgReq req = {
.vgId = pConsumerEp->vgId,
.oldConsumerId = pConsumerEp->oldConsumerId,
.newConsumerId = pConsumerEp->consumerId,
SMqMVRebReq req = {
.vgId = pConsumerEp->vgId,
.oldConsumerId = pConsumerEp->oldConsumerId,
.newConsumerId = pConsumerEp->consumerId,
};
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
int32_t tlen = tEncodeSMqMVRebReq(NULL, &req);
void *buf = malloc(sizeof(SMsgHead) + tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -123,7 +124,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
pMsgHead->vgId = htonl(pConsumerEp->vgId);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncodeSMqSetCVgReq(&abuf, &req);
tEncodeSMqMVRebReq(&abuf, &req);
*pBuf = buf;
*pLen = tlen;
@ -132,6 +133,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
}
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) {
ASSERT(pConsumerEp->oldConsumerId != -1);
int32_t vgId = pConsumerEp->vgId;
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
@ -145,7 +147,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + tlen;
action.msgType = TDMT_VND_MQ_SET_CONN;
action.msgType = TDMT_VND_MQ_REB;
mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -158,8 +160,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC
static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) {
SMqSetCVgReq req = {0};
req.oldConsumerId = pConsumerEp->consumerId;
req.newConsumerId = -1;
req.consumerId = pConsumerEp->consumerId;
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
void *buf = malloc(sizeof(SMsgHead) + tlen);
@ -208,6 +209,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
SMqCMGetSubEpRsp rsp = {0};
int64_t consumerId = be64toh(pReq->consumerId);
int32_t epoch = ntohl(pReq->epoch);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId);
if (pConsumer == NULL) {
@ -216,10 +218,19 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
}
ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
// TODO
int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);
mTrace("try to get sub ep, old val: %d", hbStatus);
atomic_store_32(&pConsumer->hbStatus, 0);
/*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/
/*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/
/*sdbWrite(pMnode->pSdb, pConsumerRaw);*/
strcpy(rsp.cgroup, pReq->cgroup);
rsp.consumerId = consumerId;
rsp.epoch = pConsumer->epoch;
if (pReq->epoch != rsp.epoch) {
if (epoch != rsp.epoch) {
mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch);
SArray *pTopics = pConsumer->currentTopics;
int sz = taosArrayGetSize(pTopics);
rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
@ -258,6 +269,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
void *abuf = buf;
tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
tDeleteSMqCMGetSubEpRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer);
pMsg->pCont = buf;
pMsg->contLen = tlen;
return 0;
@ -299,7 +311,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
while (1) {
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) break;
int32_t hbStatus = atomic_fetch_add_32(&pConsumer->hbStatus, 1);
int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) {
int32_t old =
atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST);
@ -312,10 +324,6 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId);
}
/*pRebMsg->consumerId = pConsumer->consumerId;*/
/*SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen =
* sizeof(SMqDoRebalanceMsg)};*/
/*pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);*/
}
}
int32_t status = atomic_load_32(&pConsumer->status);
@ -373,9 +381,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
mInfo("mq remove lost consumer %ld", lostConsumerId);
for (int j = 0; j < taosArrayGetSize(pSub->consumers); j++) {
SMqConsumerEp *pConsumerEp = taosArrayGet(pSub->consumers, j);
if (pConsumerEp->consumerId == lostConsumerId) {
taosArrayPush(pSub->unassignedVg, pConsumerEp);
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
if (pSubConsumer->consumerId == lostConsumerId) {
taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo);
taosArrayPush(pSub->lostConsumers, pSubConsumer);
taosArrayRemove(pSub->consumers, j);
break;
}
@ -389,73 +398,90 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
int32_t vgEachConsumer = vgNum / consumerNum;
int32_t imbalanceVg = vgNum % consumerNum;
int32_t imbalanceSolved = 0;
SArray *unassignedVgStash = taosArrayInit(0, sizeof(SMqConsumerEp));
SArray *unassignedConsumerIdx = taosArrayInit(0, sizeof(int32_t));
// iterate all consumers, set unassignedVgStash
for (int i = 0; i < consumerNum; i++) {
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int vgThisConsumerAfterRb;
if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1;
else vgThisConsumerAfterRb = vgEachConsumer;
int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int vgThisConsumerAfterRb;
if (i < imbalanceVg)
vgThisConsumerAfterRb = vgEachConsumer + 1;
else
vgThisConsumerAfterRb = vgEachConsumer;
mInfo("mq consumer:%ld ,connectted vgroup change from %d %d", pSubConsumer->consumerId, vgThisConsumerBeforeRb, vgThisConsumerAfterRb);
mInfo("mq consumer:%ld, connectted vgroup number change from %d to %d", pSubConsumer->consumerId,
vgThisConsumerBeforeRb, vgThisConsumerAfterRb);
while(taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
ASSERT(pConsumerEp != NULL);
ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
taosArrayPush(unassignedVgStash, pConsumerEp);
taosArrayPush(pSub->unassignedVg, pConsumerEp);
}
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
int32_t status = atomic_load_32(&pRebConsumer->status);
if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
(vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
(vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)
) {
pRebConsumer->epoch++;
if (vgThisConsumerAfterRb != 0) {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
} else {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
}
mInfo("mq consumer:%ld , status change from %d %d", pRebConsumer->consumerId, status, pRebConsumer->status);
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
int32_t status = atomic_load_32(&pRebConsumer->status);
if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
(vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
(vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) {
pRebConsumer->epoch++;
if (vgThisConsumerAfterRb != 0) {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
} else {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
}
mndReleaseConsumer(pMnode, pRebConsumer);
mInfo("mq consumer:%ld, status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status);
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
}
mndReleaseConsumer(pMnode, pRebConsumer);
}
//assign to vgroup
if (taosArrayGetSize(unassignedVgStash) != 0) {
// assign to vgroup
if (taosArrayGetSize(pSub->unassignedVg) != 0) {
for (int i = 0; i < consumerNum; i++) {
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int vgThisConsumerAfterRb;
if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1;
else vgThisConsumerAfterRb = vgEachConsumer;
int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int vgThisConsumerAfterRb;
if (i < imbalanceVg)
vgThisConsumerAfterRb = vgEachConsumer + 1;
else
vgThisConsumerAfterRb = vgEachConsumer;
while(taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerBeforeRb) {
SMqConsumerEp* pConsumerEp = taosArrayPop(unassignedVgStash);
while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg);
ASSERT(pConsumerEp != NULL);
ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
mInfo("mq consumer:%ld , assign vgroup %d, previously assigned to consumer %ld", pSubConsumer->consumerId, pConsumerEp->vgId, pConsumerEp->oldConsumerId);
if (pConsumerEp->oldConsumerId == -1) {
char *topic;
char *cgroup;
mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic,
pConsumerEp->consumerId);
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
mndReleaseTopic(pMnode, pTopic);
free(topic);
free(cgroup);
} else {
mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId,
pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
}
}
}
}
ASSERT(taosArrayGetSize(unassignedVgStash) == 0);
ASSERT(taosArrayGetSize(pSub->unassignedVg) == 0);
// TODO: log rebalance statistics
SSdbRaw *pSubRaw = mndSubActionEncode(pSub);
@ -466,10 +492,12 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
taosHashCleanup(pReq->rebSubHash);
mndTransDrop(pTrans);
return -1;
}
taosHashCleanup(pReq->rebSubHash);
mndTransDrop(pTrans);
return 0;
}
@ -695,11 +723,6 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub
mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
return -1;
}
/*if (pArray && taosArrayGetSize(pArray) != 1) {*/
/*terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;*/
/*mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray));*/
/*return -1;*/
/*}*/
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
@ -721,13 +744,13 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub
static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
const SMqConsumerEp *pConsumerEp) {
ASSERT(pConsumerEp->oldConsumerId == -1);
int32_t vgId = pConsumerEp->vgId;
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
SMqSetCVgReq req = {
.vgId = vgId,
.oldConsumerId = pConsumerEp->oldConsumerId,
.newConsumerId = pConsumerEp->consumerId,
.consumerId = pConsumerEp->consumerId,
.sql = pTopic->sql,
.logicalPlan = pTopic->logicalPlan,
.physicalPlan = pTopic->physicalPlan,
@ -880,7 +903,7 @@ SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const c
SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key);
free(key);
if (pSub == NULL) {
/*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/
terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
return pSub;
}
@ -889,7 +912,7 @@ SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key) {
SSdb *pSdb = pMnode->pSdb;
SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key);
if (pSub == NULL) {
/*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/
terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
return pSub;
}
@ -1018,8 +1041,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = consumerId;
taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp);
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
atomic_store_32(&pConsumer->hbStatus, MQ_CONSUMER_STATUS__ACTIVE);
if (pConsumerEp->oldConsumerId == -1) {
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
} else {
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
}
// do not set status active to trigger rebalance
/*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/
}
SSdbRaw *pRaw = mndSubActionEncode(pSub);

2
source/dnode/mnode/impl/src/mnode.c
View File

@ -389,7 +389,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
return NULL;
}
if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER) {
if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) {
SRpcConnInfo connInfo = {0};
if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
taosFreeQitem(pMsg);

148
source/dnode/vnode/inc/tq.h
View File

@ -35,155 +35,14 @@
extern "C" {
#endif
#define TQ_BUFFER_SIZE 8
typedef struct STqRspHandle {
void* handle;
void* ahandle;
} STqRspHandle;
typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
typedef struct STqTaskItem {
int8_t status;
int64_t offset;
void* dst;
qTaskInfo_t task;
STqReadHandle* pReadHandle;
SSubQueryMsg* pQueryMsg;
} STqTaskItem;
// new version
typedef struct STqBuffer {
int64_t firstOffset;
int64_t lastOffset;
STqTaskItem output[TQ_BUFFER_SIZE];
} STqBuffer;
typedef struct STqTopicHandle {
char topicName[TSDB_TOPIC_FNAME_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
int64_t committedOffset;
int64_t currentOffset;
STqBuffer buffer;
SWalReadHandle* pReadhandle;
} STqTopicHandle;
typedef struct STqConsumerHandle {
int64_t consumerId;
int64_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqClientTopic>
} STqConsumerHandle;
typedef struct STQ STQ;
// memory allocator provided by vnode
typedef struct STqMemRef {
SMemAllocatorFactory* pAllocatorFactory;
SMemAllocator* pAllocator;
} STqMemRef;
typedef struct STqSerializedHead {
int16_t ver;
int16_t action;
int32_t checksum;
int64_t ssize;
char content[];
} STqSerializedHead;
typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
typedef void (*FTqDelete)(void*);
#define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096
// key + offset + size
#define TQ_IDX_SIZE 24
// 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170
// 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080
// 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3
#define TQ_SVER 0
// TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct STqMetaHandle {
int64_t key;
int64_t offset;
int64_t serializedSize;
void* valueInUse;
void* valueInTxn;
} STqMetaHandle;
typedef struct STqMetaList {
STqMetaHandle handle;
struct STqMetaList* next;
// struct STqMetaList* inTxnPrev;
// struct STqMetaList* inTxnNext;
struct STqMetaList* unpersistPrev;
struct STqMetaList* unpersistNext;
} STqMetaList;
typedef struct STqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// topics that are not connectted
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
char* dirPath;
int32_t tqConfigFlag;
FTqSerialize pSerializer;
FTqDeserialize pDeserializer;
FTqDelete pDeleter;
} STqMetaStore;
typedef struct STQ {
// the collection of groups
// the handle of meta kvstore
char* path;
STqCfg* tqConfig;
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
SWal* pWal;
SMeta* pMeta;
} STQ;
typedef struct STqMgmt {
int8_t inited;
tmr_h timer;
} STqMgmt;
static STqMgmt tqMgmt;
// init once
int tqInit();
void tqCleanUp();
@ -192,12 +51,13 @@ void tqCleanUp();
STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac);
void tqClose(STQ*);
// void* will be replace by a msg type
// required by vnode
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
int32_t tqProcessRebReq(STQ* pTq, char* msg);
#ifdef __cplusplus
}

1
source/dnode/vnode/inc/vnode.h
View File

@ -42,6 +42,7 @@ typedef struct STqCfg {
typedef struct SVnodeCfg {
int32_t vgId;
uint64_t dbId;
SDnode *pDnode;
STfs *pTfs;
uint64_t wsize;

141
source/dnode/vnode/src/inc/tqInt.h
View File

@ -63,8 +63,145 @@ extern int32_t tqDebugFlag;
} \
}
int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**);
const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**);
#define TQ_BUFFER_SIZE 8
#define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096
// key + offset + size
#define TQ_IDX_SIZE 24
// 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170
// 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080
// 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3
#define TQ_SVER 0
// TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
typedef struct {
int16_t ver;
int16_t action;
int32_t checksum;
int64_t ssize;
char content[];
} STqSerializedHead;
typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
typedef void (*FTqDelete)(void*);
typedef struct STqMetaHandle {
int64_t key;
int64_t offset;
int64_t serializedSize;
void* valueInUse;
void* valueInTxn;
} STqMetaHandle;
typedef struct STqMetaList {
STqMetaHandle handle;
struct STqMetaList* next;
// struct STqMetaList* inTxnPrev;
// struct STqMetaList* inTxnNext;
struct STqMetaList* unpersistPrev;
struct STqMetaList* unpersistNext;
} STqMetaList;
typedef struct {
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// topics that are not connectted
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
char* dirPath;
int32_t tqConfigFlag;
FTqSerialize pSerializer;
FTqDeserialize pDeserializer;
FTqDelete pDeleter;
} STqMetaStore;
struct STQ {
// the collection of groups
// the handle of meta kvstore
char* path;
STqCfg* tqConfig;
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
SWal* pWal;
SMeta* pMeta;
};
typedef struct {
int8_t inited;
tmr_h timer;
} STqMgmt;
static STqMgmt tqMgmt;
typedef struct {
int8_t status;
int64_t offset;
qTaskInfo_t task;
STqReadHandle* pReadHandle;
} STqTaskItem;
// new version
typedef struct {
int64_t firstOffset;
int64_t lastOffset;
STqTaskItem output[TQ_BUFFER_SIZE];
} STqBuffer;
typedef struct {
char topicName[TSDB_TOPIC_FNAME_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
char* qmsg;
int64_t committedOffset;
int64_t currentOffset;
STqBuffer buffer;
SWalReadHandle* pReadhandle;
} STqTopicHandle;
typedef struct {
int64_t consumerId;
int64_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqClientTopic>
} STqConsumerHandle;
int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**);
const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**);
static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }

62
source/dnode/vnode/src/tq/tq.c
View File

@ -134,7 +134,6 @@ const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHan
return NULL;
}
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
int64_t reqId = pReq->reqId;
@ -144,6 +143,8 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL};
/*printf("vg %d get consume req\n", pReq->head.vgId);*/
STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
if (pConsumer == NULL) {
pMsg->pCont = NULL;
@ -158,6 +159,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i);
// TODO: support multiple topic in one req
if (strcmp(pTopic->topicName, pReq->topic) != 0) {
/*ASSERT(false);*/
continue;
}
@ -171,7 +173,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
}
if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) {
pTopic->committedOffset = pReq->offset-1;
pTopic->committedOffset = pReq->offset - 1;
}
rsp.committedOffset = pTopic->committedOffset;
@ -181,6 +183,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
if (fetchOffset <= pTopic->committedOffset) {
fetchOffset = pTopic->committedOffset + 1;
}
/*printf("vg %d fetch Offset %ld\n", pReq->head.vgId, fetchOffset);*/
int8_t pos;
int8_t skip = 0;
SWalHead* pHead;
@ -231,7 +234,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
break;
}
}
//TODO copy
// TODO copy
rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper;
rsp.rspOffset = fetchOffset;
@ -266,11 +269,11 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
void* abuf = buf;
tEncodeSMqConsumeRsp(&abuf, &rsp);
if (rsp.pBlockData) {
taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock);
taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
rsp.pBlockData = NULL;
/*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/
/*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/
/*tDeleteSSDataBlock(pBlock);*/
/*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/
/*tDeleteSSDataBlock(pBlock);*/
/*}*/
/*taosArrayDestroy(rsp.pBlockData);*/
}
@ -281,30 +284,36 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
return 0;
}
int32_t tqProcessRebReq(STQ* pTq, char* msg) {
SMqMVRebReq req = {0};
tDecodeSMqMVRebReq(msg, &req);
STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId);
ASSERT(pConsumer);
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
tqHandlePurge(pTq->tqMeta, req.oldConsumerId);
terrno = TSDB_CODE_SUCCESS;
return 0;
}
int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
SMqSetCVgReq req = {0};
tDecodeSMqSetCVgReq(msg, &req);
STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId);
/*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/
STqConsumerHandle* pConsumer = calloc(1, sizeof(STqConsumerHandle));
if (pConsumer == NULL) {
pConsumer = calloc(sizeof(STqConsumerHandle), 1);
if (pConsumer == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
} else {
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
tqHandlePurge(pTq->tqMeta, req.oldConsumerId);
terrno = TSDB_CODE_SUCCESS;
return 0;
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
strcpy(pConsumer->cgroup, req.cgroup);
pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle));
pConsumer->consumerId = req.newConsumerId;
pConsumer->consumerId = req.consumerId;
pConsumer->epoch = 0;
STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1);
STqTopicHandle* pTopic = calloc(1, sizeof(STqTopicHandle));
if (pTopic == NULL) {
free(pConsumer);
return -1;
@ -313,6 +322,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
pTopic->sql = req.sql;
pTopic->logicalPlan = req.logicalPlan;
pTopic->physicalPlan = req.physicalPlan;
pTopic->qmsg = req.qmsg;
pTopic->committedOffset = -1;
pTopic->currentOffset = -1;
@ -324,13 +334,13 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta };
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta};
pTopic->buffer.output[i].pReadHandle = pReadHandle;
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
}
taosArrayPush(pConsumer->topics, pTopic);
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.consumerId);
terrno = TSDB_CODE_SUCCESS;
return 0;
}
@ -416,7 +426,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
int32_t numOfCols = pHandle->pSchema->numOfCols;
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
//TODO: stable case
// TODO: stable case
if (colNumNeed > pSchemaWrapper->nCols) {
colNumNeed = pSchemaWrapper->nCols;
}
@ -432,7 +442,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
colInfo.info.bytes = pColSchema->bytes;

1
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -289,7 +289,6 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) {
return 0;
}
// TODO: wrap in tfile
int32_t tqStorePersist(STqMetaStore* pMeta) {
STqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer;

1
source/dnode/vnode/src/vnd/vnodeMain.c
View File

@ -29,6 +29,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) {
cfg.vgId = pVnodeCfg->vgId;
cfg.pDnode = pVnodeCfg->pDnode;
cfg.pTfs = pVnodeCfg->pTfs;
cfg.dbId = pVnodeCfg->dbId;
}
// Validate options

3
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -93,6 +93,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
if (pTbCfg->type == META_CHILD_TABLE) {
pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid);
if (pStbCfg == NULL) {
code = TSDB_CODE_VND_TB_NOT_EXIST;
goto _exit;
}
@ -116,9 +117,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols);
pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen);
if (pTbMetaMsg == NULL) {
code = TSDB_CODE_VND_OUT_OF_MEMORY;
goto _exit;
}
pTbMetaMsg->dbId = htobe64(pVnode->config.dbId);
memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName));
strcpy(pTbMetaMsg->tbName, pReq->tbName);
if (pTbCfg->type == META_CHILD_TABLE) {

5
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -130,6 +130,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO: handle error
}
} break;
case TDMT_VND_MQ_REB: {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) {
}
} break;
default:
ASSERT(0);
break;

97
source/libs/catalog/inc/catalogInt.h
View File

@ -27,7 +27,7 @@ extern "C" {
#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000
#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000
#define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10
@ -47,9 +47,19 @@ enum {
CTG_RENT_STABLE,
};
enum {
CTG_ACT_UPDATE_VG = 0,
CTG_ACT_UPDATE_TBL,
CTG_ACT_REMOVE_DB,
CTG_ACT_REMOVE_STB,
CTG_ACT_REMOVE_TBL,
CTG_ACT_MAX
};
typedef struct SCtgDebug {
bool lockDebug;
bool cacheDebug;
bool apiDebug;
uint32_t showCachePeriodSec;
} SCtgDebug;
@ -65,7 +75,7 @@ typedef struct SCtgDBCache {
SRWLatch vgLock;
uint64_t dbId;
int8_t deleted;
SDBVgroupInfo *vgInfo;
SDBVgInfo *vgInfo;
SCtgTbMetaCache tbCache;
} SCtgDBCache;
@ -85,7 +95,6 @@ typedef struct SCtgRentMgmt {
typedef struct SCatalog {
uint64_t clusterId;
SRWLatch dbLock;
SHashObj *dbCache; //key:dbname, value:SCtgDBCache
SCtgRentMgmt dbRent;
SCtgRentMgmt stbRent;
@ -96,7 +105,8 @@ typedef struct SCtgApiStat {
} SCtgApiStat;
typedef struct SCtgRuntimeStat {
uint64_t qNum;
uint64_t qDoneNum;
} SCtgRuntimeStat;
typedef struct SCtgCacheStat {
@ -109,15 +119,70 @@ typedef struct SCatalogStat {
SCtgCacheStat cache;
} SCatalogStat;
typedef struct SCtgUpdateVgMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SDBVgInfo* dbInfo;
} SCtgUpdateVgMsg;
typedef struct SCtgUpdateTblMsg {
SCatalog* pCtg;
STableMetaOutput* output;
} SCtgUpdateTblMsg;
typedef struct SCtgRemoveDBMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
} SCtgRemoveDBMsg;
typedef struct SCtgRemoveStbMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t suid;
} SCtgRemoveStbMsg;
typedef struct SCtgMetaAction {
int32_t act;
void *data;
} SCtgMetaAction;
typedef struct SCtgQNode {
SCtgMetaAction action;
struct SCtgQNode *next;
} SCtgQNode;
typedef struct SCatalogMgmt {
bool exit;
SRWLatch lock;
SRWLatch qlock;
SCtgQNode *head;
SCtgQNode *tail;
tsem_t sem;
uint64_t qRemainNum;
pthread_t updateThread;
SHashObj *pCluster; //key: clusterId, value: SCatalog*
SCatalogStat stat;
SCatalogCfg cfg;
} SCatalogMgmt;
typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
typedef int32_t (*ctgActFunc)(SCtgMetaAction *);
typedef struct SCtgAction {
int32_t actId;
char name[32];
ctgActFunc func;
} SCtgAction;
#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1)
#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1)
#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1)
#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1)
#define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
#define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
@ -130,20 +195,26 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0)
#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST)
#define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema))
#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCatalog, __VA_ARGS__)
#define ctgError(param, ...) qError("CTG:%p " param, pCatalog, __VA_ARGS__)
#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCatalog, __VA_ARGS__)
#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCatalog, __VA_ARGS__)
#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCatalog, __VA_ARGS__)
#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCatalog, __VA_ARGS__)
#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST)
#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST)
#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__)
#define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0)
#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0)
#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiDebug) { qDebug(__VA_ARGS__); } } while (0)
#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock))
#define CTG_LOCK(type, _lock) do { \
if (CTG_READ == (type)) { \
assert(atomic_load_32((_lock)) >= 0); \
@ -181,8 +252,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0)
#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0)
#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0)
#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0)

1880
source/libs/catalog/src/catalog.c
View File

File diff suppressed because it is too large Load Diff

405
source/libs/catalog/test/catalogTests.cpp
View File

@ -33,18 +33,31 @@
#include "tep.h"
#include "trpc.h"
#include "tvariant.h"
#include "catalogInt.h"
namespace {
extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
int32_t *exist);
extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output);
extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type);
extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action);
extern "C" int32_t ctgDbgEnableDebug(char *option);
extern "C" int32_t ctgDbgGetStatNum(char *option, void *res);
void ctgTestSetPrepareTableMeta();
void ctgTestSetPrepareCTableMeta();
void ctgTestSetPrepareSTableMeta();
void ctgTestSetPrepareMultiSTableMeta();
void ctgTestSetRspTableMeta();
void ctgTestSetRspCTableMeta();
void ctgTestSetRspSTableMeta();
void ctgTestSetRspMultiSTableMeta();
extern "C" SCatalogMgmt gCtgMgmt;
enum {
CTGT_RSP_VGINFO = 1,
CTGT_RSP_TBMETA,
CTGT_RSP_CTBMETA,
CTGT_RSP_STBMETA,
CTGT_RSP_MSTBMETA,
};
bool ctgTestStop = false;
bool ctgTestEnableSleep = false;
@ -68,6 +81,9 @@ char *ctgTestTablename = "table1";
char *ctgTestCTablename = "ctable1";
char *ctgTestSTablename = "stable1";
int32_t ctgTestRspFunc[10] = {0};
int32_t ctgTestRspIdx = 0;
void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
SCreateDbReq createReq = {0};
strcpy(createReq.db, "1.db1");
@ -113,6 +129,8 @@ void ctgTestInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
ctgDbgEnableDebug("api");
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
@ -179,11 +197,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
strcpy(s->name, "tag1s");
}
void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) {
void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
static int32_t vgVersion = ctgTestVgVersion + 1;
int32_t vgNum = 0;
SVgroupInfo vgInfo = {0};
SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo));
SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo));
dbVgroup->vgVersion = vgVersion++;
@ -250,7 +268,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
return;
}
void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
SUseDbRsp usedbRsp = {0};
strcpy(usedbRsp.db, ctgTestDbname);
usedbRsp.vgVersion = ctgTestVgVersion;
@ -290,7 +308,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
pRsp->pCont = pReq;
}
void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
STableMetaRsp *rspMsg = NULL; // todo
pRsp->code = 0;
@ -326,7 +344,7 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
return;
}
void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
STableMetaRsp *rspMsg = NULL; // todo
pRsp->code = 0;
@ -369,7 +387,7 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
return;
}
void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
STableMetaRsp *rspMsg = NULL; // todo
pRsp->code = 0;
@ -412,7 +430,7 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
return;
}
void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
STableMetaRsp *rspMsg = NULL; // todo
static int32_t idx = 1;
@ -458,151 +476,193 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg,
return;
}
void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetPrepareTableMeta();
void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (ctgTestRspFunc[ctgTestRspIdx]) {
case CTGT_RSP_VGINFO:
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBMETA:
ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_CTBMETA:
ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_STBMETA:
ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_MSTBMETA:
ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp);
break;
default:
break;
}
ctgTestRspIdx++;
return;
}
void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetPrepareCTableMeta();
void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetRspTableMeta();
return;
}
void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
void ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetPrepareSTableMeta();
ctgTestSetRspCTableMeta();
return;
}
void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
void ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetPrepareMultiSTableMeta();
ctgTestSetRspSTableMeta();
return;
}
void ctgTestSetPrepareDbVgroups() {
void ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
ctgTestSetRspMultiSTableMeta();
return;
}
void ctgTestSetRspDbVgroups() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareDbVgroups);
stub.set(rpcSendRecv, ctgTestRspDbVgroups);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareDbVgroups);
stub.set(f.second, ctgTestRspDbVgroups);
}
}
}
void ctgTestSetPrepareTableMeta() {
void ctgTestSetRspTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareTableMeta);
stub.set(rpcSendRecv, ctgTestRspTableMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareTableMeta);
stub.set(f.second, ctgTestRspTableMeta);
}
}
}
void ctgTestSetPrepareCTableMeta() {
void ctgTestSetRspCTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareCTableMeta);
stub.set(rpcSendRecv, ctgTestRspCTableMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareCTableMeta);
stub.set(f.second, ctgTestRspCTableMeta);
}
}
}
void ctgTestSetPrepareSTableMeta() {
void ctgTestSetRspSTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareSTableMeta);
stub.set(rpcSendRecv, ctgTestRspSTableMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareSTableMeta);
stub.set(f.second, ctgTestRspSTableMeta);
}
}
}
void ctgTestSetPrepareMultiSTableMeta() {
void ctgTestSetRspMultiSTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta);
stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareMultiSTableMeta);
stub.set(f.second, ctgTestRspMultiSTableMeta);
}
}
}
void ctgTestSetPrepareDbVgroupsAndNormalMeta() {
void ctgTestSetRspByIdx() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta);
stub.set(rpcSendRecv, ctgTestRspByIdx);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta);
stub.set(f.second, ctgTestRspByIdx);
}
}
}
void ctgTestSetPrepareDbVgroupsAndChildMeta() {
void ctgTestSetRspDbVgroupsAndNormalMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta);
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta);
stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta);
}
}
}
void ctgTestSetPrepareDbVgroupsAndSuperMeta() {
void ctgTestSetRspDbVgroupsAndChildMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta);
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta);
stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta);
}
}
}
void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() {
void ctgTestSetRspDbVgroupsAndSuperMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta);
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta);
stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta);
}
}
}
void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta);
{
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta);
}
}
}
@ -617,7 +677,7 @@ void *ctgTestGetDbVgroupThread(void *param) {
int32_t n = 0;
while (!ctgTestStop) {
code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
if (code) {
assert(0);
}
@ -640,12 +700,12 @@ void *ctgTestGetDbVgroupThread(void *param) {
void *ctgTestSetSameDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgroupInfo *dbVgroup = NULL;
SDBVgInfo *dbVgroup = NULL;
int32_t n = 0;
while (!ctgTestStop) {
ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
if (code) {
assert(0);
}
@ -664,12 +724,12 @@ void *ctgTestSetSameDbVgroupThread(void *param) {
void *ctgTestSetDiffDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgroupInfo *dbVgroup = NULL;
SDBVgInfo *dbVgroup = NULL;
int32_t n = 0;
while (!ctgTestStop) {
ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup);
if (code) {
assert(0);
}
@ -719,14 +779,24 @@ void *ctgTestGetCtableMetaThread(void *param) {
void *ctgTestSetCtableMetaThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgroupInfo dbVgroup = {0};
SDBVgInfo dbVgroup = {0};
int32_t n = 0;
STableMetaOutput output = {0};
STableMetaOutput *output = NULL;
ctgTestBuildCTableMetaOutput(&output);
SCtgMetaAction action = {0};
action.act = CTG_ACT_UPDATE_TBL;
while (!ctgTestStop) {
code = ctgUpdateTableMetaCache(pCtg, &output);
output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput));
ctgTestBuildCTableMetaOutput(output);
SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg));
msg->pCtg = pCtg;
msg->output = output;
action.data = msg;
code = ctgActUpdateTbl(&action);
if (code) {
assert(0);
}
@ -739,8 +809,6 @@ void *ctgTestSetCtableMetaThread(void *param) {
}
}
tfree(output.tbMeta);
return NULL;
}
@ -751,7 +819,7 @@ TEST(tableMeta, normalTable) {
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroups();
ctgTestSetRspDbVgroups();
initQueryModuleMsgHandle();
@ -772,7 +840,11 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ctgTestSetPrepareTableMeta();
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
usleep(10000);
}
ctgTestSetRspTableMeta();
STableMeta *tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
@ -786,6 +858,16 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(10000);
} else {
break;
}
}
tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
@ -834,6 +916,7 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(allStbNum, 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, childTableCase) {
@ -843,7 +926,7 @@ TEST(tableMeta, childTableCase) {
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndChildMeta();
ctgTestSetRspDbVgroupsAndChildMeta();
initQueryModuleMsgHandle();
@ -870,6 +953,16 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(10000);
} else {
break;
}
}
tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
@ -932,6 +1025,7 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(allStbNum, 1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, superTableCase) {
@ -939,7 +1033,7 @@ TEST(tableMeta, superTableCase) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
ctgTestSetPrepareDbVgroupsAndSuperMeta();
ctgTestSetRspDbVgroupsAndSuperMeta();
initQueryModuleMsgHandle();
@ -968,7 +1062,17 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
ctgTestSetPrepareCTableMeta();
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(10000);
} else {
break;
}
}
ctgTestSetRspCTableMeta();
tableMeta = NULL;
@ -985,8 +1089,18 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (2 != n) {
usleep(10000);
} else {
break;
}
}
tableMeta = NULL;
code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -1034,6 +1148,7 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(allStbNum, 1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, rmStbMeta) {
@ -1043,7 +1158,7 @@ TEST(tableMeta, rmStbMeta) {
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndSuperMeta();
ctgTestSetRspDbVgroupsAndSuperMeta();
initQueryModuleMsgHandle();
@ -1072,9 +1187,30 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(10000);
} else {
break;
}
}
code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid);
ASSERT_EQ(code, 0);
while (true) {
int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
if (n || m) {
usleep(10000);
} else {
break;
}
}
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
@ -1082,6 +1218,7 @@ TEST(tableMeta, rmStbMeta) {
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableMeta, updateStbMeta) {
@ -1091,7 +1228,7 @@ TEST(tableMeta, updateStbMeta) {
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndSuperMeta();
ctgTestSetRspDbVgroupsAndSuperMeta();
initQueryModuleMsgHandle();
@ -1120,6 +1257,16 @@ TEST(tableMeta, updateStbMeta) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(10000);
} else {
break;
}
}
tfree(tableMeta);
STableMetaRsp rsp = {0};
@ -1128,6 +1275,16 @@ TEST(tableMeta, updateStbMeta) {
code = catalogUpdateSTableMeta(pCtg, &rsp);
ASSERT_EQ(code, 0);
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
usleep(10000);
} else {
break;
}
}
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
@ -1150,6 +1307,7 @@ TEST(tableMeta, updateStbMeta) {
tfree(tableMeta);
catalogDestroy();
memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat));
}
TEST(tableDistVgroup, normalTable) {
@ -1158,7 +1316,15 @@ TEST(tableDistVgroup, normalTable) {
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestSetPrepareDbVgroupsAndNormalMeta();
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestRspFunc[2] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1174,7 +1340,7 @@ TEST(tableDistVgroup, normalTable) {
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -1182,6 +1348,7 @@ TEST(tableDistVgroup, normalTable) {
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableDistVgroup, childTableCase) {
@ -1190,7 +1357,16 @@ TEST(tableDistVgroup, childTableCase) {
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestSetPrepareDbVgroupsAndChildMeta();
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_CTBMETA;
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1206,7 +1382,7 @@ TEST(tableDistVgroup, childTableCase) {
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -1214,6 +1390,7 @@ TEST(tableDistVgroup, childTableCase) {
ASSERT_EQ(vgInfo->epset.numOfEps, 4);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(tableDistVgroup, superTableCase) {
@ -1222,7 +1399,18 @@ TEST(tableDistVgroup, superTableCase) {
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestSetPrepareDbVgroupsAndSuperMeta();
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_STBMETA;
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1237,7 +1425,7 @@ TEST(tableDistVgroup, superTableCase) {
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -1251,6 +1439,7 @@ TEST(tableDistVgroup, superTableCase) {
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(dbVgroup, getSetDbVgroupCase) {
@ -1258,12 +1447,19 @@ TEST(dbVgroup, getSetDbVgroupCase) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgroupInfo *dbVgroup = NULL;
SDBVgInfo *dbVgroup = NULL;
SArray *vgList = NULL;
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndNormalMeta();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1279,16 +1475,21 @@ TEST(dbVgroup, getSetDbVgroupCase) {
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
usleep(10000);
}
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -1297,15 +1498,26 @@ TEST(dbVgroup, getSetDbVgroupCase) {
taosArrayDestroy(vgList);
ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
ASSERT_EQ(code, 0);
while (true) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
usleep(10000);
} else {
break;
}
}
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 7);
ASSERT_EQ(vgInfo.epset.numOfEps, 2);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -1314,6 +1526,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
taosArrayDestroy(vgList);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, getSetRmSameDbVgroup) {
@ -1321,13 +1534,13 @@ TEST(multiThread, getSetRmSameDbVgroup) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgroupInfo dbVgroup = {0};
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroups();
ctgTestSetRspDbVgroups();
initQueryModuleMsgHandle();
@ -1365,6 +1578,7 @@ TEST(multiThread, getSetRmSameDbVgroup) {
sleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, getSetRmDiffDbVgroup) {
@ -1372,13 +1586,13 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgroupInfo dbVgroup = {0};
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroups();
ctgTestSetRspDbVgroups();
initQueryModuleMsgHandle();
@ -1416,6 +1630,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
sleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(multiThread, ctableMeta) {
@ -1423,13 +1638,13 @@ TEST(multiThread, ctableMeta) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgroupInfo dbVgroup = {0};
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndChildMeta();
ctgTestSetRspDbVgroupsAndChildMeta();
initQueryModuleMsgHandle();
@ -1466,6 +1681,7 @@ TEST(multiThread, ctableMeta) {
sleep(2);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(rentTest, allRent) {
@ -1473,7 +1689,7 @@ TEST(rentTest, allRent) {
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgroupInfo dbVgroup = {0};
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
SDbVgVersion *dbs = NULL;
@ -1482,7 +1698,7 @@ TEST(rentTest, allRent) {
ctgTestInitLogFile();
ctgTestSetPrepareDbVgroupsAndMultiSuperMeta();
ctgTestSetRspDbVgroupsAndMultiSuperMeta();
initQueryModuleMsgHandle();
@ -1512,6 +1728,10 @@ TEST(rentTest, allRent) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
usleep(10000);
}
code = catalogGetExpiredDBs(pCtg, &dbs, &num);
ASSERT_EQ(code, 0);
printf("%d - expired dbNum:%d\n", i, num);
@ -1538,6 +1758,7 @@ TEST(rentTest, allRent) {
}
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
int main(int argc, char **argv) {

5
source/libs/function/src/functionMgt.c
View File

@ -47,10 +47,11 @@ int32_t fmGetHandle(FuncMgtHandle* pHandle) {
int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
SFuncMgtService* pService = (SFuncMgtService*)handle;
pFuncId = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName));
if (NULL == pFuncId) {
void* pVal = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName));
if (NULL == pVal) {
return TSDB_CODE_FAILED;
}
*pFuncId = *(int32_t*)pVal;
if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) {
return TSDB_CODE_FAILED;
}

1
source/libs/parser/inc/astCreateFuncs.h
View File

@ -55,6 +55,7 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToke
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);

8
source/libs/parser/inc/new_sql.y
View File

@ -238,6 +238,7 @@ joined_table(A) ::=
%type join_type { EJoinType }
%destructor join_type { PARSER_DESTRUCTOR_TRACE; }
join_type(A) ::= . { PARSER_TRACE; A = JOIN_TYPE_INNER; }
join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; }
/************************************************ query_specification *************************************************/
@ -315,7 +316,12 @@ fill_mode(A) ::= NEXT.
%type group_by_clause_opt { SNodeList* }
%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); }
group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; }
group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; }
group_by_clause_opt(A) ::= GROUP BY group_by_list(B). { PARSER_TRACE; A = B; }
%type group_by_list { SNodeList* }
%destructor group_by_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); }
group_by_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, B))); }
group_by_list(A) ::= group_by_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, C))); }
having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; }
having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; }

12
source/libs/parser/src/astCreateFuncs.c
View File

@ -242,6 +242,9 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order
CHECK_OUT_OF_MEM(orderByExpr);
orderByExpr->pExpr = pExpr;
orderByExpr->order = order;
if (NULL_ORDER_DEFAULT == nullOrder) {
nullOrder = (ORDER_ASC == order ? NULL_ORDER_FIRST : NULL_ORDER_LAST);
}
orderByExpr->nullOrder = nullOrder;
return (SNode*)orderByExpr;
}
@ -279,6 +282,15 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) {
return (SNode*)fill;
}
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) {
SGroupingSetNode* groupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET);
CHECK_OUT_OF_MEM(groupingSet);
groupingSet->groupingSetType = GP_TYPE_NORMAL;
groupingSet->pParameterList = nodesMakeList();
nodesListAppend(groupingSet->pParameterList, pNode);
return (SNode*)groupingSet;
}
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) {
strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n);
return pNode;

2
source/libs/parser/src/astValidate.c
View File

@ -3647,7 +3647,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
//TODO remove it
int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) {
SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList);
int32_t code = catalogGetTableDistVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList);
if (code != TSDB_CODE_SUCCESS) {
return code;
}

2
source/libs/parser/src/dCDAstProcess.c
View File

@ -50,7 +50,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array);
int32_t code = catalogGetDBVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return code;

1159
source/libs/parser/src/new_sql.c
View File

File diff suppressed because it is too large Load Diff

242
source/libs/parser/src/parserImpl.c
View File

@ -116,10 +116,6 @@ static uint32_t toNewTokenId(uint32_t tokenId) {
return NEW_TK_FROM;
case TK_JOIN:
return NEW_TK_JOIN;
// case TK_ON:
// return NEW_TK_ON;
// case TK_INNER:
// return NEW_TK_INNER;
// case TK_PARTITION:
// return NEW_TK_PARTITION;
case TK_SESSION:
@ -163,6 +159,8 @@ static uint32_t toNewTokenId(uint32_t tokenId) {
case TK_OFFSET:
return NEW_TK_OFFSET;
case TK_SPACE:
case NEW_TK_ON:
case NEW_TK_INNER:
break;
default:
printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!tokenId = %d\n", tokenId);
@ -248,6 +246,10 @@ typedef enum ESqlClause {
} ESqlClause;
static bool afterGroupBy(ESqlClause clause) {
return clause > SQL_CLAUSE_GROUP_BY;
}
static bool beforeHaving(ESqlClause clause) {
return clause < SQL_CLAUSE_HAVING;
}
@ -259,7 +261,7 @@ typedef struct STranslateContext {
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
ESqlClause currClause;
void* pExt;
SSelectStmt* pCurrStmt;
} STranslateContext;
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
@ -284,6 +286,12 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "There mustn't be aggregation";
case TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT:
return "ORDER BY item must be the number of a SELECT-list expression";
case TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION:
return "Not a GROUP BY expression";
case TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION:
return "Not SELECTed expression";
case TSDB_CODE_PAR_NOT_SINGLE_GROUP:
return "Not a single-group group function";
default:
return "Unknown error";
}
@ -329,7 +337,7 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST
if ('\0' != pCol->dbName[0]) {
cmp = strcmp(pCol->dbName, pTable->dbName);
} else {
cmp = strcmp(currentDb, pTable->dbName);
cmp = (QUERY_NODE_REAL_TABLE == nodeType(pTable) ? strcmp(currentDb, pTable->dbName) : 0);
}
if (0 == cmp) {
cmp = strcmp(pCol->tableAlias, pTable->tableAlias);
@ -415,23 +423,29 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) {
return found;
}
static bool translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) {
static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t nums = taosArrayGetSize(pTables);
bool foundTable = false;
for (size_t i = 0; i < nums; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i);
if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) {
foundTable = true;
if (findAndSetColumn(pCol, pTable)) {
break;
}
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName);
return false;
return DEAL_RES_ERROR;
}
}
return true;
if (!foundTable) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pCol->tableAlias);
return DEAL_RES_ERROR;
}
return DEAL_RES_CONTINUE;
}
static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) {
static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t nums = taosArrayGetSize(pTables);
bool found = false;
@ -440,20 +454,20 @@ static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* p
if (findAndSetColumn(pCol, pTable)) {
if (found) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName);
return false;
return DEAL_RES_ERROR;
}
found = true;
}
}
if (!found) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName);
return false;
return DEAL_RES_ERROR;
}
return true;
return DEAL_RES_CONTINUE;
}
static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol) {
SNodeList* pProjectionList = pCxt->pExt;
SNodeList* pProjectionList = pCxt->pCurrStmt->pProjectionList;
SNode* pNode;
FOREACH(pNode, pProjectionList) {
SExprNode* pExpr = (SExprNode*)pNode;
@ -465,10 +479,10 @@ static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode* pCol)
return false;
}
static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
// count(*)/first(*)/last(*)
if (0 == strcmp(pCol->colName, "*")) {
return true;
return DEAL_RES_CONTINUE;
}
if ('\0' != pCol->tableAlias[0]) {
return translateColumnWithPrefix(pCxt, pCol);
@ -477,7 +491,7 @@ static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
if (SQL_CLAUSE_ORDER_BY == pCxt->currClause) {
found = translateColumnUseAlias(pCxt, pCol);
}
return found ? true : translateColumnWithoutPrefix(pCxt, pCol);
return found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol);
}
static int32_t trimStringCopy(const char* src, int32_t len, char* dst) {
@ -500,12 +514,12 @@ static int32_t trimStringCopy(const char* src, int32_t len, char* dst) {
return j;
}
static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) {
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
if (pVal->isDuration) {
char unit = 0;
if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
return false;
return DEAL_RES_ERROR;
}
} else {
switch (pVal->node.resType.type) {
@ -552,7 +566,7 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) {
if (taosParseTime(tmp, &pVal->datum.u, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
tfree(tmp);
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
return false;
return DEAL_RES_ERROR;
}
tfree(tmp);
break;
@ -565,55 +579,56 @@ static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) {
break;
}
}
return true;
return DEAL_RES_CONTINUE;
}
static bool translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
if (nodesIsArithmeticOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type ||
TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
return false;
return DEAL_RES_ERROR;
}
pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
return true;
} else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type ||
TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
return false;
return DEAL_RES_ERROR;
}
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
return true;
} else {
// todo json operator
return true;
}
return true;
return DEAL_RES_CONTINUE;
}
static bool translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pCxt->fmgt, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
return false;
return DEAL_RES_ERROR;
}
int32_t code = fmGetFuncResultType(pFunc);
if (TSDB_CODE_SUCCESS != code) {
generateSyntaxErrMsg(pCxt, code, pFunc->functionName);
return false;
return DEAL_RES_ERROR;
}
if (fmIsAggFunc(pFunc->funcId) && afterGroupBy(pCxt->currClause)) {
if (fmIsAggFunc(pFunc->funcId) && beforeHaving(pCxt->currClause)) {
generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION);
return false;
return DEAL_RES_ERROR;
}
return true;
return DEAL_RES_CONTINUE;
}
static bool doTranslateExpr(SNode* pNode, void* pContext) {
static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) {
return (TSDB_CODE_SUCCESS == translateSubquery(pCxt, pNode) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR);
}
static EDealRes doTranslateExpr(SNode* pNode, void* pContext) {
STranslateContext* pCxt = (STranslateContext*)pContext;
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN:
@ -625,11 +640,11 @@ static bool doTranslateExpr(SNode* pNode, void* pContext) {
case QUERY_NODE_FUNCTION:
return translateFunction(pCxt, (SFunctionNode*)pNode);
case QUERY_NODE_TEMP_TABLE:
return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery);
return translateExprSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery);
default:
break;
}
return true;
return DEAL_RES_CONTINUE;
}
static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) {
@ -642,6 +657,103 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) {
return pCxt->errCode;
}
static bool isAliasColumn(SColumnNode* pCol) {
return ('\0' == pCol->tableAlias[0]);
}
static bool isDistinctOrderBy(STranslateContext* pCxt) {
return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct);
}
static SNodeList* getGroupByList(STranslateContext* pCxt) {
if (isDistinctOrderBy(pCxt)) {
return pCxt->pCurrStmt->pProjectionList;
}
return pCxt->pCurrStmt->pGroupByList;
}
static SNode* getGroupByNode(SNode* pNode) {
if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) {
return nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0);
}
return pNode;
}
static int32_t getGroupByErrorCode(STranslateContext* pCxt) {
if (isDistinctOrderBy(pCxt)) {
return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION;
}
return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION;
}
static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) {
STranslateContext* pCxt = (STranslateContext*)pContext;
if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) {
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && !isDistinctOrderBy(pCxt)) {
return DEAL_RES_IGNORE_CHILD;
}
SNode* pGroupNode;
FOREACH(pGroupNode, getGroupByList(pCxt)) {
if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) {
return DEAL_RES_IGNORE_CHILD;
}
}
if (QUERY_NODE_COLUMN == nodeType(pNode) ||
(QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && isDistinctOrderBy(pCxt))) {
generateSyntaxErrMsg(pCxt, getGroupByErrorCode(pCxt));
return DEAL_RES_ERROR;
}
return DEAL_RES_CONTINUE;
}
static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) {
nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt);
return pCxt->errCode;
}
static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList) {
if (NULL == getGroupByList(pCxt)) {
return TSDB_CODE_SUCCESS;
}
nodesWalkList(pList, doCheckExprForGroupBy, pCxt);
return pCxt->errCode;
}
typedef struct CheckAggColCoexistCxt {
STranslateContext* pTranslateCxt;
bool existAggFunc;
bool existCol;
} CheckAggColCoexistCxt;
static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) {
CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) {
pCxt->existAggFunc = true;
return DEAL_RES_IGNORE_CHILD;
}
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
pCxt->existCol = true;
}
return DEAL_RES_CONTINUE;
}
static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (NULL != pSelect->pGroupByList) {
return TSDB_CODE_SUCCESS;
}
CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false };
nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
if (!pSelect->isDistinct) {
nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
}
if (cxt.existAggFunc && cxt.existCol) {
return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_NOT_SINGLE_GROUP);
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pTable)) {
@ -732,12 +844,13 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec
*pOther = false;
SNode* pNode;
FOREACH(pNode, pOrderByList) {
if (QUERY_NODE_VALUE == nodeType(pNode)) {
SValueNode* pVal = (SValueNode*)pNode;
if (translateValue(pCxt, pVal)) {
SNode* pExpr = ((SOrderByExprNode*)pNode)->pExpr;
if (QUERY_NODE_VALUE == nodeType(pExpr)) {
SValueNode* pVal = (SValueNode*)pExpr;
if (!translateValue(pCxt, pVal)) {
return false;
}
int32_t pos = getPositionValue((SValueNode*)pNode);
int32_t pos = getPositionValue((SValueNode*)pExpr);
if (pos < 0) {
ERASE_NODE(pOrderByList);
nodesDestroyNode(pNode);
@ -747,9 +860,9 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec
return false;
} else {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos), pCol);
REPLACE_NODE(pCol);
nodesDestroyNode(pNode);
setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), pCol);
((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol;
nodesDestroyNode(pExpr);
}
} else {
*pOther = true;
@ -758,17 +871,20 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec
return true;
}
static int32_t translateOrderBy(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList) {
static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
bool other;
if (!translateOrderByPosition(pCxt, pProjectionList, pOrderByList, &other)) {
if (!translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other)) {
return pCxt->errCode;
}
if (!other) {
return TSDB_CODE_SUCCESS;
}
pCxt->currClause = SQL_CLAUSE_ORDER_BY;
pCxt->pExt = pProjectionList;
return translateExprList(pCxt, pOrderByList);
int32_t code = translateExprList(pCxt, pSelect->pOrderByList);
if (TSDB_CODE_SUCCESS == code) {
code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList);
}
return code;
}
static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -778,12 +894,22 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect
pCxt->currClause = SQL_CLAUSE_SELECT;
code = translateExprList(pCxt, pSelect->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList);
}
return code;
}
static int32_t translateHaving(STranslateContext* pCxt, SNode* pHaving) {
static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) {
return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION);
}
pCxt->currClause = SQL_CLAUSE_HAVING;
return translateExpr(pCxt, pHaving);
int32_t code = translateExpr(pCxt, pSelect->pHaving);
if (TSDB_CODE_SUCCESS == code) {
code = checkExprForGroupBy(pCxt, pSelect->pHaving);
}
return code;
}
static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList) {
@ -818,8 +944,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) {
// } SSelectStmt;
static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = TSDB_CODE_SUCCESS;
code = translateFrom(pCxt, pSelect->pFromTable);
pCxt->pCurrStmt = pSelect;
int32_t code = translateFrom(pCxt, pSelect->pFromTable);
if (TSDB_CODE_SUCCESS == code) {
code = translateWhere(pCxt, pSelect->pWhere);
}
@ -833,15 +959,17 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
code = translateGroupBy(pCxt, pSelect->pGroupByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateHaving(pCxt, pSelect->pHaving);
code = translateHaving(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateSelectList(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateOrderBy(pCxt, pSelect->pProjectionList, pSelect->pOrderByList);
code = translateOrderBy(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkAggColCoexist(pCxt, pSelect);
}
// printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, code);
return code;
}
@ -860,11 +988,11 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
++(pCxt->currLevel);
ESqlClause currClause = pCxt->currClause;
void* pExt = pCxt->pExt;
SSelectStmt* pCurrStmt = pCxt->pCurrStmt;
int32_t code = translateQuery(pCxt, pNode);
--(pCxt->currLevel);
pCxt->currClause = currClause;
pCxt->pExt = pExt;
pCxt->pCurrStmt = pCurrStmt;
return code;
}

4
source/libs/parser/src/ttokenizer.c
View File

@ -227,7 +227,9 @@ static SKeyword keywordTable[] = {
{"OUTPUTTYPE", TK_OUTPUTTYPE},
{"AGGREGATE", TK_AGGREGATE},
{"BUFSIZE", TK_BUFSIZE},
{"PORT", TK_PORT},
{"PORT", TK_PORT},
{"INNER", NEW_TK_INNER},
{"ON", NEW_TK_ON},
};
static const char isIdChar[] = {

241
source/libs/parser/test/newParserTest.cpp
View File

@ -53,8 +53,8 @@ protected:
code = doTranslate(&cxt_, &query_);
// cout << "doTranslate return " << code << endl;
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl;
return (TSDB_CODE_SUCCESS != translateCode);
cout << "sql:[" << cxt_.pSql << "] code:" << code << ", " << translateCode << ", msg:" << errMagBuf_ << endl;
return (code == translateCode);
}
if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) {
cout << "input sql : [" << cxt_.pSql << "]" << endl;
@ -71,6 +71,24 @@ protected:
private:
static const int max_err_len = 1024;
void exprNodeToStr(const SNode* node, string& str, bool isProject) {
switch (nodeType(node)) {
case QUERY_NODE_COLUMN:
case QUERY_NODE_VALUE:
case QUERY_NODE_OPERATOR:
case QUERY_NODE_FUNCTION: {
SExprNode* pExpr = (SExprNode*)node;
str.append(" [" + dataTypeToStr(pExpr->resType) + "]");
if (isProject) {
str.append(" AS " + string(pExpr->aliasName));
}
break;
}
default:
break;
}
}
string dataTypeToStr(const SDataType& dt) {
switch (dt.type) {
case TSDB_DATA_TYPE_NULL:
@ -119,7 +137,7 @@ private:
return "Unknown Data Type " + to_string(dt.type);
}
void valueNodeToStr(const SValueNode* pVal, string& str, bool isProject) {
void valueNodeToStr(const SValueNode* pVal, string& str) {
switch (pVal->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
str.append("null");
@ -160,10 +178,41 @@ private:
default:
break;
}
str.append(" [" + dataTypeToStr(pVal->node.resType) + "]");
if (isProject) {
str.append(" AS " + string(pVal->node.aliasName));
}
void columnNodeToStr(const SColumnNode* pCol, string& str) {
if ('\0' != pCol->dbName[0]) {
str.append(pCol->dbName);
str.append(".");
}
if ('\0' != pCol->tableAlias[0]) {
str.append(pCol->tableAlias);
str.append(".");
}
str.append(pCol->colName);
}
void operatorToStr(const SOperatorNode* pOp, string& str) {
nodeToStr(pOp->pLeft, str, false);
str.append(opTypeToStr(pOp->opType));
nodeToStr(pOp->pRight, str, false);
}
void functionToStr(const SFunctionNode* pFunc, string& str) {
str.append(pFunc->functionName);
str.append("(");
nodeListToStr(pFunc->pParameterList, "", str, false, ", ");
str.append(")");
}
void groupingSetToStr(SGroupingSetNode* pGroup, string& str) {
nodeToStr(nodesListGetNode(pGroup->pParameterList, 0), str, false);
}
void orderByExprToStr(SOrderByExprNode* pOrderBy, string& str) {
nodeToStr(pOrderBy->pExpr, str, false);
str.append((ORDER_ASC == pOrderBy->order ? " ASC" : " DESC"));
str.append((NULL_ORDER_FIRST == pOrderBy->nullOrder ? " NULLS FIRST" : " NULLS LAST"));
}
void nodeToStr(const SNode* node, string& str, bool isProject) {
@ -173,48 +222,41 @@ private:
switch (nodeType(node)) {
case QUERY_NODE_COLUMN: {
SColumnNode* pCol = (SColumnNode*)node;
if ('\0' != pCol->dbName[0]) {
str.append(pCol->dbName);
str.append(".");
}
if ('\0' != pCol->tableAlias[0]) {
str.append(pCol->tableAlias);
str.append(".");
}
str.append(pCol->colName);
str.append(" [" + dataTypeToStr(pCol->node.resType) + "]");
if (isProject) {
str.append(" AS " + string(pCol->node.aliasName));
}
columnNodeToStr((SColumnNode*)node, str);
break;
}
case QUERY_NODE_VALUE: {
valueNodeToStr((SValueNode*)node, str, isProject);
valueNodeToStr((SValueNode*)node, str);
break;
}
case QUERY_NODE_OPERATOR: {
SOperatorNode* pOp = (SOperatorNode*)node;
nodeToStr(pOp->pLeft, str, false);
str.append(opTypeToStr(pOp->opType));
nodeToStr(pOp->pRight, str, false);
str.append(" [" + dataTypeToStr(pOp->node.resType) + "]");
if (isProject) {
str.append(" AS " + string(pOp->node.aliasName));
}
operatorToStr((SOperatorNode*)node, str);
break;
}
case QUERY_NODE_FUNCTION: {
functionToStr((SFunctionNode*)node, str);
break;
}
case QUERY_NODE_GROUPING_SET: {
groupingSetToStr((SGroupingSetNode*)node, str);
break;
}
case QUERY_NODE_ORDER_BY_EXPR: {
orderByExprToStr((SOrderByExprNode*)node, str);
break;
}
default:
break;
}
exprNodeToStr(node, str, isProject);
}
void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false) {
void nodeListToStr(const SNodeList* nodelist, const string& prefix, string& str, bool isProject = false, const string& sep = string("\n")) {
SNode* node = nullptr;
FOREACH(node, nodelist) {
str.append(prefix);
nodeToStr(node, str, isProject);
str.append("\n");
str.append(sep);
}
}
@ -265,8 +307,20 @@ private:
}
str.append("\n");
nodeListToStr(select->pProjectionList, prefix + "\t", str, true);
str.append("\n" + prefix + "FROM\n");
str.append(prefix + "FROM\n");
tableToStr(select->pFromTable, prefix + "\t", str);
if (nullptr != select->pWhere) {
str.append("\n" + prefix + "WHERE\n\t");
nodeToStr(select->pWhere, str, false);
}
if (nullptr != select->pGroupByList) {
str.append("\n" + prefix + "GROUP BY\n");
nodeListToStr(select->pGroupByList, prefix + "\t", str, true);
}
if (nullptr != select->pOrderByList) {
str.append(prefix + "ORDER BY\n");
nodeListToStr(select->pOrderByList, prefix + "\t", str, true);
}
}
void selectToSql(const SNode* node, string& sql) {
@ -332,15 +386,23 @@ private:
case OP_TYPE_SUB:
return " - ";
case OP_TYPE_MULTI:
return " * ";
case OP_TYPE_DIV:
return " / ";
case OP_TYPE_MOD:
return " % ";
case OP_TYPE_GREATER_THAN:
return " > ";
case OP_TYPE_GREATER_EQUAL:
return " >= ";
case OP_TYPE_LOWER_THAN:
return " < ";
case OP_TYPE_LOWER_EQUAL:
return " <= ";
case OP_TYPE_EQUAL:
return " = ";
case OP_TYPE_NOT_EQUAL:
return " != ";
case OP_TYPE_IN:
case OP_TYPE_NOT_IN:
case OP_TYPE_LIKE:
@ -454,11 +516,41 @@ TEST_F(NewParserTest, selectExpression) {
TEST_F(NewParserTest, selectClause) {
setDatabase("root", "test");
// GROUP BY clause
bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0");
ASSERT_TRUE(run());
bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2");
ASSERT_TRUE(run());
bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 HAVING count(c1) > 10");
ASSERT_TRUE(run());
bind("SELECT count(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1");
ASSERT_TRUE(run());
bind("SELECT count(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2");
ASSERT_TRUE(run());
// ORDER BY clause
bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY cnt");
ASSERT_TRUE(run());
bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1");
ASSERT_TRUE(run());
// DISTINCT clause
bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1");
ASSERT_TRUE(run());
bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2");
ASSERT_TRUE(run());
bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2");
ASSERT_TRUE(run());
bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)");
ASSERT_TRUE(run());
}
TEST_F(NewParserTest, selectSyntaxError) {
@ -480,15 +572,88 @@ TEST_F(NewParserTest, selectSyntaxError) {
TEST_F(NewParserTest, selectSemanticError) {
setDatabase("root", "test");
bind("SELECT * FROM t10");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED));
// TSDB_CODE_PAR_INVALID_COLUMN
bind("SELECT c1, c3 FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED));
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN));
bind("SELECT t1.c1, t1.c3 FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN));
// TSDB_CODE_PAR_TABLE_NOT_EXIST
bind("SELECT * FROM t10");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST));
bind("SELECT * FROM test.t10");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST));
bind("SELECT t2.c1 FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST));
// TSDB_CODE_PAR_AMBIGUOUS_COLUMN
bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED));
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN));
// TSDB_CODE_PAR_WRONG_VALUE_TYPE
bind("SELECT 10n FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE));
bind("SELECT TIMESTAMP '2010' FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE));
// TSDB_CODE_PAR_INVALID_FUNTION
bind("SELECT cnt(*) FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_FUNTION));
// TSDB_CODE_PAR_FUNTION_PARA_NUM
// TSDB_CODE_PAR_FUNTION_PARA_TYPE
// TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION
bind("SELECT c2 FROM t1 tt1 JOIN t1 tt2 ON count(*) > 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION));
bind("SELECT c2 FROM t1 where count(*) > 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED));
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION));
bind("SELECT c2 FROM t1 GROUP BY count(*)");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION));
// TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT
bind("SELECT c2 FROM t1 ORDER BY 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT));
bind("SELECT c2 FROM t1 ORDER BY 2");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT));
// TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION
bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION));
bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c1 > 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION));
bind("SELECT count(*), c1 cnt FROM t1 GROUP BY c2 HAVING c2 > 0");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION));
bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION));
// TSDB_CODE_PAR_NOT_SINGLE_GROUP
bind("SELECT count(*), c1 FROM t1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP));
bind("SELECT count(*) FROM t1 ORDER BY c1");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP));
bind("SELECT c1 FROM t1 ORDER BY count(*)");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP));
// TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION
bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION));
bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION));
bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION));
}

4
source/libs/qcom/src/querymsg.c
View File

@ -96,9 +96,9 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo));
pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo));
if (NULL == pOut->dbVgroup) {
qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo));
qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo));
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}

14
source/libs/transport/inc/transComm.h
View File

@ -151,7 +151,8 @@ typedef struct {
char version : 4; // RPC version
char comp : 4; // compression algorithm, 0:no compression 1:lz4
char resflag : 2; // reserved bits
char spi : 3; // security parameter index
char spi : 1; // security parameter index
char secured : 2;
char encrypt : 3; // encrypt algorithm, 0: no encryption
uint32_t code; // del later
@ -170,6 +171,11 @@ typedef struct {
uint8_t auth[TSDB_AUTH_LEN];
} STransDigestMsg;
typedef struct {
uint8_t user[TSDB_UNI_LEN];
uint8_t secret[TSDB_PASSWORD_LEN];
} STransUserMsg;
#pragma pack(pop)
#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member)))
@ -227,7 +233,7 @@ typedef struct {
uv_async_t* asyncs;
} SAsyncPool;
SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb);
SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb);
void transDestroyAsyncPool(SAsyncPool* pool);
int transSendAsync(SAsyncPool* pool, queue* mq);
@ -236,4 +242,8 @@ int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf);
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
// int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen);
// int transUnpackMsg(char *msg, SRpcMsg *pMsg, bool );
#endif

13
source/libs/transport/src/trans.c
View File

@ -30,12 +30,21 @@ void* rpcOpen(const SRpcInit* pInit) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
}
pRpc->cfp = pInit->cfp;
// pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->numOfThreads = pInit->numOfThreads;
if (pInit->connType == TAOS_CONN_SERVER) {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
} else {
pRpc->numOfThreads = pInit->numOfThreads;
}
pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
pRpc->parent = pInit->parent;
if (pInit->user) {
memcpy(pRpc->user, pInit->user, strlen(pInit->user));
}
if (pInit->secret) {
memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret));
}
return pRpc;
}

70
source/libs/transport/src/transCli.c
View File

@ -38,6 +38,7 @@ typedef struct SCliConn {
int32_t ref;
// debug and log info
struct sockaddr_in addr;
struct sockaddr_in locaddr;
} SCliConn;
typedef struct SCliMsg {
@ -130,14 +131,16 @@ static void clientHandleResp(SCliConn* conn) {
rpcMsg.msgType = pHead->msgType;
rpcMsg.ahandle = pCtx->ahandle;
tDebug("client conn %p %s received from %s:%d", conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType),
inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr),
ntohs(conn->locaddr.sin_port));
if (conn->push != NULL && conn->notifyCount != 0) {
(*conn->push->callback)(conn->push->arg, &rpcMsg);
conn->push = NULL;
} else {
if (pCtx->pSem == NULL) {
tTrace("client conn(sync) %p handle resp", conn);
tTrace("client conn%p handle resp", conn);
(pRpc->cfp)(pRpc->parent, &rpcMsg, NULL);
} else {
tTrace("client conn(sync) %p handle resp", conn);
@ -146,6 +149,7 @@ static void clientHandleResp(SCliConn* conn) {
}
}
conn->notifyCount += 1;
conn->secured = pHead->secured;
// buf's mem alread translated to rpcMsg.pCont
transClearBuffer(&conn->readBuf);
@ -154,12 +158,12 @@ static void clientHandleResp(SCliConn* conn) {
SCliThrdObj* pThrd = conn->hostThrd;
// user owns conn->persist = 1
if (conn->push != NULL) {
if (conn->push == NULL) {
addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
}
destroyCmsg(pMsg);
conn->data = NULL;
destroyCmsg(conn->data);
conn->data = NULL;
}
// start thread's timer of conn pool if not active
if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) {
uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
@ -182,21 +186,23 @@ static void clientHandleExcept(SCliConn* pConn) {
if (pConn->push != NULL && pConn->notifyCount != 0) {
(*pConn->push->callback)(pConn->push->arg, &rpcMsg);
pConn->push = NULL;
} else {
if (pCtx->pSem == NULL) {
(pCtx->pTransInst->cfp)(pCtx->pTransInst->parent, &rpcMsg, NULL);
} else {
memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg));
// SRpcMsg rpcMsg
tsem_post(pCtx->pSem);
}
if (pConn->push != NULL) {
(*pConn->push->callback)(pConn->push->arg, &rpcMsg);
}
pConn->push = NULL;
}
if (pConn->push == NULL) {
destroyCmsg(pConn->data);
pConn->data = NULL;
}
destroyCmsg(pMsg);
pConn->data = NULL;
// transDestroyConnCtx(pCtx);
clientConnDestroy(pConn, true);
pConn->notifyCount += 1;
@ -382,17 +388,40 @@ static void clientWriteCb(uv_write_t* req, int status) {
static void clientWrite(SCliConn* pConn) {
SCliMsg* pCliMsg = pConn->data;
SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg);
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
STransConnCtx* pCtx = pCliMsg->ctx;
SRpcInfo* pTransInst = pCtx->pTransInst;
int msgLen = transMsgLenFromCont(pMsg->contLen);
SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg);
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
int msgLen = transMsgLenFromCont(pMsg->contLen);
if (!pConn->secured) {
char* buf = calloc(1, msgLen + sizeof(STransUserMsg));
memcpy(buf, (char*)pHead, msgLen);
STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen);
memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user));
memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret));
// to avoid mem leak
destroyUserdata(pMsg);
pMsg->pCont = (char*)buf + sizeof(STransMsgHead);
pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead);
pHead = (STransMsgHead*)buf;
pHead->secured = 1;
msgLen += sizeof(STransUserMsg);
}
pHead->msgType = pMsg->msgType;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
tDebug("client conn %p %s is send to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port));
tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType),
inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port));
uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb);
}
static void clientConnCb(uv_connect_t* req, int status) {
@ -407,6 +436,9 @@ static void clientConnCb(uv_connect_t* req, int status) {
int addrlen = sizeof(pConn->addr);
uv_tcp_getpeername((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->addr, &addrlen);
addrlen = sizeof(pConn->locaddr);
uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen);
tTrace("client conn %p create", pConn);
assert(pConn->stream == req->handle);
@ -428,10 +460,6 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
tTrace("client msg tran time cost: %" PRIu64 "", el);
et = taosGetTimestampUs();
// if (pMsg->msg.handle != NULL) {
// // handle
//}
STransConnCtx* pCtx = pMsg->ctx;
SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) {
@ -557,7 +585,7 @@ static SCliThrdObj* createThrdObj() {
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, clientAsyncCb);
pThrd->timer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pThrd->timer);

10
source/libs/transport/src/transComm.c
View File

@ -211,7 +211,8 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
/*
* formate of data buffer:
* |<--------------------------data from socket------------------------------->|
* |<------STransMsgHead------->|<-------------------other data--------------->|
* |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user
* info--->|
*/
static const int CAPACITY = 1024;
@ -239,6 +240,9 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
}
return 0;
}
int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {}
int transUnpackMsg(STransMsgHead* msgHead) {}
int transDestroyBuffer(SConnBuffer* buf) {
if (buf->cap > 0) {
tfree(buf->buf);
@ -246,9 +250,7 @@ int transDestroyBuffer(SConnBuffer* buf) {
transClearBuffer(buf);
}
SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) {
static int sz = 10;
SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) {
SAsyncPool* pool = calloc(1, sizeof(SAsyncPool));
pool->index = 0;
pool->nAsync = sz;

90
source/libs/transport/src/transSrv.c
View File

@ -31,9 +31,11 @@ typedef struct SSrvConn {
void* pTransInst; // rpc init
void* ahandle; //
void* hostThrd;
void* pSrvMsg;
SArray* srvMsgs;
// void* pSrvMsg;
struct sockaddr_in addr;
struct sockaddr_in locaddr;
// SRpcMsg sendMsg;
// del later
@ -94,6 +96,7 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf)
static void uvWorkerAsyncCb(uv_async_t* handle);
static void uvAcceptAsyncCb(uv_async_t* handle);
static void uvStartSendRespInternal(SSrvMsg* smsg);
static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb);
static void uvStartSendResp(SSrvMsg* msg);
@ -231,6 +234,11 @@ static void uvHandleReq(SSrvConn* pConn) {
p->chandle = NULL;
STransMsgHead* pHead = (STransMsgHead*)p->msg;
if (pHead->secured == 1) {
STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg));
memcpy(pConn->user, uMsg->user, tListLen(uMsg->user));
memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret));
}
pConn->inType = pHead->msgType;
assert(transIsReq(pHead->msgType));
@ -258,8 +266,9 @@ static void uvHandleReq(SSrvConn* pConn) {
transClearBuffer(&pConn->readBuf);
pConn->ref++;
tDebug("server conn %p %s received from %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port));
tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType),
inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port));
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
// uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth
@ -305,14 +314,19 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
void uvOnWriteCb(uv_write_t* req, int status) {
SSrvConn* conn = req->data;
SSrvMsg* smsg = conn->pSrvMsg;
destroySmsg(smsg);
conn->pSrvMsg = NULL;
transClearBuffer(&conn->readBuf);
if (status == 0) {
tTrace("server conn %p data already was written on stream", conn);
assert(taosArrayGetSize(conn->srvMsgs) >= 1);
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0);
taosArrayRemove(conn->srvMsgs, 0);
destroySmsg(msg);
// send second data, just use for push
if (taosArrayGetSize(conn->srvMsgs) > 0) {
msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0);
uvStartSendRespInternal(msg);
}
} else {
tError("server conn %p failed to write data, %s", conn, uv_err_name(status));
//
@ -331,41 +345,55 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
// impl later;
tTrace("server conn %p prepare to send resp", smsg->pConn);
SRpcMsg* pMsg = &smsg->msg;
SSrvConn* pConn = smsg->pConn;
SRpcMsg* pMsg = &smsg->msg;
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
pMsg->contLen = 0;
}
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
pHead->secured = pMsg->code == 0 ? 1 : 0; //
pHead->msgType = smsg->pConn->inType + 1;
pHead->code = htonl(pMsg->code);
// add more info
char* msg = (char*)pHead;
int32_t len = transMsgLenFromCont(pMsg->contLen);
if (transCompressMsg(msg, len, NULL)) {
// impl later
}
tDebug("server conn %p %s is sent to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port));
tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType),
inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port));
pHead->msgLen = htonl(len);
wb->base = msg;
wb->len = len;
}
static void uvStartSendResp(SSrvMsg* smsg) {
// impl
static void uvStartSendRespInternal(SSrvMsg* smsg) {
uv_buf_t wb;
uvPrepareSendData(smsg, &wb);
SSrvConn* pConn = smsg->pConn;
uv_timer_stop(pConn->pTimer);
pConn->pSrvMsg = smsg;
// pConn->pSrvMsg = smsg;
// conn->pWriter->data = smsg;
uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb);
// SRpcMsg* rpcMsg = smsg->msg;
}
static void uvStartSendResp(SSrvMsg* smsg) {
// impl
SSrvConn* pConn = smsg->pConn;
if (taosArrayGetSize(pConn->srvMsgs) > 0) {
tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
taosArrayPush(pConn->srvMsgs, &smsg);
return;
}
taosArrayPush(pConn->srvMsgs, &smsg);
uvStartSendRespInternal(smsg);
return;
}
static void destroySmsg(SSrvMsg* smsg) {
@ -487,13 +515,23 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_os_fd_t fd;
uv_fileno((const uv_handle_t*)pConn->pTcp, &fd);
tTrace("server conn %p created, fd: %d", pConn, fd);
int addrlen = sizeof(pConn->addr);
if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) {
tError("server conn %p failed to get peer info", pConn);
destroyConn(pConn, true);
} else {
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
return;
}
addrlen = sizeof(pConn->locaddr);
if (0 != uv_tcp_getsockname(pConn->pTcp, (struct sockaddr*)&pConn->locaddr, &addrlen)) {
tError("server conn %p failed to get local info", pConn);
destroyConn(pConn, true);
return;
}
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
} else {
tDebug("failed to create new connection");
destroyConn(pConn, true);
@ -522,7 +560,7 @@ static bool addHandleToWorkloop(void* arg) {
QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb);
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
return true;
}
@ -562,6 +600,7 @@ void* workerThread(void* arg) {
static SSrvConn* createConn() {
SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn));
pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); //
tTrace("conn %p created", pConn);
++pConn->ref;
return pConn;
@ -576,8 +615,15 @@ static void destroyConn(SSrvConn* conn, bool clear) {
return;
}
transDestroyBuffer(&conn->readBuf);
destroySmsg(conn->pSrvMsg);
conn->pSrvMsg = NULL;
for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) {
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i);
destroySmsg(msg);
}
taosArrayDestroy(conn->srvMsgs);
// destroySmsg(conn->pSrvMsg);
// conn->pSrvMsg = NULL;
if (clear) {
uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);

37
source/libs/transport/test/CMakeLists.txt
View File

@ -3,12 +3,13 @@ add_executable(client "")
add_executable(server "")
add_executable(transUT "")
add_executable(syncClient "")
add_executable(pushClient "")
add_executable(pushServer "")
target_sources(transUT
PRIVATE
"transUT.cc"
)
target_sources(transportTest
PRIVATE
"transportTests.cc"
@ -26,6 +27,15 @@ target_sources (syncClient
"syncClient.c"
)
target_sources(pushClient
PRIVATE
"pushClient.c"
)
target_sources(pushServer
PRIVATE
"pushServer.c"
)
target_include_directories(transportTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/transport"
@ -92,4 +102,29 @@ target_link_libraries (syncClient
transport
)
target_include_directories(pushClient
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (pushClient
os
util
common
gtest_main
transport
)
target_include_directories(pushServer
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (pushServer
os
util
common
gtest_main
transport
)

15
source/libs/transport/test/pushClient.c
View File

@ -49,9 +49,8 @@ static int tcount = 0;
typedef struct SPushArg {
tsem_t sem;
} SPushArg;
// ping
int pushCallback(void *arg, SRpcMsg *msg) {
SPushArg *push = arg;
tsem_post(&push->sem);
@ -59,7 +58,8 @@ int pushCallback(void *arg, SRpcMsg *msg) {
SRpcPush *createPushArg() {
SRpcPush *push = calloc(1, sizeof(SRpcPush));
push->arg = calloc(1, sizeof(SPushArg));
tsem_init(&push->arg->sem, 0, 0);
tsem_init(&(((SPushArg *)push->arg)->sem), 0, 0);
push->callback = pushCallback;
return push;
}
@ -83,14 +83,17 @@ static void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
rpcMsg.push = push;
;
// tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
int64_t start = taosGetTimestampUs();
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem); // ping->pong
// tsem_wait(&pInfo->rspSem);
tsem_wait(&push->sem);
SPushArg *arg = push->arg;
/// e
tsem_wait(&arg->sem); // push callback
// query_fetch(client->h)
int64_t end = taosGetTimestampUs() - start;
if (end <= 100) {
u100++;

199
source/libs/transport/test/pushServer.c Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//#define _DEFAULT_SOURCE
#include "os.h"
#include "rpcLog.h"
#include "tglobal.h"
#include "tqueue.h"
#include "trpc.h"
int msgSize = 128;
int commit = 0;
int dataFd = -1;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
void processShellMsg() {
static int num = 0;
STaosQall *qall;
SRpcMsg * pRpcMsg, rpcMsg;
int type;
void * pvnode;
qall = taosAllocateQall();
while (1) {
int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode, NULL);
tDebug("%d shell msgs are received", numOfMsgs);
if (numOfMsgs <= 0) break;
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
}
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
if (num % 10000 == 0) {
tInfo("%d request have been written into disk", num);
}
}
taosResetQitems(qall);
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
rpcFreeCont(pRpcMsg->pCont);
memset(&rpcMsg, 0, sizeof(rpcMsg));
rpcMsg.pCont = rpcMallocCont(msgSize);
rpcMsg.contLen = msgSize;
rpcMsg.handle = pRpcMsg->handle;
rpcMsg.code = 0;
rpcSendResponse(&rpcMsg);
void *handle = pRpcMsg->handle;
taosFreeQitem(pRpcMsg);
{
// sleep(1);
SRpcMsg nRpcMsg = {0};
nRpcMsg.pCont = rpcMallocCont(msgSize);
nRpcMsg.contLen = msgSize;
nRpcMsg.handle = handle;
nRpcMsg.code = TSDB_CODE_CTG_NOT_READY;
rpcSendResponse(&nRpcMsg);
}
}
}
taosFreeQall(qall);
}
int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
// app shall retrieve the auth info based on meterID from DB or a data file
// demo code here only for simple demo
int ret = 0;
if (strcmp(meterId, "michael") == 0) {
*spi = 1;
*encrypt = 0;
strcpy(secret, "mypassword");
strcpy(ckey, "key");
} else if (strcmp(meterId, "jeff") == 0) {
*spi = 0;
*encrypt = 0;
} else {
ret = -1; // user not there
}
return ret;
}
void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SRpcMsg *pTemp;
pTemp = taosAllocateQitem(sizeof(SRpcMsg));
memcpy(pTemp, pMsg, sizeof(SRpcMsg));
tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp);
taosWriteQitem(qhandle, pTemp);
}
int main(int argc, char *argv[]) {
SRpcInit rpcInit;
char dataName[20] = "server.data";
taosBlockSIGPIPE();
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 7000;
rpcInit.label = "SER";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg;
rpcInit.sessions = 1000;
rpcInit.idleTime = 2 * 1500;
rpcInit.afp = retrieveAuthInfo;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
rpcInit.localPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
msgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) {
rpcInit.sessions = atoi(argv[++i]);
} else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) {
tsCompressMsgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) {
commit = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
rpcDebugFlag = atoi(argv[++i]);
dDebugFlag = rpcDebugFlag;
uDebugFlag = rpcDebugFlag;
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit);
printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
printf(" [-h help]: print out this help\n\n");
exit(0);
}
}
tsAsyncLog = 0;
rpcInit.connType = TAOS_CONN_SERVER;
taosInitLog("server.log", 100000, 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {
tError("failed to start RPC server");
return -1;
}
// sleep(5);
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
taosAddIntoQset(qset, qhandle, NULL);
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
remove(dataName);
}
return 0;
}

42
source/nodes/src/nodesTraverseFuncs.c
View File

@ -20,18 +20,22 @@ typedef enum ETraversalOrder {
TRAVERSAL_POSTORDER
} ETraversalOrder;
static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext);
static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext);
static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) {
static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) {
if (NULL == pNode) {
return true;
return DEAL_RES_CONTINUE;
}
if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) {
return false;
EDealRes res = DEAL_RES_CONTINUE;
if (TRAVERSAL_PREORDER == order) {
res = walker(pNode, pContext);
if (DEAL_RES_CONTINUE != res) {
return res;
}
}
bool res = true;
switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN:
case QUERY_NODE_VALUE:
@ -41,7 +45,7 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke
case QUERY_NODE_OPERATOR: {
SOperatorNode* pOpNode = (SOperatorNode*)pNode;
res = walkNode(pOpNode->pLeft, order, walker, pContext);
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pOpNode->pRight, order, walker, pContext);
}
break;
@ -61,10 +65,10 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke
case QUERY_NODE_JOIN_TABLE: {
SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode;
res = walkNode(pJoinTableNode->pLeft, order, walker, pContext);
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pJoinTableNode->pRight, order, walker, pContext);
}
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext);
}
break;
@ -84,13 +88,13 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke
case QUERY_NODE_INTERVAL_WINDOW: {
SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode;
res = walkNode(pInterval->pInterval, order, walker, pContext);
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pInterval->pOffset, order, walker, pContext);
}
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pInterval->pSliding, order, walker, pContext);
}
if (res) {
if (DEAL_RES_ERROR != res) {
res = walkNode(pInterval->pFill, order, walker, pContext);
}
break;
@ -108,21 +112,21 @@ static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walke
break;
}
if (res && TRAVERSAL_POSTORDER == order) {
if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) {
res = walker(pNode, pContext);
}
return res;
}
static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) {
static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) {
SNode* node;
FOREACH(node, pNodeList) {
if (!walkNode(node, order, walker, pContext)) {
return false;
if (DEAL_RES_ERROR == walkNode(node, order, walker, pContext)) {
return DEAL_RES_ERROR;
}
}
return true;
return DEAL_RES_CONTINUE;
}
void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
@ -140,7 +144,3 @@ void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContex
void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) {
(void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext);
}
bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) {
}

22
source/nodes/src/nodesUtilFuncs.c
View File

@ -76,7 +76,7 @@ SNode* nodesMakeNode(ENodeType type) {
return NULL;
}
static bool destroyNode(SNode* pNode, void* pContext) {
static EDealRes destroyNode(SNode* pNode, void* pContext) {
switch (nodeType(pNode)) {
case QUERY_NODE_VALUE:
tfree(((SValueNode*)pNode)->literal);
@ -85,6 +85,7 @@ static bool destroyNode(SNode* pNode, void* pContext) {
break;
}
tfree(pNode);
return DEAL_RES_CONTINUE;
}
void nodesDestroyNode(SNode* pNode) {
@ -116,9 +117,23 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) {
pList->pTail->pNext = p;
}
pList->pTail = p;
++(pList->length);
return pList;
}
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
if (NULL == pCell->pPrev) {
pList->pHead = pCell->pNext;
} else {
pCell->pPrev->pNext = pCell->pNext;
pCell->pNext->pPrev = pCell->pPrev;
}
SListCell* pNext = pCell->pNext;
tfree(pCell);
--(pList->length);
return pNext;
}
SNode* nodesListGetNode(SNodeList* pList, int32_t index) {
SNode* node;
FOREACH(node, pList) {
@ -137,6 +152,11 @@ void nodesDestroyList(SNodeList* pList) {
tfree(pList);
}
bool nodesIsExprNode(const SNode* pNode) {
ENodeType type = nodeType(pNode);
return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);
}
bool nodesIsArithmeticOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_ADD: