xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: add connection type for tmq

This commit is contained in:
Liu Jicong 2022-04-14 14:42:51 +08:00
parent 4473d23633
commit 7adb395b85
13 changed files with 450 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
example/src/tmq.c
View File

@ -144,6 +144,7 @@ void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_
} }
tmq_t* build_consumer() { tmq_t* build_consumer() {
#if 0
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); assert(pConn != NULL);
@ -152,11 +153,15 @@ tmq_t* build_consumer() {
printf("error in use db, reason:%s\n", taos_errstr(pRes)); printf("error in use db, reason:%s\n", taos_errstr(pRes));
} }
taos_free_result(pRes); taos_free_result(pRes);
#endif
tmq_conf_t* conf = tmq_conf_new(); tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "group.id", "tg2"); tmq_conf_set(conf, "group.id", "tg2");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "td.connect.db", "abc1");
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
return tmq; return tmq;
} }

4
include/client/taos.h
View File

@ -247,10 +247,10 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
// will be removed in 3.0 #if 1
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
#endif
// will replace last one
DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);

2
include/common/tmsg.h
View File

@ -332,6 +332,7 @@ int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
void* taosDecodeSEpSet(void* buf, SEpSet* pEp); void* taosDecodeSEpSet(void* buf, SEpSet* pEp);
typedef struct { typedef struct {
int8_t connType;
int32_t pid; int32_t pid;
char app[TSDB_APP_NAME_LEN]; char app[TSDB_APP_NAME_LEN];
char db[TSDB_DB_NAME_LEN]; char db[TSDB_DB_NAME_LEN];
@ -346,6 +347,7 @@ typedef struct {
int64_t clusterId; int64_t clusterId;
int32_t connId; int32_t connId;
int8_t superUser; int8_t superUser;
int8_t connType;
SEpSet epSet; SEpSet epSet;
char sVersion[128]; char sVersion[128];
} SConnectRsp; } SConnectRsp;

9
source/client/inc/clientInt.h
View File

@ -45,6 +45,11 @@ extern "C" {
#define HEARTBEAT_INTERVAL 1500 // ms #define HEARTBEAT_INTERVAL 1500 // ms
enum {
CONN_TYPE__QUERY = 1,
CONN_TYPE__TMQ,
};
typedef struct SAppInstInfo SAppInstInfo; typedef struct SAppInstInfo SAppInstInfo;
typedef struct { typedef struct {
@ -132,9 +137,9 @@ typedef struct STscObj {
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char ver[128]; char ver[128];
int8_t connType;
int32_t acctId; int32_t acctId;
uint32_t connId; uint32_t connId;
int32_t connType;
uint64_t id; // ref ID returned by taosAddRef uint64_t id; // ref ID returned by taosAddRef
TdThreadMutex mutex; // used to protect the operation on db TdThreadMutex mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection int32_t numOfReqs; // number of sqlObj bound to this connection
@ -272,7 +277,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
void initMsgHandleFp(); void initMsgHandleFp();
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port); uint16_t port, int connType);
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);

15
source/client/src/clientHb.c
View File

@ -23,6 +23,8 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread(); static int32_t hbCreateThread();
static void hbStopThread(); static void hbStopThread();
static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
@ -297,11 +299,10 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
void hbMgrInitMqHbHandle() { void hbMgrInitMqHbHandle() {
clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
clientHbMgr.reqHandle[HEARTBEAT_TYPE_MQ] = hbMqHbReqHandle; clientHbMgr.reqHandle[HEARTBEAT_TYPE_MQ] = hbMqHbReqHandle;
clientHbMgr.rspHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbRspHandle; clientHbMgr.rspHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbRspHandle;
clientHbMgr.rspHandle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; clientHbMgr.rspHandle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle;
} }
@ -568,7 +569,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
SClientHbKey connKey = { SClientHbKey connKey = {
.connId = connId, .connId = connId,
.hbType = HEARTBEAT_TYPE_QUERY, .hbType = hbType,
}; };
SHbConnInfo info = {0}; SHbConnInfo info = {0};
@ -578,16 +579,14 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3
*pClusterId = clusterId; *pClusterId = clusterId;
info.param = pClusterId; info.param = pClusterId;
break; return hbRegisterConnImpl(pAppHbMgr, connKey, &info);
} }
case HEARTBEAT_TYPE_MQ: { case HEARTBEAT_TYPE_MQ: {
break; return 0;
} }
default: default:
break; return 0;
} }
return hbRegisterConnImpl(pAppHbMgr, connKey, &info);
} }
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {

17
source/client/src/clientImpl.c
View File

@ -11,7 +11,7 @@
#include "tref.h" #include "tref.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static bool stringLengthCheck(const char* str, size_t maxsize) { static bool stringLengthCheck(const char* str, size_t maxsize) {
@ -40,10 +40,10 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
} }
static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo); SAppInstInfo* pAppInfo, int connType);
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port) { uint16_t port, int connType) {
if (taos_init() != TSDB_CODE_SUCCESS) { if (taos_init() != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
} }
@ -111,7 +111,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
taosThreadMutexUnlock(&appInfo.mutex); taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key); taosMemoryFreeClear(key);
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
} }
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) { int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) {
@ -425,7 +425,7 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
} }
STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo) { SAppInstInfo* pAppInfo, int connType) {
STscObj* pTscObj = createTscObj(user, auth, db, pAppInfo); STscObj* pTscObj = createTscObj(user, auth, db, pAppInfo);
if (NULL == pTscObj) { if (NULL == pTscObj) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -439,7 +439,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
return NULL; return NULL;
} }
SMsgSendInfo* body = buildConnectMsg(pRequest); SMsgSendInfo* body = buildConnectMsg(pRequest, connType);
int64_t transporterId = 0; int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
@ -462,7 +462,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
return pTscObj; return pTscObj;
} }
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) {
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pMsgSendInfo == NULL) { if (pMsgSendInfo == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -485,6 +485,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
} }
taosMemoryFreeClear(db); taosMemoryFreeClear(db);
connectReq.connType = connType;
connectReq.pid = htonl(appInfo.pid); connectReq.pid = htonl(appInfo.pid);
connectReq.startTime = htobe64(appInfo.startTime); connectReq.startTime = htobe64(appInfo.startTime);
tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
@ -570,7 +571,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
return NULL; return NULL;
} }
return taos_connect_internal(ip, user, NULL, auth, db, port); return taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY);
} }
TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,

4
source/client/src/clientMain.c
View File

@ -87,7 +87,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
pass = TSDB_DEFAULT_PASS; pass = TSDB_DEFAULT_PASS;
} }
return taos_connect_internal(ip, user, pass, NULL, db, port); return taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
} }
void taos_close(TAOS *taos) { void taos_close(TAOS *taos) {
@ -124,8 +124,10 @@ const char *taos_errstr(TAOS_RES *res) {
} }
void taos_free_result(TAOS_RES *res) { void taos_free_result(TAOS_RES *res) {
if (TD_RES_QUERY(res)) {
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest); destroyRequest(pRequest);
}
} }
int taos_field_count(TAOS_RES *res) { int taos_field_count(TAOS_RES *res) {

48
source/client/src/clientMsgHandler.c
View File

@ -13,13 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h" #include "catalog.h"
#include "tdef.h"
#include "tname.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "catalog.h" #include "os.h"
#include "query.h" #include "query.h"
#include "tdef.h"
#include "tname.h"
int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
@ -69,9 +69,9 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pTscObj->pAppInfo->clusterId = connectRsp.clusterId; pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
pTscObj->connType = HEARTBEAT_TYPE_QUERY; pTscObj->connType = connectRsp.connType;
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connectRsp.connId, connectRsp.clusterId, HEARTBEAT_TYPE_QUERY); hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connectRsp.connId, connectRsp.clusterId, connectRsp.connType);
// pRequest->body.resInfo.pRspMsg = pMsg->pData; // pRequest->body.resInfo.pRspMsg = pMsg->pData;
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
@ -82,7 +82,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
return 0; return 0;
} }
SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) {
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestObjRefId = pRequest->self;
@ -119,7 +119,9 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
pMsgSendInfo->msgInfo = pRequest->body.requestMsg; pMsgSendInfo->msgInfo = pRequest->body.requestMsg;
} }
pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)
? genericRspCallback
: handleRequestRspFp[TMSG_INDEX(pRequest->type)];
return pMsgSendInfo; return pMsgSendInfo;
} }
@ -133,7 +135,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SShowRsp showRsp = {0}; SShowRsp showRsp = {0};
tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp);
STableMetaRsp *pMetaMsg = &showRsp.tableMeta; STableMetaRsp* pMetaMsg = &showRsp.tableMeta;
taosMemoryFreeClear(pRequest->body.resInfo.pRspMsg); taosMemoryFreeClear(pRequest->body.resInfo.pRspMsg);
pRequest->body.resInfo.pRspMsg = pMsg->pData; pRequest->body.resInfo.pRspMsg = pMsg->pData;
@ -169,8 +171,8 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
} }
int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj *pRequest = param; SRequestObj* pRequest = param;
SReqResultInfo *pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
taosMemoryFreeClear(pResInfo->pRspMsg); taosMemoryFreeClear(pResInfo->pRspMsg);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -181,7 +183,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code)
assert(pMsg->len >= sizeof(SRetrieveTableRsp)); assert(pMsg->len >= sizeof(SRetrieveTableRsp));
SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)pMsg->pData;
pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->numOfRows = htonl(pRetrieve->numOfRows);
pRetrieve->precision = htons(pRetrieve->precision); pRetrieve->precision = htons(pRetrieve->precision);
@ -191,9 +193,9 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code)
pResInfo->completed = pRetrieve->completed; pResInfo->completed = pRetrieve->completed;
pResInfo->current = 0; pResInfo->current = 0;
// setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); // setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, tscDebug("0x%" PRIx64 " numOfRows:%d, complete:%d, qId:0x%" PRIx64, pRequest->self, pRetrieve->numOfRows,
pRetrieve->completed, pRequest->body.showInfo.execId); pRetrieve->completed, pRequest->body.showInfo.execId);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
@ -216,7 +218,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pResInfo->pRspMsg = pMsg->pData; pResInfo->pRspMsg = pMsg->pData;
SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *) pMsg->pData; SVShowTablesFetchRsp* pFetchRsp = (SVShowTablesFetchRsp*)pMsg->pData;
pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows);
pFetchRsp->precision = htons(pFetchRsp->precision); pFetchRsp->precision = htons(pFetchRsp->precision);
@ -225,9 +227,9 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pResInfo->pData = pFetchRsp->data; pResInfo->pData = pFetchRsp->data;
pResInfo->current = 0; pResInfo->current = 0;
// setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); // setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows, tscDebug("0x%" PRIx64 " numOfRows:%d, complete:%d, qId:0x%" PRIx64, pRequest->self, pFetchRsp->numOfRows,
pFetchRsp->completed, pRequest->body.showInfo.execId); pFetchRsp->completed, pRequest->body.showInfo.execId);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
@ -251,12 +253,13 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
if (TSDB_CODE_MND_DB_NOT_EXIST == code) { if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
SUseDbRsp usedbRsp = {0}; SUseDbRsp usedbRsp = {0};
tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
struct SCatalog *pCatalog = NULL; struct SCatalog* pCatalog = NULL;
if (usedbRsp.vgVersion >= 0) { if (usedbRsp.vgVersion >= 0) {
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
tstrerror(code));
} else { } else {
catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid); catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid);
} }
@ -276,7 +279,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
SName name = {0}; SName name = {0};
tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB);
SUseDbOutput output = {0}; SUseDbOutput output = {0};
code = queryBuildUseDbOutput(&output, &usedbRsp); code = queryBuildUseDbOutput(&output, &usedbRsp);
@ -288,11 +291,12 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
tscError("failed to build use db output since %s", terrstr()); tscError("failed to build use db output since %s", terrstr());
} else { } else {
struct SCatalog *pCatalog = NULL; struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
tstrerror(code));
} else { } else {
catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup); catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
} }

10
source/client/src/tmq.c
View File

@ -357,7 +357,15 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (pTmq == NULL) { if (pTmq == NULL) {
return NULL; return NULL;
} }
pTmq->pTscObj = taos_connect(conf->ip, conf->user, conf->pass, conf->db, conf->port); const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user;
const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass;
ASSERT(user);
ASSERT(pass);
ASSERT(conf->db);
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) return NULL;
pTmq->inWaiting = 0; pTmq->inWaiting = 0;
pTmq->status = 0; pTmq->status = 0;

4
source/common/src/tmsg.c
View File

@ -2532,6 +2532,7 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI8(&encoder, pReq->connType) < 0) return -1;
if (tEncodeI32(&encoder, pReq->pid) < 0) return -1; if (tEncodeI32(&encoder, pReq->pid) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; if (tEncodeCStr(&encoder, pReq->app) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
@ -2548,6 +2549,7 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->connType) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
@ -2567,6 +2569,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
if (tEncodeI64(&encoder, pRsp->clusterId) < 0) return -1; if (tEncodeI64(&encoder, pRsp->clusterId) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->connId) < 0) return -1; if (tEncodeI32(&encoder, pRsp->connId) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->connType) < 0) return -1;
if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->sVersion) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVersion) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
@ -2585,6 +2588,7 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
if (tDecodeI64(&decoder, &pRsp->clusterId) < 0) return -1; if (tDecodeI64(&decoder, &pRsp->clusterId) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->connId) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->connId) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->connType) < 0) return -1;
if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1;
if (tDecodeCStrTo(&decoder, pRsp->sVersion) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVersion) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);

18
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -30,6 +30,7 @@
typedef struct { typedef struct {
int32_t id; int32_t id;
int8_t connType;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc
int64_t appStartTimeMs; // app start time int64_t appStartTimeMs; // app start time
@ -44,8 +45,8 @@ typedef struct {
SQueryDesc *pQueries; SQueryDesc *pQueries;
} SConnObj; } SConnObj;
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid, static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port,
const char *app, int64_t startTime); int32_t pid, const char *app, int64_t startTime);
static void mndFreeConn(SConnObj *pConn); static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
@ -93,8 +94,8 @@ void mndCleanupProfile(SMnode *pMnode) {
} }
} }
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid, static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port,
const char *app, int64_t startTime) { int32_t pid, const char *app, int64_t startTime) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1);
@ -102,6 +103,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, ui
if (startTime == 0) startTime = taosGetTimestampMs(); if (startTime == 0) startTime = taosGetTimestampMs();
SConnObj connObj = {.id = connId, SConnObj connObj = {.id = connId,
.connType = connType,
.appStartTimeMs = startTime, .appStartTimeMs = startTime,
.pid = pid, .pid = pid,
.ip = ip, .ip = ip,
@ -159,7 +161,7 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
} }
void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) { void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
SConnObj* pConn = NULL; SConnObj *pConn = NULL;
bool hasNext = taosCacheIterNext(pIter); bool hasNext = taosCacheIterNext(pIter);
if (hasNext) { if (hasNext) {
size_t dataLen = 0; size_t dataLen = 0;
@ -210,8 +212,8 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
} }
} }
pConn = pConn = mndCreateConn(pMnode, pReq->user, connReq.connType, pReq->clientIp, pReq->clientPort, connReq.pid,
mndCreateConn(pMnode, pReq->user, pReq->clientIp, pReq->clientPort, connReq.pid, connReq.app, connReq.startTime); connReq.app, connReq.startTime);
if (pConn == NULL) { if (pConn == NULL) {
mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr()); mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
goto CONN_OVER; goto CONN_OVER;
@ -222,6 +224,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
connectRsp.superUser = pUser->superUser; connectRsp.superUser = pUser->superUser;
connectRsp.clusterId = pMnode->clusterId; connectRsp.clusterId = pMnode->clusterId;
connectRsp.connId = pConn->id; connectRsp.connId = pConn->id;
connectRsp.connType = connReq.connType;
snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo,
gitinfo); gitinfo);
@ -343,7 +346,6 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
return -1; return -1;
} }
SClientHbBatchRsp batchRsp = {0}; SClientHbBatchRsp batchRsp = {0};
batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp));

108
tests/test/c/tmqDemo.c
View File

@ -13,17 +13,15 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
// clang-format off
#include <assert.h> #include <assert.h>
#include <dirent.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#include <time.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <time.h>
#include <unistd.h> #include <unistd.h>
#include <stdlib.h>
#include <dirent.h>
#include "taos.h" #include "taos.h"
#include "taoserror.h" #include "taoserror.h"
@ -36,12 +34,7 @@
#define MAX_SQL_STR_LEN (1024 * 1024) #define MAX_SQL_STR_LEN (1024 * 1024)
#define MAX_ROW_STR_LEN (16 * 1024) #define MAX_ROW_STR_LEN (16 * 1024)
enum _RUN_MODE { enum _RUN_MODE { TMQ_RUN_INSERT_AND_CONSUME, TMQ_RUN_ONLY_INSERT, TMQ_RUN_ONLY_CONSUME, TMQ_RUN_MODE_BUTT };
TMQ_RUN_INSERT_AND_CONSUME,
TMQ_RUN_ONLY_INSERT,
TMQ_RUN_ONLY_CONSUME,
TMQ_RUN_MODE_BUTT
};
typedef struct { typedef struct {
char dbName[32]; char dbName[32];
@ -125,11 +118,9 @@ static void printHelp() {
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
void parseArgument(int32_t argc, char *argv[]) { void parseArgument(int32_t argc, char* argv[]) {
g_stConfInfo.startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 g_stConfInfo.startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000
for (int32_t i = 1; i < argc; i++) { for (int32_t i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
printHelp(); printHelp();
@ -198,37 +189,36 @@ static int running = 1;
/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ /*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/
// calc dir size (not include itself 4096Byte) // calc dir size (not include itself 4096Byte)
int64_t getDirectorySize(char *dir) int64_t getDirectorySize(char* dir) {
{
TdDirPtr pDir; TdDirPtr pDir;
TdDirEntryPtr pDirEntry; TdDirEntryPtr pDirEntry;
int64_t totalSize=0; int64_t totalSize = 0;
if ((pDir = taosOpenDir(dir)) == NULL) { if ((pDir = taosOpenDir(dir)) == NULL) {
fprintf(stderr, "Cannot open dir: %s\n", dir); fprintf(stderr, "Cannot open dir: %s\n", dir);
return -1; return -1;
} }
//lstat(dir, &statbuf); // lstat(dir, &statbuf);
//totalSize+=statbuf.st_size; // totalSize+=statbuf.st_size;
while ((pDirEntry = taosReadDir(pDir)) != NULL) { while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char subdir[1024]; char subdir[1024];
char* fileName = taosGetDirEntryName(pDirEntry); char* fileName = taosGetDirEntryName(pDirEntry);
sprintf(subdir, "%s/%s", dir, fileName); sprintf(subdir, "%s/%s", dir, fileName);
//printf("===d_name: %s\n", entry->d_name); // printf("===d_name: %s\n", entry->d_name);
if (taosIsDir(subdir)) { if (taosIsDir(subdir)) {
if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) {
continue; continue;
} }
int64_t subDirSize = getDirectorySize(subdir); int64_t subDirSize = getDirectorySize(subdir);
totalSize+=subDirSize; totalSize += subDirSize;
} else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file
int64_t file_size = 0; int64_t file_size = 0;
taosStatFile(subdir, &file_size, NULL); taosStatFile(subdir, &file_size, NULL);
totalSize+=file_size; totalSize += file_size;
} }
} }
@ -236,18 +226,17 @@ int64_t getDirectorySize(char *dir)
return totalSize; return totalSize;
} }
int queryDB(TAOS* taos, char* command) {
int queryDB(TAOS *taos, char *command) { TAOS_RES* pRes = taos_query(taos, command);
TAOS_RES *pRes = taos_query(taos, command);
int code = taos_errno(pRes); int code = taos_errno(pRes);
//if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
if (code != 0) { if (code != 0) {
pError("failed to reason:%s, sql: %s", tstrerror(code), command); pError("failed to reason:%s, sql: %s", tstrerror(code), command);
taos_free_result(pRes); taos_free_result(pRes);
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
return 0 ; return 0;
} }
int32_t init_env() { int32_t init_env() {
@ -282,19 +271,19 @@ int32_t init_env() {
int32_t dataLen = 0; int32_t dataLen = 0;
int32_t sqlLen = 0; int32_t sqlLen = 0;
sqlLen += sprintf(sqlStr+sqlLen, "create stable if not exists %s (ts timestamp, ", g_stConfInfo.stbName); sqlLen += sprintf(sqlStr + sqlLen, "create stable if not exists %s (ts timestamp, ", g_stConfInfo.stbName);
for (int32_t i = 0; i < g_stConfInfo.numOfColumn; i++) { for (int32_t i = 0; i < g_stConfInfo.numOfColumn; i++) {
if (i == g_stConfInfo.numOfColumn - 1) { if (i == g_stConfInfo.numOfColumn - 1) {
sqlLen += sprintf(sqlStr+sqlLen, "c%d int) ", i); sqlLen += sprintf(sqlStr + sqlLen, "c%d int) ", i);
memcpy(g_pRowValue + dataLen, "66778899", strlen("66778899")); memcpy(g_pRowValue + dataLen, "66778899", strlen("66778899"));
dataLen += strlen("66778899"); dataLen += strlen("66778899");
} else { } else {
sqlLen += sprintf(sqlStr+sqlLen, "c%d int, ", i); sqlLen += sprintf(sqlStr + sqlLen, "c%d int, ", i);
memcpy(g_pRowValue + dataLen, "66778899, ", strlen("66778899, ")); memcpy(g_pRowValue + dataLen, "66778899, ", strlen("66778899, "));
dataLen += strlen("66778899, "); dataLen += strlen("66778899, ");
} }
} }
sqlLen += sprintf(sqlStr+sqlLen, "tags (t0 int)"); sqlLen += sprintf(sqlStr + sqlLen, "tags (t0 int)");
pRes = taos_query(pConn, sqlStr); pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
@ -313,7 +302,7 @@ int32_t init_env() {
taos_free_result(pRes); taos_free_result(pRes);
} }
//const char* sql = "select * from tu1"; // const char* sql = "select * from tu1";
sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName); sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName);
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/
pRes = taos_query(pConn, sqlStr); pRes = taos_query(pConn, sqlStr);
@ -327,6 +316,7 @@ int32_t init_env() {
} }
tmq_t* build_consumer() { tmq_t* build_consumer() {
#if 0
char sqlStr[1024] = {0}; char sqlStr[1024] = {0};
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
@ -338,10 +328,16 @@ tmq_t* build_consumer() {
printf("error in use db, reason:%s\n", taos_errstr(pRes)); printf("error in use db, reason:%s\n", taos_errstr(pRes));
} }
taos_free_result(pRes); taos_free_result(pRes);
#endif
tmq_conf_t* conf = tmq_conf_new(); tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "group.id", "tg2"); tmq_conf_set(conf, "group.id", "tg2");
tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
assert(tmq);
tmq_conf_destroy(conf);
return tmq; return tmq;
} }
@ -417,7 +413,9 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog
} else { } else {
printf("{consume success: %d}", totalMsgs); printf("{consume success: %d}", totalMsgs);
} }
taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, (double)walLogSize / 1024.0 / batchCnt); taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime,
(double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime,
(double)walLogSize / 1024.0 / batchCnt);
err = tmq_consumer_close(tmq); err = tmq_consumer_close(tmq);
if (err) { if (err) {
@ -474,7 +472,7 @@ int32_t syncWriteData() {
} }
int code = queryDB(pConn, buffer); int code = queryDB(pConn, buffer);
if (0 != code){ if (0 != code) {
fprintf(stderr, "insert data error!\n"); fprintf(stderr, "insert data error!\n");
taosMemoryFreeClear(buffer); taosMemoryFreeClear(buffer);
return -1; return -1;
@ -492,7 +490,6 @@ int32_t syncWriteData() {
return totalMsgs; return totalMsgs;
} }
// sync insertion // sync insertion
int32_t syncWriteDataByRatio() { int32_t syncWriteDataByRatio() {
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
@ -535,7 +532,7 @@ int32_t syncWriteDataByRatio() {
if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) {
continue; continue;
} }
} else if (1 == tID){ } else if (1 == tID) {
tmp_time = tsOfT2; tmp_time = tsOfT2;
if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) {
continue; continue;
@ -553,7 +550,7 @@ int32_t syncWriteDataByRatio() {
if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) {
break; break;
} }
} else if (1 == tID){ } else if (1 == tID) {
insertedOfT2++; insertedOfT2++;
if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) {
break; break;
@ -566,7 +563,7 @@ int32_t syncWriteDataByRatio() {
} }
int code = queryDB(pConn, buffer); int code = queryDB(pConn, buffer);
if (0 != code){ if (0 != code) {
fprintf(stderr, "insert data error!\n"); fprintf(stderr, "insert data error!\n");
taosMemoryFreeClear(buffer); taosMemoryFreeClear(buffer);
return -1; return -1;
@ -574,24 +571,26 @@ int32_t syncWriteDataByRatio() {
if (0 == tID) { if (0 == tID) {
tsOfT1 = tmp_time; tsOfT1 = tmp_time;
} else if (1 == tID){ } else if (1 == tID) {
tsOfT2 = tmp_time; tsOfT2 = tmp_time;
} }
totalMsgs++; totalMsgs++;
} }
} }
pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl,
g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2);
taosMemoryFreeClear(buffer); taosMemoryFreeClear(buffer);
return totalMsgs; return totalMsgs;
} }
void printParaIntoFile() { void printParaIntoFile() {
// FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); // FILE *fp = fopen(g_stConfInfo.resultFileName, "a");
TdFilePtr pFile = taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); TdFilePtr pFile =
taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM);
if (NULL == pFile) { if (NULL == pFile) {
fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName);
exit -1; exit - 1;
}; };
g_fp = pFile; g_fp = pFile;
@ -615,12 +614,17 @@ void printParaIntoFile() {
taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "###################################################################\n");
taosFprintfFile(pFile, "|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------|\n"); taosFprintfFile(pFile,
taosFprintfFile(pFile, "|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) |\n"); "|-------------------------------insert "
"info-----------------------------|--------------------------------consume "
"info---------------------------------|\n");
taosFprintfFile(pFile,
"|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume "
"time(s) | msgs/s | MB/s | avg msg size(KB) |\n");
taosFprintfFile(g_fp, "|%10d", g_stConfInfo.batchNumOfRow); taosFprintfFile(g_fp, "|%10d", g_stConfInfo.batchNumOfRow);
} }
int main(int32_t argc, char *argv[]) { int main(int32_t argc, char* argv[]) {
parseArgument(argc, argv); parseArgument(argc, argv);
printParaIntoFile(); printParaIntoFile();
@ -636,7 +640,6 @@ int main(int32_t argc, char *argv[]) {
int32_t totalMsgs = 0; int32_t totalMsgs = 0;
if (g_stConfInfo.runMode != TMQ_RUN_ONLY_CONSUME) { if (g_stConfInfo.runMode != TMQ_RUN_ONLY_CONSUME) {
int64_t startTs = taosGetTimestampUs(); int64_t startTs = taosGetTimestampUs();
if (1 == g_stConfInfo.ratio) { if (1 == g_stConfInfo.ratio) {
totalMsgs = syncWriteData(); totalMsgs = syncWriteData();
@ -662,21 +665,22 @@ int main(int32_t argc, char *argv[]) {
float rowsSpeed = totalRows / seconds; float rowsSpeed = totalRows / seconds;
float msgsSpeed = totalMsgs / seconds; float msgsSpeed = totalMsgs / seconds;
if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) { if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) {
walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath);
if (walLogSize <= 0) { if (walLogSize <= 0) {
printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); printf("%s size incorrect!", g_stConfInfo.vnodeWalPath);
exit(-1); exit(-1);
} else { } else {
pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize / (1024 * 1024.0));
} }
} }
if (0 == g_stConfInfo.simCase) { if (0 == g_stConfInfo.simCase) {
pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows,
totalMsgs, seconds, rowsSpeed, msgsSpeed);
} }
taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, (double)walLogSize/(1024 * 1024.0)); taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed,
(double)walLogSize / (1024 * 1024.0));
} }
if (g_stConfInfo.runMode == TMQ_RUN_ONLY_INSERT) { if (g_stConfInfo.runMode == TMQ_RUN_ONLY_INSERT) {
@ -685,7 +689,7 @@ int main(int32_t argc, char *argv[]) {
tmq_t* tmq = build_consumer(); tmq_t* tmq = build_consumer();
tmq_list_t* topic_list = build_topic_list(); tmq_list_t* topic_list = build_topic_list();
if ((NULL == tmq) || (NULL == topic_list)){ if ((NULL == tmq) || (NULL == topic_list)) {
return -1; return -1;
} }

75
tests/test/c/tmqSim.c
View File

@ -191,11 +191,11 @@ void parseInputString() {
} }
token = strtok(g_stConfInfo.topicString1, delim); token = strtok(g_stConfInfo.topicString1, delim);
while(token != NULL) { while (token != NULL) {
//printf("%s\n", token ); // printf("%s\n", token );
strcpy(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1], token); strcpy(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1], token);
ltrim(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1]); ltrim(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1]);
//printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
g_stConfInfo.numOfTopic1++; g_stConfInfo.numOfTopic1++;
token = strtok(NULL, delim); token = strtok(NULL, delim);
@ -219,15 +219,16 @@ void parseInputString() {
} }
token = strtok(g_stConfInfo.keyString1, delim); token = strtok(g_stConfInfo.keyString1, delim);
while(token != NULL) { while (token != NULL) {
//printf("%s\n", token ); // printf("%s\n", token );
{ {
char* pstr = token; char* pstr = token;
ltrim(pstr); ltrim(pstr);
char *ret = strchr(pstr, ch); char* ret = strchr(pstr, ch);
memcpy(g_stConfInfo.key1[g_stConfInfo.numOfKey1], pstr, ret-pstr); memcpy(g_stConfInfo.key1[g_stConfInfo.numOfKey1], pstr, ret - pstr);
strcpy(g_stConfInfo.value1[g_stConfInfo.numOfKey1], ret+1); strcpy(g_stConfInfo.value1[g_stConfInfo.numOfKey1], ret + 1);
//printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], g_stConfInfo.value[g_stConfInfo.numOfKey]); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
// g_stConfInfo.value[g_stConfInfo.numOfKey]);
g_stConfInfo.numOfKey1++; g_stConfInfo.numOfKey1++;
} }
@ -235,7 +236,6 @@ void parseInputString() {
} }
} }
static int running = 1; static int running = 1;
/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ /*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/
@ -253,6 +253,7 @@ int queryDB(TAOS* taos, char* command) {
} }
tmq_t* build_consumer() { tmq_t* build_consumer() {
#if 0
char sqlStr[1024] = {0}; char sqlStr[1024] = {0};
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
@ -266,13 +267,19 @@ tmq_t* build_consumer() {
exit(-1); exit(-1);
} }
taos_free_result(pRes); taos_free_result(pRes);
#endif
tmq_conf_t* conf = tmq_conf_new(); tmq_conf_t* conf = tmq_conf_new();
// tmq_conf_set(conf, "group.id", "tg2"); // tmq_conf_set(conf, "group.id", "tg2");
for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) { for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) {
tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]); tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]);
} }
tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
assert(tmq);
tmq_conf_destroy(conf);
return tmq; return tmq;
} }
@ -285,8 +292,8 @@ tmq_list_t* build_topic_list() {
return topic_list; return topic_list;
} }
tmq_t* build_consumer_x() { tmq_t* build_consumer_x() {
#if 0
char sqlStr[1024] = {0}; char sqlStr[1024] = {0};
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
@ -300,19 +307,25 @@ tmq_t* build_consumer_x() {
exit(-1); exit(-1);
} }
taos_free_result(pRes); taos_free_result(pRes);
#endif
tmq_conf_t* conf = tmq_conf_new(); tmq_conf_t* conf = tmq_conf_new();
//tmq_conf_set(conf, "group.id", "tg2"); // tmq_conf_set(conf, "group.id", "tg2");
for (int32_t i = 0; i < g_stConfInfo.numOfKey1; i++) { for (int32_t i = 0; i < g_stConfInfo.numOfKey1; i++) {
tmq_conf_set(conf, g_stConfInfo.key1[i], g_stConfInfo.value1[i]); tmq_conf_set(conf, g_stConfInfo.key1[i], g_stConfInfo.value1[i]);
} }
tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
assert(tmq);
tmq_conf_destroy(conf);
return tmq; return tmq;
} }
tmq_list_t* build_topic_list_x() { tmq_list_t* build_topic_list_x() {
tmq_list_t* topic_list = tmq_list_new(); tmq_list_t* topic_list = tmq_list_new();
//tmq_list_append(topic_list, "test_stb_topic_1"); // tmq_list_append(topic_list, "test_stb_topic_1");
for (int32_t i = 0; i < g_stConfInfo.numOfTopic1; i++) { for (int32_t i = 0; i < g_stConfInfo.numOfTopic1; i++) {
tmq_list_append(topic_list, g_stConfInfo.topics1[i]); tmq_list_append(topic_list, g_stConfInfo.topics1[i]);
} }
@ -367,9 +380,9 @@ int32_t parallel_consume(tmq_t* tmq, int threadLable) {
if (tmqMsg) { if (tmqMsg) {
totalMsgs++; totalMsgs++;
//printf("threadFlag: %d, totalMsgs: %d\n", threadLable, totalMsgs); // printf("threadFlag: %d, totalMsgs: %d\n", threadLable, totalMsgs);
#if 0 #if 0
TAOS_ROW row; TAOS_ROW row;
while (NULL != (row = tmq_get_row(tmqMsg))) { while (NULL != (row = tmq_get_row(tmqMsg))) {
totalRows++; totalRows++;
@ -396,19 +409,18 @@ int32_t parallel_consume(tmq_t* tmq, int threadLable) {
exit(-1); exit(-1);
} }
//printf("%d", totalMsgs); // output to sim for check result // printf("%d", totalMsgs); // output to sim for check result
return totalMsgs; return totalMsgs;
} }
void* threadFunc(void* param) {
void *threadFunc(void *param) {
int32_t totalMsgs = 0; int32_t totalMsgs = 0;
SThreadInfo *pInfo = (SThreadInfo *)param; SThreadInfo* pInfo = (SThreadInfo*)param;
tmq_t* tmq = build_consumer_x(); tmq_t* tmq = build_consumer_x();
tmq_list_t* topic_list = build_topic_list_x(); tmq_list_t* topic_list = build_topic_list_x();
if ((NULL == tmq) || (NULL == topic_list)){ if ((NULL == tmq) || (NULL == topic_list)) {
return NULL; return NULL;
} }
@ -418,9 +430,9 @@ void *threadFunc(void *param) {
exit(-1); exit(-1);
} }
//if (0 == g_stConfInfo.consumeMsgCnt) { // if (0 == g_stConfInfo.consumeMsgCnt) {
// loop_consume(tmq); // loop_consume(tmq);
//} else { // } else {
pInfo->consumeMsgCnt = parallel_consume(tmq, 1); pInfo->consumeMsgCnt = parallel_consume(tmq, 1);
//} //}
@ -434,7 +446,6 @@ void *threadFunc(void *param) {
return NULL; return NULL;
} }
int main(int32_t argc, char* argv[]) { int main(int32_t argc, char* argv[]) {
parseArgument(argc, argv); parseArgument(argc, argv);
parseInputString(); parseInputString();
@ -443,14 +454,14 @@ int main(int32_t argc, char* argv[]) {
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); SThreadInfo* pInfo = (SThreadInfo*)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo));
if (g_stConfInfo.numOfTopic1) { if (g_stConfInfo.numOfTopic1) {
// pthread_create one thread to consume // pthread_create one thread to consume
for (int32_t i = 0; i < numOfThreads; ++i) { for (int32_t i = 0; i < numOfThreads; ++i) {
pInfo[i].expectMsgCnt = 0; pInfo[i].expectMsgCnt = 0;
pInfo[i].consumeMsgCnt = 0; pInfo[i].consumeMsgCnt = 0;
taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void*)(pInfo + i));
} }
} }
@ -484,7 +495,7 @@ int main(int32_t argc, char* argv[]) {
taosThreadJoin(pInfo[i].thread, NULL); taosThreadJoin(pInfo[i].thread, NULL);
} }
//printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
if (0 == g_stConfInfo.checkMode) { if (0 == g_stConfInfo.checkMode) {
if ((totalMsgs + pInfo->consumeMsgCnt) == g_stConfInfo.consumeMsgCnt) { if ((totalMsgs + pInfo->consumeMsgCnt) == g_stConfInfo.consumeMsgCnt) {
printf("success"); printf("success");
@ -510,10 +521,10 @@ int main(int32_t argc, char* argv[]) {
printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt);
} }
} else if (4 == g_stConfInfo.checkMode) { } else if (4 == g_stConfInfo.checkMode) {
if (((totalMsgs == 0) && (pInfo->consumeMsgCnt == 3 * g_stConfInfo.consumeMsgCnt)) if (((totalMsgs == 0) && (pInfo->consumeMsgCnt == 3 * g_stConfInfo.consumeMsgCnt)) ||
|| ((pInfo->consumeMsgCnt == 0) && (totalMsgs == 3 * g_stConfInfo.consumeMsgCnt)) ((pInfo->consumeMsgCnt == 0) && (totalMsgs == 3 * g_stConfInfo.consumeMsgCnt)) ||
|| ((pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt) && (totalMsgs == 2 * g_stConfInfo.consumeMsgCnt)) ((pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt) && (totalMsgs == 2 * g_stConfInfo.consumeMsgCnt)) ||
|| ((pInfo->consumeMsgCnt == 2 * g_stConfInfo.consumeMsgCnt) && (totalMsgs == g_stConfInfo.consumeMsgCnt))) { ((pInfo->consumeMsgCnt == 2 * g_stConfInfo.consumeMsgCnt) && (totalMsgs == g_stConfInfo.consumeMsgCnt))) {
printf("success"); printf("success");
} else { } else {
printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt);